From 009098701a57f19ab67daef6dfaa5af01a4677b8 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Wed, 12 Jun 2019 13:19:37 -0400 Subject: [PATCH 01/12] Update CI Execution Time (#141) This PR removes redundant sleep calls after printing and moves the dependency installation to CI server startup. Commit log: * Save performance in ci * Add flask to install * Add error handling in run_ci and fix docs * Delete new line --- ci/README.md | 2 +- ci/manager.sh | 23 --------------- ci/run_ci.sh | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 24 deletions(-) create mode 100755 ci/run_ci.sh diff --git a/ci/README.md b/ci/README.md index e490ea5d2..a84513c52 100644 --- a/ci/README.md +++ b/ci/README.md @@ -3,7 +3,7 @@ ### Setting up CI Run a Flask server that listens for new events from github, will get triggered when a new PR is created or when keyword `@onvm` is mentioned. ```sh -python3 webhook-receiver.py 0.0.0.0 8080 @onvm webhook-config.json +./run_ci.sh 0.0.0.0 8080 @onvm webhook-config.json ``` To run CI tests manually, requires a config file, the github PR ID, request message and a response message. diff --git a/ci/manager.sh b/ci/manager.sh index a4b9bbd96..d8a830d92 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -47,7 +47,6 @@ fi . $1 # source the variables from config file -print_header "ONVM Performance Testing" print_header "Checking Required Variables" if [[ -z "$WORKER_LIST" ]] @@ -100,28 +99,6 @@ done # turn off error checking set +e -print_header "Ensuring Dependencies are Installed" - -sudo apt install -y python3 -sudo apt install -y python3-pip - -python3 -V -check_exit_code "ERROR: Python not installed" - -pip3 -V -check_exit_code "ERROR: Pip not installed" - -sudo -H pip3 install virtualenv --upgrade - -sudo -H pip3 install paramiko -check_exit_code "ERROR: Paramiko not installed or failed to install" - -sudo -H pip3 install --pre github3.py -check_exit_code "ERROR: GitHub3.py not installed or failed to install" - -sudo -H pip3 install pexpect -check_exit_code "ERROR: Pexpect not installed or failed to install" - print_header "Fetching and Checking Out Pull Request" python3 clone-and-checkout-pr.py $GITHUB_CREDS "{\"id\": $PR_ID}" $REPO_OWNER $REPO_NAME check_exit_code "ERROR: Failed to fetch and checkout pull request" diff --git a/ci/run_ci.sh b/ci/run_ci.sh new file mode 100755 index 000000000..4b05c4e3d --- /dev/null +++ b/ci/run_ci.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +# exit on error +set -e + +# source helper functions file +. helper-functions.sh +SCRIPT_LOC=$(pwd) + +print_header "Validating Input Variables" + +if [[ -z "$1" ]] +then + echo "ERROR: Missing first argument, Host!" + exit 1 +else + HOST=$1 +fi + +if [[ -z "$2" ]] +then + echo "ERROR: Missing second argument, Flask server port!" + exit 1 +else + PORT=$2 +fi + +if [[ -z "$3" ]] +then + echo "ERROR: Missing third argument, Request Keyword!" + exit 1 +else + KEYWORD=$3 +fi + +if [[ -z "$4" ]] +then + echo "ERROR: Missing fourth argument, path to config file!" + exit 1 +else + if [[ ! -f $4 ]] + then + echo "ERROR: Fourth argument, Config file, is not a file!" + exit 1 + else + CFG_NAME=$4 + fi +fi + +# install all dependencies only once at beginning of CI +print_header "Installing dependencies" + +sudo apt update +sudo apt install -y build-essential libssl-dev libffi-dev +sudo apt install -y python3 +sudo apt install -y python3-pip + +python3 -V +check_exit_code "ERROR: Python not installed" + +pip3 -V +check_exit_code "ERROR: Pip not installed" + +sudo -H pip3 install virtualenv --upgrade +check_exit_code "ERROR: virtualenv had trouble upgrading" + +sudo -H pip3 install flask --upgrade +check_exit_code "ERROR: Flask not installed or failed to install" + +sudo -H pip3 install --pre github3.py +check_exit_code "ERROR: GitHub3.py not installed or failed to install" + +sudo -H pip3 install paramiko +check_exit_code "ERROR: Paramiko not installed or failed to install" + +sudo -H pip3 install pexpect +check_exit_code "ERROR: Pexpect not installed or failed to install" + +print_header "Done Installing, running CI" + +# run the web server with the input arguments +python3 webhook-receiver.py $HOST $PORT $KEYWORD $CFG_NAME From 247d1c05d8e7851ac50aa734fce8562375e21eb6 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Wed, 12 Jun 2019 13:36:53 -0400 Subject: [PATCH 02/12] Allow Basic CI Checks for Unauthorized Users (#140) This PR allows CI to run the following for unauthorized users: - Linter check - Branch check Commit log: * Save performance in ci * Add flask to install * Allowed for public linting of unauthorized users * Added run mode for future advancements * Remove exit statement * Add error handling in run_ci and fix docs * Delete new line --- ci/helper-functions.sh | 2 -- ci/manager.sh | 17 +++++++++++++++++ ci/post-msg.py | 15 ++++++++------- ci/webhook-receiver.py | 9 ++++----- 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/ci/helper-functions.sh b/ci/helper-functions.sh index 6a269e54e..0e28417f5 100644 --- a/ci/helper-functions.sh +++ b/ci/helper-functions.sh @@ -36,8 +36,6 @@ print_header() { done echo "--" echo "" - - sleep 1 } # sets up dpdk, sets env variables, and runs the install script diff --git a/ci/manager.sh b/ci/manager.sh index d8a830d92..dce469143 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -45,6 +45,14 @@ else REQUEST=$4 fi +if [[ -z "$5" ]] +then + echo "ERROR: Missing fifth argument, Run mode type!" + exit 1 +else + RUN_MODE=$5 +fi + . $1 # source the variables from config file print_header "Checking Required Variables" @@ -109,6 +117,15 @@ rm -f ../linter-output.txt run_linter ../linter-output.txt cd .. +if [[ "$RUN_MODE" -eq "1" ]] +then + # only run linter and develop checks if unauthorized + print_header "Posting Results in Comment on GitHub" + python3 post-msg.py $GITHUB_CREDS "{\"id\": $PR_ID,\"request\":\"$REQUEST\",\"linter\": 1,\"review\": 1}" $REPO_OWNER $REPO_NAME "Run successful see results:" + check_exit_code "ERROR: Failed to post results to GitHub" + exit 0 +fi + print_header "Preparing Workers" for worker_tuple in "${WORKER_LIST[@]}" diff --git a/ci/post-msg.py b/ci/post-msg.py index 3c0f8771d..b9b1124c9 100644 --- a/ci/post-msg.py +++ b/ci/post-msg.py @@ -61,13 +61,14 @@ ACTION = 'REQUEST_CHANGES' # PR must not affect performance - with open('./results_summary.stats') as f: - results = json.load(f) - if (results['pass_performance_check']): - comment_body += " :heavy_check_mark: Speed tester performance check passed\n" - else: - comment_body += " :x: PR drops speed tester perforamce bellow minimum requirement\n" - ACTION = 'REQUEST_CHANGES' + if POST_RESULTS: + with open('./results_summary.stats') as f: + results = json.load(f) + if (results['pass_performance_check']): + comment_body += " :heavy_check_mark: Speed tester performance check passed\n" + else: + comment_body += " :x: PR drops speed tester perforamce below minimum requirement\n" + ACTION = 'REQUEST_CHANGES' # PR must pass linter check linter_output = None diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index e6230b6e4..1549a9d7a 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -160,6 +160,7 @@ def filter_to_prs_and_pr_comments(json): @app.route(EVENT_URL, methods=['POST']) def init_ci_pipeline(): + run_mode = 0 request_ctx = filter_to_prs_and_pr_comments(request.json) if request_ctx is None: logging.debug("Request filter doesn't match request") @@ -180,11 +181,9 @@ def init_ci_pipeline(): return jsonify({"success": True}) if (request_ctx['repo'] == 'openNetVM' and request_ctx['user'] not in authorized_users): + run_mode = 1 + # not an authorized user, tell manager to only run linter print("Incoming request is from an unathorized user") - log_access_denied(request_ctx, "Incoming request is from an unathorized user") - os.system("./ci_busy.sh config {} \"{}\" \"{}\" \"User not authorized to run CI, please contact one of the repo maintainers\"" - .format(request_ctx['id'], request_ctx['repo'], request_ctx['body'])) - return jsonify({"success": True}) print("Request matches filter, we should RUN CI. {}".format(get_request_info(request_ctx))) @@ -202,7 +201,7 @@ def init_ci_pipeline(): .format(request_ctx['id'], request_ctx['repo'], request_ctx['body'])) else: log_access_granted(request_ctx, "Running CI") - os.system("./manager.sh config {} \"{}\" \"{}\"".format(request_ctx['id'], request_ctx['repo'], request_ctx['body'])) + os.system("./manager.sh config {} \"{}\" \"{}\" {}".format(request_ctx['id'], request_ctx['repo'], request_ctx['body'], run_mode)) return jsonify({"status": "ONLINE"}) From 5a40435f9a721b1923728340a0d33f8ec91e0c73 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Fri, 28 Jun 2019 18:06:22 -0400 Subject: [PATCH 03/12] Add CI Request Queue (#143) Previously CI didn't have an execution queue, thus, every request sent when CI was running was disregarded. This pr implements a queue of requests for CI to resolve this issue. Commit log: * Save performance in ci * Add flask to install * Allowed for public linting of unauthorized users * Starting the queue * Start to fixing the queue problem * Added run mode for future advancements * Major fixes to ci list system * Remove debug lines * Clean up definitions * Remove exit statement * Add error handling in run_ci and fix docs * Create polling thread for requests * Delete new line * Account for run mode from #140 * Errors are much more difficult in threads :( * Smarter than using tuples * Added event-based handling instead of polling * Let request handler clears event --- ci/webhook-receiver.py | 84 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 16 deletions(-) diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index 1549a9d7a..f97a00a8b 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -13,6 +13,8 @@ import os import subprocess import logging +import time +import threading # Global vars EVENT_URL = "/github-webhook" @@ -23,6 +25,10 @@ secret_file_name = None private_key_file = None secret = None +queue_lock = None +request_event = None +ci_request_list = None +manager_running = False app = Flask(__name__) @@ -158,6 +164,33 @@ def filter_to_prs_and_pr_comments(json): return None +# run the manager, and rest of CI process +def run_manager(request_ctx): + # requests are tuples of (context, run mode) + log_access_granted(request_ctx, "Running CI") + os.system("./manager.sh config {} \"{}\" \"{}\" {}".format(request_ctx['id'], request_ctx['repo'], request_ctx['body'], request_ctx['mode'])) + +def request_handler(): + global manager_running + while True: + if ci_request_list: + # we have a request + manager_running = True + run_manager(ci_request_list.pop(0)) + # remove flag for other threads + manager_running = False + if not ci_request_list: + # list empty, go to sleep until signaled + request_event.wait() + # remove the flag, so we don't get stuck in loop + request_event.clear() + +def add_request(request_ctx): + ci_request_list.append(request_ctx) + if not request_event.isSet(): + # wake up event listener, if it was asleep + request_event.set() + @app.route(EVENT_URL, methods=['POST']) def init_ci_pipeline(): run_mode = 0 @@ -187,21 +220,31 @@ def init_ci_pipeline(): print("Request matches filter, we should RUN CI. {}".format(get_request_info(request_ctx))) - # Check if there is another CI run in progress - proc1 = subprocess.Popen(['ps', 'cax'], stdout=subprocess.PIPE) - proc2 = subprocess.Popen(['grep', 'manager.sh'], stdin=proc1.stdout, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - proc1.stdout.close() - out, err = proc2.communicate() - - if (out): - print("Can't run CI, another CI run in progress") - log_access_granted(request_ctx, "CI busy, posting busy msg") - os.system("./ci_busy.sh config {} \"{}\" \"{}\" \"Another CI run in progress, please try again in 15 minutes\"" - .format(request_ctx['id'], request_ctx['repo'], request_ctx['body'])) - else: - log_access_granted(request_ctx, "Running CI") - os.system("./manager.sh config {} \"{}\" \"{}\" {}".format(request_ctx['id'], request_ctx['repo'], request_ctx['body'], run_mode)) + request_ctx['mode'] = run_mode + + duplicate_req = False + + if (manager_running or ci_request_list): + busy_msg = "Another CI run in progress, adding request to the end of the list" + log_access_granted(request_ctx, "CI busy, placing request in queue") + if ci_request_list: + # only check for duplicates if list isn't empty + with queue_lock: + for req in ci_request_list: + # make sure this is the same PR + if req['id'] == request_ctx['id'] and req['repo'] == request_ctx['repo']: + duplicate_req = True + break + # ending frees the lock + if duplicate_req: + # let user know we're not running this request, it's a duplicate + busy_msg = "Duplicate request already waiting, ignoring message" + os.system("./ci_busy.sh config {} \"{}\" \"{}\" \"{}\"" + .format(request_ctx['id'], request_ctx['repo'], request_ctx['body'], busy_msg)) + + if not duplicate_req: + # we are not a duplicate, add to list + add_request(request_ctx) return jsonify({"status": "ONLINE"}) @@ -250,8 +293,17 @@ def parse_config(cfg_name): cfg_name = sys.argv[4] parse_config(cfg_name) - secret = decrypt_secret() + queue_lock = threading.Lock() + ci_request_list = [] + + # create an event handler for requests + request_event = threading.Event() + # dedicate thread to check for and run new requests + poll_request = threading.Thread(target=request_handler, args=[]) + # run as daemon to catch ^C termination + poll_request.daemon = True + poll_request.start() logging.info("Starting the CI service") app.run(host=host, port=port) From 6bf1c9f9319bfd362f44e84ed101de0717ef52a2 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Fri, 28 Jun 2019 18:17:07 -0400 Subject: [PATCH 04/12] Add Macro for Running Flow Table Lookup (#147) Running a flow table lookup on every packet is expensive and as the functionality isn't always used we have decided to create a macro to disable it. The functionality will still be enabled by default, but if it is unused disabling it should boost performance. Commit log: * Macros for running flow table lookup * Updated comment about disabling flow table lookup --- onvm/onvm_mgr/Makefile | 7 +++++++ onvm/onvm_mgr/onvm_pkt.c | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/onvm/onvm_mgr/Makefile b/onvm/onvm_mgr/Makefile index af332529b..90cda1fa2 100644 --- a/onvm/onvm_mgr/Makefile +++ b/onvm/onvm_mgr/Makefile @@ -35,6 +35,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# set this to 0 to disable flow table lookup for incoming packets +ENABLE_FLOW_LOOKUP=1 + ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif @@ -62,6 +65,10 @@ CFLAGS += -I$(SRCDIR)/../ -I$(SRCDIR)/../onvm_nflib/ -I$(SRCDIR)/../lib/ LDFLAGS += $(SRCDIR)/../lib/$(RTE_TARGET)/libonvmhelper.a LDFLAGS += $(SRCDIR)/../onvm_nflib/$(RTE_TARGET)/libonvm.a +ifeq ($(ENABLE_FLOW_LOOKUP), 1) +CFLAGS +=-D FLOW_LOOKUP +endif + # for newer gcc, e.g. 4.4, no-strict-aliasing may not be necessary # and so the next line can be removed in those cases. EXTRA_CFLAGS += -fno-strict-aliasing diff --git a/onvm/onvm_mgr/onvm_pkt.c b/onvm/onvm_mgr/onvm_pkt.c index 43a702591..9e6076e87 100644 --- a/onvm/onvm_mgr/onvm_pkt.c +++ b/onvm/onvm_mgr/onvm_pkt.c @@ -57,9 +57,11 @@ void onvm_pkt_process_rx_batch(struct queue_mgr *rx_mgr, struct rte_mbuf *pkts[], uint16_t rx_count) { uint16_t i; struct onvm_pkt_meta *meta; +#ifdef FLOW_LOOKUP struct onvm_flow_entry *flow_entry; struct onvm_service_chain *sc; int ret; +#endif if (rx_mgr == NULL || pkts == NULL) return; @@ -68,15 +70,19 @@ onvm_pkt_process_rx_batch(struct queue_mgr *rx_mgr, struct rte_mbuf *pkts[], uin meta = (struct onvm_pkt_meta *)&(((struct rte_mbuf *)pkts[i])->udata64); meta->src = 0; meta->chain_index = 0; +#ifdef FLOW_LOOKUP ret = onvm_flow_dir_get_pkt(pkts[i], &flow_entry); if (ret >= 0) { sc = flow_entry->sc; meta->action = onvm_sc_next_action(sc, pkts[i]); meta->destination = onvm_sc_next_destination(sc, pkts[i]); } else { +#endif meta->action = onvm_sc_next_action(default_chain, pkts[i]); meta->destination = onvm_sc_next_destination(default_chain, pkts[i]); +#ifdef FLOW_LOOKUP } +#endif /* PERF: this might hurt performance since it will cause cache * invalidations. Ideally the data modified by the NF manager * would be a different line than that modified/read by NFs. From 4122e10fcaa19a1ea7a2e21ecc4e669eb5d0ce05 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Sun, 7 Jul 2019 15:33:03 -0400 Subject: [PATCH 05/12] Update cJSON library to version 1.7.12 (#126) Updates the cJSON library version to v1.7.12 , doesn't have any collisions with onvm code. --- onvm/lib/cJSON.c | 3414 +++++++++++++++++++++++++++------------------- onvm/lib/cJSON.h | 239 +++- 2 files changed, 2205 insertions(+), 1448 deletions(-) diff --git a/onvm/lib/cJSON.c b/onvm/lib/cJSON.c index a01d31936..60b72c018 100644 --- a/onvm/lib/cJSON.c +++ b/onvm/lib/cJSON.c @@ -1,5 +1,5 @@ /* - Copyright (c) 2009 Dave Gamble + Copyright (c) 2009-2017 Dave Gamble and cJSON contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -23,42 +23,76 @@ /* cJSON */ /* JSON parser in C. */ +/* disable warnings about old C89 functions in MSVC */ +#if !defined(_CRT_SECURE_NO_DEPRECATE) && defined(_MSC_VER) +#define _CRT_SECURE_NO_DEPRECATE +#endif + +#ifdef __GNUC__ +#pragma GCC visibility push(default) +#endif +#if defined(_MSC_VER) +#pragma warning (push) +/* disable warning about single line comments in system headers */ +#pragma warning (disable : 4001) +#endif + #include #include #include #include -#include #include #include -#include "cJSON.h" -/* Determine the number of bits that an integer has using the preprocessor */ -#if INT_MAX == 32767 - /* 16 bits */ - #define INTEGER_SIZE 0x0010 -#elif INT_MAX == 2147483647 - /* 32 bits */ - #define INTEGER_SIZE 0x0100 -#elif INT_MAX == 9223372036854775807 - /* 64 bits */ - #define INTEGER_SIZE 0x1000 -#else - #error "Failed to determine the size of an integer" +#ifdef ENABLE_LOCALES +#include +#endif + +#if defined(_MSC_VER) +#pragma warning (pop) +#endif +#ifdef __GNUC__ +#pragma GCC visibility pop #endif +#include "cJSON.h" + /* define our own boolean type */ -typedef int cjbool; -#define true ((cjbool)1) -#define false ((cjbool)0) +#ifdef true +#undef true +#endif +#define true ((cJSON_bool)1) + +#ifdef false +#undef false +#endif +#define false ((cJSON_bool)0) -static const char *global_ep = NULL; +typedef struct { + const unsigned char *json; + size_t position; +} error; +static error global_error = { NULL, 0 }; -const char *cJSON_GetErrorPtr(void) +CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void) { - return global_ep; + return (const char*) (global_error.json + global_error.position); +} + +CJSON_PUBLIC(char *) cJSON_GetStringValue(cJSON *item) { + if (!cJSON_IsString(item)) { + return NULL; + } + + return item->valuestring; } -extern const char* cJSON_Version(void) +/* This is a safeguard to prevent copy-pasters from using incompatible C and header files */ +#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || (CJSON_VERSION_PATCH != 12) + #error cJSON.h and cJSON.c have different versions. Make sure that both have the same. +#endif + +CJSON_PUBLIC(const char*) cJSON_Version(void) { static char version[15]; sprintf(version, "%i.%i.%i", CJSON_VERSION_MAJOR, CJSON_VERSION_MINOR, CJSON_VERSION_PATCH); @@ -66,69 +100,118 @@ extern const char* cJSON_Version(void) return version; } -/* case insensitive strcmp */ -static int cJSON_strcasecmp(const char *s1, const char *s2) +/* Case insensitive string comparison, doesn't consider two NULL pointers equal though */ +static int case_insensitive_strcmp(const unsigned char *string1, const unsigned char *string2) { - if (!s1) + if ((string1 == NULL) || (string2 == NULL)) { - return (s1 == s2) ? 0 : 1; /* both NULL? */ + return 1; } - if (!s2) + + if (string1 == string2) { - return 1; + return 0; } - for(; tolower(*(const unsigned char *)s1) == tolower(*(const unsigned char *)s2); ++s1, ++s2) + + for(; tolower(*string1) == tolower(*string2); (void)string1++, string2++) { - if (*s1 == '\0') + if (*string1 == '\0') { return 0; } } - return tolower(*(const unsigned char *)s1) - tolower(*(const unsigned char *)s2); + return tolower(*string1) - tolower(*string2); +} + +typedef struct internal_hooks +{ + void *(CJSON_CDECL *allocate)(size_t size); + void (CJSON_CDECL *deallocate)(void *pointer); + void *(CJSON_CDECL *reallocate)(void *pointer, size_t size); +} internal_hooks; + +#if defined(_MSC_VER) +/* work around MSVC error C2322: '...' address of dillimport '...' is not static */ +static void * CJSON_CDECL internal_malloc(size_t size) +{ + return malloc(size); } +static void CJSON_CDECL internal_free(void *pointer) +{ + free(pointer); +} +static void * CJSON_CDECL internal_realloc(void *pointer, size_t size) +{ + return realloc(pointer, size); +} +#else +#define internal_malloc malloc +#define internal_free free +#define internal_realloc realloc +#endif -static void *(*cJSON_malloc)(size_t sz) = malloc; -static void (*cJSON_free)(void *ptr) = free; +/* strlen of character literals resolved at compile time */ +#define static_strlen(string_literal) (sizeof(string_literal) - sizeof("")) -static char* cJSON_strdup(const char* str) +static internal_hooks global_hooks = { internal_malloc, internal_free, internal_realloc }; + +static unsigned char* cJSON_strdup(const unsigned char* string, const internal_hooks * const hooks) { - size_t len = 0; - char *copy = NULL; + size_t length = 0; + unsigned char *copy = NULL; - if (str == NULL) + if (string == NULL) { return NULL; } - len = strlen(str) + 1; - if (!(copy = (char*)cJSON_malloc(len))) + length = strlen((const char*)string) + sizeof(""); + copy = (unsigned char*)hooks->allocate(length); + if (copy == NULL) { return NULL; } - memcpy(copy, str, len); + memcpy(copy, string, length); return copy; } -void cJSON_InitHooks(cJSON_Hooks* hooks) +CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks) { - if (!hooks) + if (hooks == NULL) { /* Reset hooks */ - cJSON_malloc = malloc; - cJSON_free = free; + global_hooks.allocate = malloc; + global_hooks.deallocate = free; + global_hooks.reallocate = realloc; return; } - cJSON_malloc = (hooks->malloc_fn) ? hooks->malloc_fn : malloc; - cJSON_free = (hooks->free_fn) ? hooks->free_fn : free; + global_hooks.allocate = malloc; + if (hooks->malloc_fn != NULL) + { + global_hooks.allocate = hooks->malloc_fn; + } + + global_hooks.deallocate = free; + if (hooks->free_fn != NULL) + { + global_hooks.deallocate = hooks->free_fn; + } + + /* use realloc only if both free and malloc are used */ + global_hooks.reallocate = NULL; + if ((global_hooks.allocate == malloc) && (global_hooks.deallocate == free)) + { + global_hooks.reallocate = realloc; + } } /* Internal constructor. */ -static cJSON *cJSON_New_Item(void) +static cJSON *cJSON_New_Item(const internal_hooks * const hooks) { - cJSON* node = (cJSON*)cJSON_malloc(sizeof(cJSON)); + cJSON* node = (cJSON*)hooks->allocate(sizeof(cJSON)); if (node) { memset(node, '\0', sizeof(cJSON)); @@ -138,138 +221,188 @@ static cJSON *cJSON_New_Item(void) } /* Delete a cJSON structure. */ -void cJSON_Delete(cJSON *c) +CJSON_PUBLIC(void) cJSON_Delete(cJSON *item) { cJSON *next = NULL; - while (c) + while (item != NULL) { - next = c->next; - if (!(c->type & cJSON_IsReference) && c->child) + next = item->next; + if (!(item->type & cJSON_IsReference) && (item->child != NULL)) { - cJSON_Delete(c->child); + cJSON_Delete(item->child); } - if (!(c->type & cJSON_IsReference) && c->valuestring) + if (!(item->type & cJSON_IsReference) && (item->valuestring != NULL)) { - cJSON_free(c->valuestring); + global_hooks.deallocate(item->valuestring); } - if (!(c->type & cJSON_StringIsConst) && c->string) + if (!(item->type & cJSON_StringIsConst) && (item->string != NULL)) { - cJSON_free(c->string); + global_hooks.deallocate(item->string); } - cJSON_free(c); - c = next; + global_hooks.deallocate(item); + item = next; } } -/* Parse the input text to generate a number, and populate the result into item. */ -static const char *parse_number(cJSON *item, const char *num) +/* get the decimal point character of the current locale */ +static unsigned char get_decimal_point(void) { - double n = 0; - double sign = 1; - double scale = 0; - int subscale = 0; - int signsubscale = 1; +#ifdef ENABLE_LOCALES + struct lconv *lconv = localeconv(); + return (unsigned char) lconv->decimal_point[0]; +#else + return '.'; +#endif +} - /* Has sign? */ - if (*num == '-') - { - sign = -1; - num++; +typedef struct +{ + const unsigned char *content; + size_t length; + size_t offset; + size_t depth; /* How deeply nested (in arrays/objects) is the input at the current offset. */ + internal_hooks hooks; +} parse_buffer; + +/* check if the given size is left to read in a given parse buffer (starting with 1) */ +#define can_read(buffer, size) ((buffer != NULL) && (((buffer)->offset + size) <= (buffer)->length)) +/* check if the buffer can be accessed at the given index (starting with 0) */ +#define can_access_at_index(buffer, index) ((buffer != NULL) && (((buffer)->offset + index) < (buffer)->length)) +#define cannot_access_at_index(buffer, index) (!can_access_at_index(buffer, index)) +/* get a pointer to the buffer at the position */ +#define buffer_at_offset(buffer) ((buffer)->content + (buffer)->offset) + +/* Parse the input text to generate a number, and populate the result into item. */ +static cJSON_bool parse_number(cJSON * const item, parse_buffer * const input_buffer) +{ + double number = 0; + unsigned char *after_end = NULL; + unsigned char number_c_string[64]; + unsigned char decimal_point = get_decimal_point(); + size_t i = 0; + + if ((input_buffer == NULL) || (input_buffer->content == NULL)) + { + return false; + } + + /* copy the number into a temporary buffer and replace '.' with the decimal point + * of the current locale (for strtod) + * This also takes care of '\0' not necessarily being available for marking the end of the input */ + for (i = 0; (i < (sizeof(number_c_string) - 1)) && can_access_at_index(input_buffer, i); i++) + { + switch (buffer_at_offset(input_buffer)[i]) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case '+': + case '-': + case 'e': + case 'E': + number_c_string[i] = buffer_at_offset(input_buffer)[i]; + break; + + case '.': + number_c_string[i] = decimal_point; + break; + + default: + goto loop_end; + } } - /* is zero */ - if (*num == '0') +loop_end: + number_c_string[i] = '\0'; + + number = strtod((const char*)number_c_string, (char**)&after_end); + if (number_c_string == after_end) { - num++; + return false; /* parse_error */ } - /* Number? */ - if ((*num >= '1') && (*num <= '9')) + + item->valuedouble = number; + + /* use saturation in case of overflow */ + if (number >= INT_MAX) { - do - { - n = (n * 10.0) + (*num++ - '0'); - } - while ((*num >= '0') && (*num<='9')); + item->valueint = INT_MAX; } - /* Fractional part? */ - if ((*num == '.') && (num[1] >= '0') && (num[1] <= '9')) + else if (number <= (double)INT_MIN) { - num++; - do - { - n = (n *10.0) + (*num++ - '0'); - scale--; - } while ((*num >= '0') && (*num <= '9')); + item->valueint = INT_MIN; } - /* Exponent? */ - if ((*num == 'e') || (*num == 'E')) + else { - num++; - /* With sign? */ - if (*num == '+') - { - num++; - } - else if (*num == '-') - { - signsubscale = -1; - num++; - } - /* Number? */ - while ((*num>='0') && (*num<='9')) - { - subscale = (subscale * 10) + (*num++ - '0'); - } + item->valueint = (int)number; } - /* number = +/- number.fraction * 10^+/- exponent */ - n = sign * n * pow(10.0, (scale + subscale * signsubscale)); - - item->valuedouble = n; - item->valueint = (int)n; item->type = cJSON_Number; - return num; + input_buffer->offset += (size_t)(after_end - number_c_string); + return true; } -/* calculate the next largest power of 2 */ -static int pow2gt (int x) +/* don't ask me, but the original cJSON_SetNumberValue returns an integer or double */ +CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number) { - --x; - - x |= x >> 1; - x |= x >> 2; - x |= x >> 4; -#if INTEGER_SIZE & 0x1110 /* at least 16 bit */ - x |= x >> 8; -#endif -#if INTEGER_SIZE & 0x1100 /* at least 32 bit */ - x |= x >> 16; -#endif -#if INTEGER_SIZE & 0x1000 /* 64 bit */ - x |= x >> 32; -#endif + if (number >= INT_MAX) + { + object->valueint = INT_MAX; + } + else if (number <= (double)INT_MIN) + { + object->valueint = INT_MIN; + } + else + { + object->valueint = (int)number; + } - return x + 1; + return object->valuedouble = number; } typedef struct { - char *buffer; - int length; - int offset; - cjbool noalloc; + unsigned char *buffer; + size_t length; + size_t offset; + size_t depth; /* current nesting depth (for formatted printing) */ + cJSON_bool noalloc; + cJSON_bool format; /* is this print a formatted print */ + internal_hooks hooks; } printbuffer; /* realloc printbuffer if necessary to have at least "needed" bytes more */ -static char* ensure(printbuffer *p, int needed) +static unsigned char* ensure(printbuffer * const p, size_t needed) { - char *newbuffer = NULL; - int newsize = 0; - if (!p || !p->buffer) + unsigned char *newbuffer = NULL; + size_t newsize = 0; + + if ((p == NULL) || (p->buffer == NULL)) + { + return NULL; + } + + if ((p->length > 0) && (p->offset >= p->length)) + { + /* make sure that offset is valid */ + return NULL; + } + + if (needed > INT_MAX) { + /* sizes bigger than INT_MAX are currently not supported */ return NULL; } - needed += p->offset; + + needed += p->offset + 1; if (needed <= p->length) { return p->buffer + p->offset; @@ -279,1446 +412,1416 @@ static char* ensure(printbuffer *p, int needed) return NULL; } - newsize = pow2gt(needed); - newbuffer = (char*)cJSON_malloc(newsize); - if (!newbuffer) + /* calculate new buffer size */ + if (needed > (INT_MAX / 2)) { - cJSON_free(p->buffer); - p->length = 0; - p->buffer = NULL; + /* overflow of int, use INT_MAX if possible */ + if (needed <= INT_MAX) + { + newsize = INT_MAX; + } + else + { + return NULL; + } + } + else + { + newsize = needed * 2; + } - return NULL; + if (p->hooks.reallocate != NULL) + { + /* reallocate with realloc if available */ + newbuffer = (unsigned char*)p->hooks.reallocate(p->buffer, newsize); + if (newbuffer == NULL) + { + p->hooks.deallocate(p->buffer); + p->length = 0; + p->buffer = NULL; + + return NULL; + } } - if (newbuffer) + else { - memcpy(newbuffer, p->buffer, p->length); + /* otherwise reallocate manually */ + newbuffer = (unsigned char*)p->hooks.allocate(newsize); + if (!newbuffer) + { + p->hooks.deallocate(p->buffer); + p->length = 0; + p->buffer = NULL; + + return NULL; + } + if (newbuffer) + { + memcpy(newbuffer, p->buffer, p->offset + 1); + } + p->hooks.deallocate(p->buffer); } - cJSON_free(p->buffer); p->length = newsize; p->buffer = newbuffer; return newbuffer + p->offset; } -/* calculate the new length of the string in a printbuffer */ -static int update(const printbuffer *p) +/* calculate the new length of the string in a printbuffer and update the offset */ +static void update_offset(printbuffer * const buffer) { - char *str = NULL; - if (!p || !p->buffer) + const unsigned char *buffer_pointer = NULL; + if ((buffer == NULL) || (buffer->buffer == NULL)) { - return 0; + return; } - str = p->buffer + p->offset; + buffer_pointer = buffer->buffer + buffer->offset; - return p->offset + strlen(str); + buffer->offset += strlen((const char*)buffer_pointer); } /* Render the number nicely from the given item into a string. */ -static char *print_number(const cJSON *item, printbuffer *p) +static cJSON_bool print_number(const cJSON * const item, printbuffer * const output_buffer) { - char *str = NULL; + unsigned char *output_pointer = NULL; double d = item->valuedouble; - /* special case for 0. */ - if (d == 0) + int length = 0; + size_t i = 0; + unsigned char number_buffer[26]; /* temporary buffer to print the number into */ + unsigned char decimal_point = get_decimal_point(); + double test; + + if (output_buffer == NULL) { - if (p) - { - str = ensure(p, 2); - } - else - { - str = (char*)cJSON_malloc(2); - } - if (str) + return false; + } + + /* This checks for NaN and Infinity */ + if ((d * 0) != 0) + { + length = sprintf((char*)number_buffer, "null"); + } + else + { + /* Try 15 decimal places of precision to avoid nonsignificant nonzero digits */ + length = sprintf((char*)number_buffer, "%1.15g", d); + + /* Check whether the original double can be recovered */ + if ((sscanf((char*)number_buffer, "%lg", &test) != 1) || ((double)test != d)) { - strcpy(str,"0"); + /* If not, print with 17 decimal places of precision */ + length = sprintf((char*)number_buffer, "%1.17g", d); } } - /* value is an int */ - else if ((fabs(((double)item->valueint) - d) <= DBL_EPSILON) && (d <= INT_MAX) && (d >= INT_MIN)) + + /* sprintf failed or buffer overrun occurred */ + if ((length < 0) || (length > (int)(sizeof(number_buffer) - 1))) + { + return false; + } + + /* reserve appropriate space in the output */ + output_pointer = ensure(output_buffer, (size_t)length + sizeof("")); + if (output_pointer == NULL) + { + return false; + } + + /* copy the printed number to the output and replace locale + * dependent decimal point with '.' */ + for (i = 0; i < ((size_t)length); i++) { - if (p) + if (number_buffer[i] == decimal_point) { - str = ensure(p, 21); + output_pointer[i] = '.'; + continue; } - else + + output_pointer[i] = number_buffer[i]; + } + output_pointer[i] = '\0'; + + output_buffer->offset += (size_t)length; + + return true; +} + +/* parse 4 digit hexadecimal number */ +static unsigned parse_hex4(const unsigned char * const input) +{ + unsigned int h = 0; + size_t i = 0; + + for (i = 0; i < 4; i++) + { + /* parse digit */ + if ((input[i] >= '0') && (input[i] <= '9')) { - /* 2^64+1 can be represented in 21 chars. */ - str = (char*)cJSON_malloc(21); + h += (unsigned int) input[i] - '0'; } - if (str) + else if ((input[i] >= 'A') && (input[i] <= 'F')) { - sprintf(str, "%d", item->valueint); + h += (unsigned int) 10 + input[i] - 'A'; } - } - /* value is a floating point number */ - else - { - if (p) + else if ((input[i] >= 'a') && (input[i] <= 'f')) { - /* This is a nice tradeoff. */ - str = ensure(p, 64); + h += (unsigned int) 10 + input[i] - 'a'; } - else + else /* invalid */ { - /* This is a nice tradeoff. */ - str=(char*)cJSON_malloc(64); + return 0; } - if (str) + + if (i < 3) { - /* This checks for NaN and Infinity */ - if ((d * 0) != 0) - { - sprintf(str, "null"); - } - else if ((fabs(floor(d) - d) <= DBL_EPSILON) && (fabs(d) < 1.0e60)) - { - sprintf(str, "%.0f", d); - } - else if ((fabs(d) < 1.0e-6) || (fabs(d) > 1.0e9)) - { - sprintf(str, "%e", d); - } - else - { - sprintf(str, "%f", d); - } + /* shift left to make place for the next nibble */ + h = h << 4; } } - return str; + + return h; } -/* parse 4 digit hexadecimal number */ -static unsigned parse_hex4(const char *str) +/* converts a UTF-16 literal to UTF-8 + * A literal can be one or two sequences of the form \uXXXX */ +static unsigned char utf16_literal_to_utf8(const unsigned char * const input_pointer, const unsigned char * const input_end, unsigned char **output_pointer) { - unsigned h = 0; - /* first digit */ - if ((*str >= '0') && (*str <= '9')) - { - h += (*str) - '0'; - } - else if ((*str >= 'A') && (*str <= 'F')) - { - h += 10 + (*str) - 'A'; - } - else if ((*str >= 'a') && (*str <= 'f')) - { - h += 10 + (*str) - 'a'; - } - else /* invalid */ + long unsigned int codepoint = 0; + unsigned int first_code = 0; + const unsigned char *first_sequence = input_pointer; + unsigned char utf8_length = 0; + unsigned char utf8_position = 0; + unsigned char sequence_length = 0; + unsigned char first_byte_mark = 0; + + if ((input_end - first_sequence) < 6) { - return 0; + /* input ends unexpectedly */ + goto fail; } + /* get the first utf16 sequence */ + first_code = parse_hex4(first_sequence + 2); - /* second digit */ - h = h << 4; - str++; - if ((*str >= '0') && (*str <= '9')) - { - h += (*str) - '0'; - } - else if ((*str >= 'A') && (*str <= 'F')) + /* check that the code is valid */ + if (((first_code >= 0xDC00) && (first_code <= 0xDFFF))) { - h += 10 + (*str) - 'A'; + goto fail; } - else if ((*str >= 'a') && (*str <= 'f')) + + /* UTF16 surrogate pair */ + if ((first_code >= 0xD800) && (first_code <= 0xDBFF)) { - h += 10 + (*str) - 'a'; + const unsigned char *second_sequence = first_sequence + 6; + unsigned int second_code = 0; + sequence_length = 12; /* \uXXXX\uXXXX */ + + if ((input_end - second_sequence) < 6) + { + /* input ends unexpectedly */ + goto fail; + } + + if ((second_sequence[0] != '\\') || (second_sequence[1] != 'u')) + { + /* missing second half of the surrogate pair */ + goto fail; + } + + /* get the second utf16 sequence */ + second_code = parse_hex4(second_sequence + 2); + /* check that the code is valid */ + if ((second_code < 0xDC00) || (second_code > 0xDFFF)) + { + /* invalid second half of the surrogate pair */ + goto fail; + } + + + /* calculate the unicode codepoint from the surrogate pair */ + codepoint = 0x10000 + (((first_code & 0x3FF) << 10) | (second_code & 0x3FF)); } - else /* invalid */ + else { - return 0; + sequence_length = 6; /* \uXXXX */ + codepoint = first_code; } - /* third digit */ - h = h << 4; - str++; - if ((*str >= '0') && (*str <= '9')) + /* encode as UTF-8 + * takes at maximum 4 bytes to encode: + * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ + if (codepoint < 0x80) { - h += (*str) - '0'; + /* normal ascii, encoding 0xxxxxxx */ + utf8_length = 1; } - else if ((*str >= 'A') && (*str <= 'F')) + else if (codepoint < 0x800) { - h += 10 + (*str) - 'A'; + /* two bytes, encoding 110xxxxx 10xxxxxx */ + utf8_length = 2; + first_byte_mark = 0xC0; /* 11000000 */ } - else if ((*str >= 'a') && (*str <= 'f')) + else if (codepoint < 0x10000) { - h += 10 + (*str) - 'a'; + /* three bytes, encoding 1110xxxx 10xxxxxx 10xxxxxx */ + utf8_length = 3; + first_byte_mark = 0xE0; /* 11100000 */ } - else /* invalid */ + else if (codepoint <= 0x10FFFF) { - return 0; + /* four bytes, encoding 1110xxxx 10xxxxxx 10xxxxxx 10xxxxxx */ + utf8_length = 4; + first_byte_mark = 0xF0; /* 11110000 */ } - - /* fourth digit */ - h = h << 4; - str++; - if ((*str >= '0') && (*str <= '9')) + else { - h += (*str) - '0'; + /* invalid unicode codepoint */ + goto fail; } - else if ((*str >= 'A') && (*str <= 'F')) + + /* encode as utf8 */ + for (utf8_position = (unsigned char)(utf8_length - 1); utf8_position > 0; utf8_position--) { - h += 10 + (*str) - 'A'; + /* 10xxxxxx */ + (*output_pointer)[utf8_position] = (unsigned char)((codepoint | 0x80) & 0xBF); + codepoint >>= 6; } - else if ((*str >= 'a') && (*str <= 'f')) + /* encode first byte */ + if (utf8_length > 1) { - h += 10 + (*str) - 'a'; + (*output_pointer)[0] = (unsigned char)((codepoint | first_byte_mark) & 0xFF); } - else /* invalid */ + else { - return 0; + (*output_pointer)[0] = (unsigned char)(codepoint & 0x7F); } - return h; -} + *output_pointer += utf8_length; -/* first bytes of UTF8 encoding for a given length in bytes */ -static const unsigned char firstByteMark[7] = -{ - 0x00, /* should never happen */ - 0x00, /* 0xxxxxxx */ - 0xC0, /* 110xxxxx */ - 0xE0, /* 1110xxxx */ - 0xF0, /* 11110xxx */ - 0xF8, - 0xFC -}; + return sequence_length; + +fail: + return 0; +} -/* Parse the input text into an unescaped cstring, and populate item. */ -static const char *parse_string(cJSON *item, const char *str, const char **ep) +/* Parse the input text into an unescaped cinput, and populate item. */ +static cJSON_bool parse_string(cJSON * const item, parse_buffer * const input_buffer) { - const char *ptr = str + 1; - const char *end_ptr =str + 1; - char *ptr2 = NULL; - char *out = NULL; - int len = 0; - unsigned uc = 0; - unsigned uc2 = 0; + const unsigned char *input_pointer = buffer_at_offset(input_buffer) + 1; + const unsigned char *input_end = buffer_at_offset(input_buffer) + 1; + unsigned char *output_pointer = NULL; + unsigned char *output = NULL; - /* not a string! */ - if (*str != '\"') + /* not a string */ + if (buffer_at_offset(input_buffer)[0] != '\"') { - *ep = str; - return NULL; + goto fail; } - while ((*end_ptr != '\"') && *end_ptr) { - if (*end_ptr++ == '\\') + /* calculate approximate size of the output (overestimate) */ + size_t allocation_length = 0; + size_t skipped_bytes = 0; + while (((size_t)(input_end - input_buffer->content) < input_buffer->length) && (*input_end != '\"')) { - if (*end_ptr == '\0') + /* is escape sequence */ + if (input_end[0] == '\\') { - /* prevent buffer overflow when last input character is a backslash */ - return NULL; + if ((size_t)(input_end + 1 - input_buffer->content) >= input_buffer->length) + { + /* prevent buffer overflow when last input character is a backslash */ + goto fail; + } + skipped_bytes++; + input_end++; } - /* Skip escaped quotes. */ - end_ptr++; + input_end++; + } + if (((size_t)(input_end - input_buffer->content) >= input_buffer->length) || (*input_end != '\"')) + { + goto fail; /* string ended unexpectedly */ } - len++; - } - /* This is at most how long we need for the string, roughly. */ - out = (char*)cJSON_malloc(len + 1); - if (!out) - { - return NULL; + /* This is at most how much we need for the output */ + allocation_length = (size_t) (input_end - buffer_at_offset(input_buffer)) - skipped_bytes; + output = (unsigned char*)input_buffer->hooks.allocate(allocation_length + sizeof("")); + if (output == NULL) + { + goto fail; /* allocation failure */ + } } - item->valuestring = out; /* assign here so out will be deleted during cJSON_Delete() later */ - item->type = cJSON_String; - ptr = str + 1; - ptr2 = out; + output_pointer = output; /* loop through the string literal */ - while (ptr < end_ptr) + while (input_pointer < input_end) { - if (*ptr != '\\') + if (*input_pointer != '\\') { - *ptr2++ = *ptr++; + *output_pointer++ = *input_pointer++; } /* escape sequence */ else { - ptr++; - switch (*ptr) + unsigned char sequence_length = 2; + if ((input_end - input_pointer) < 1) + { + goto fail; + } + + switch (input_pointer[1]) { case 'b': - *ptr2++ = '\b'; + *output_pointer++ = '\b'; break; case 'f': - *ptr2++ = '\f'; + *output_pointer++ = '\f'; break; case 'n': - *ptr2++ = '\n'; + *output_pointer++ = '\n'; break; case 'r': - *ptr2++ = '\r'; + *output_pointer++ = '\r'; break; case 't': - *ptr2++ = '\t'; + *output_pointer++ = '\t'; break; case '\"': case '\\': case '/': - *ptr2++ = *ptr; + *output_pointer++ = input_pointer[1]; break; - case 'u': - /* transcode utf16 to utf8. See RFC2781 and RFC3629. */ - uc = parse_hex4(ptr + 1); /* get the unicode char. */ - ptr += 4; - if (ptr >= end_ptr) - { - /* invalid */ - *ep = str; - return NULL; - } - /* check for invalid. */ - if (((uc >= 0xDC00) && (uc <= 0xDFFF)) || (uc == 0)) - { - *ep = str; - return NULL; - } - - /* UTF16 surrogate pairs. */ - if ((uc >= 0xD800) && (uc<=0xDBFF)) - { - if ((ptr + 6) > end_ptr) - { - /* invalid */ - *ep = str; - return NULL; - } - if ((ptr[1] != '\\') || (ptr[2] != 'u')) - { - /* missing second-half of surrogate. */ - *ep = str; - return NULL; - } - uc2 = parse_hex4(ptr + 3); - ptr += 6; /* \uXXXX */ - if ((uc2 < 0xDC00) || (uc2 > 0xDFFF)) - { - /* invalid second-half of surrogate. */ - *ep = str; - return NULL; - } - /* calculate unicode codepoint from the surrogate pair */ - uc = 0x10000 + (((uc & 0x3FF) << 10) | (uc2 & 0x3FF)); - } - /* encode as UTF8 - * takes at maximum 4 bytes to encode: - * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ - len = 4; - if (uc < 0x80) - { - /* normal ascii, encoding 0xxxxxxx */ - len = 1; - } - else if (uc < 0x800) - { - /* two bytes, encoding 110xxxxx 10xxxxxx */ - len = 2; - } - else if (uc < 0x10000) + /* UTF-16 literal */ + case 'u': + sequence_length = utf16_literal_to_utf8(input_pointer, input_end, &output_pointer); + if (sequence_length == 0) { - /* three bytes, encoding 1110xxxx 10xxxxxx 10xxxxxx */ - len = 3; - } - ptr2 += len; - - switch (len) { - case 4: - /* 10xxxxxx */ - *--ptr2 = ((uc | 0x80) & 0xBF); - uc >>= 6; - /* FALLTHROUGH */ - case 3: - /* 10xxxxxx */ - *--ptr2 = ((uc | 0x80) & 0xBF); - uc >>= 6; - /* FALLTHROUGH */ - case 2: - /* 10xxxxxx */ - *--ptr2 = ((uc | 0x80) & 0xBF); - uc >>= 6; - /* FALLTHROUGH */ - case 1: - /* depending on the length in bytes this determines the - * encoding ofthe first UTF8 byte */ - *--ptr2 = (uc | firstByteMark[len]); + /* failed to convert UTF16-literal to UTF-8 */ + goto fail; } - ptr2 += len; break; + default: - *ep = str; - return NULL; + goto fail; } - ptr++; + input_pointer += sequence_length; } } - *ptr2 = '\0'; - if (*ptr == '\"') + + /* zero terminate the output */ + *output_pointer = '\0'; + + item->type = cJSON_String; + item->valuestring = (char*)output; + + input_buffer->offset = (size_t) (input_end - input_buffer->content); + input_buffer->offset++; + + return true; + +fail: + if (output != NULL) + { + input_buffer->hooks.deallocate(output); + } + + if (input_pointer != NULL) { - ptr++; + input_buffer->offset = (size_t)(input_pointer - input_buffer->content); } - return ptr; + return false; } /* Render the cstring provided to an escaped version that can be printed. */ -static char *print_string_ptr(const char *str, printbuffer *p) +static cJSON_bool print_string_ptr(const unsigned char * const input, printbuffer * const output_buffer) { - const char *ptr = NULL; - char *ptr2 = NULL; - char *out = NULL; - int len = 0; - cjbool flag = false; - unsigned char token = '\0'; + const unsigned char *input_pointer = NULL; + unsigned char *output = NULL; + unsigned char *output_pointer = NULL; + size_t output_length = 0; + /* numbers of additional characters needed for escaping */ + size_t escape_characters = 0; + + if (output_buffer == NULL) + { + return false; + } /* empty string */ - if (!str) + if (input == NULL) { - if (p) - { - out = ensure(p, 3); - } - else + output = ensure(output_buffer, sizeof("\"\"")); + if (output == NULL) { - out = (char*)cJSON_malloc(3); + return false; } - if (!out) - { - return NULL; - } - strcpy(out, "\"\""); + strcpy((char*)output, "\"\""); - return out; + return true; } /* set "flag" to 1 if something needs to be escaped */ - for (ptr = str; *ptr; ptr++) - { - flag |= (((*ptr > 0) && (*ptr < 32)) /* unprintable characters */ - || (*ptr == '\"') /* double quote */ - || (*ptr == '\\')) /* backslash */ - ? 1 - : 0; - } - /* no characters have to be escaped */ - if (!flag) - { - len = ptr - str; - if (p) - { - out = ensure(p, len + 3); - } - else - { - out = (char*)cJSON_malloc(len + 3); - } - if (!out) - { - return NULL; + for (input_pointer = input; *input_pointer; input_pointer++) + { + switch (*input_pointer) + { + case '\"': + case '\\': + case '\b': + case '\f': + case '\n': + case '\r': + case '\t': + /* one character escape sequence */ + escape_characters++; + break; + default: + if (*input_pointer < 32) + { + /* UTF-16 escape sequence uXXXX */ + escape_characters += 5; + } + break; } - - ptr2 = out; - *ptr2++ = '\"'; - strcpy(ptr2, str); - ptr2[len] = '\"'; - ptr2[len + 1] = '\0'; - - return out; } + output_length = (size_t)(input_pointer - input) + escape_characters; - ptr = str; - /* calculate additional space that is needed for escaping */ - while ((token = *ptr)) + output = ensure(output_buffer, output_length + sizeof("\"\"")); + if (output == NULL) { - ++len; - if (strchr("\"\\\b\f\n\r\t", token)) - { - len++; /* +1 for the backslash */ - } - else if (token < 32) - { - len += 5; /* +5 for \uXXXX */ - } - ptr++; + return false; } - if (p) - { - out = ensure(p, len + 3); - } - else - { - out = (char*)cJSON_malloc(len + 3); - } - if (!out) + /* no characters have to be escaped */ + if (escape_characters == 0) { - return NULL; + output[0] = '\"'; + memcpy(output + 1, input, output_length); + output[output_length + 1] = '\"'; + output[output_length + 2] = '\0'; + + return true; } - ptr2 = out; - ptr = str; - *ptr2++ = '\"'; + output[0] = '\"'; + output_pointer = output + 1; /* copy the string */ - while (*ptr) + for (input_pointer = input; *input_pointer != '\0'; (void)input_pointer++, output_pointer++) { - if (((unsigned char)*ptr > 31) && (*ptr != '\"') && (*ptr != '\\')) + if ((*input_pointer > 31) && (*input_pointer != '\"') && (*input_pointer != '\\')) { /* normal character, copy */ - *ptr2++ = *ptr++; + *output_pointer = *input_pointer; } else { /* character needs to be escaped */ - *ptr2++ = '\\'; - switch (token = *ptr++) + *output_pointer++ = '\\'; + switch (*input_pointer) { case '\\': - *ptr2++ = '\\'; + *output_pointer = '\\'; break; case '\"': - *ptr2++ = '\"'; + *output_pointer = '\"'; break; case '\b': - *ptr2++ = 'b'; + *output_pointer = 'b'; break; case '\f': - *ptr2++ = 'f'; + *output_pointer = 'f'; break; case '\n': - *ptr2++ = 'n'; + *output_pointer = 'n'; break; case '\r': - *ptr2++ = 'r'; + *output_pointer = 'r'; break; case '\t': - *ptr2++ = 't'; + *output_pointer = 't'; break; default: /* escape and print as unicode codepoint */ - sprintf(ptr2, "u%04x", token); - ptr2 += 5; + sprintf((char*)output_pointer, "u%04x", *input_pointer); + output_pointer += 4; break; } } } - *ptr2++ = '\"'; - *ptr2++ = '\0'; + output[output_length + 1] = '\"'; + output[output_length + 2] = '\0'; - return out; + return true; } /* Invoke print_string_ptr (which is useful) on an item. */ -static char *print_string(const cJSON *item, printbuffer *p) +static cJSON_bool print_string(const cJSON * const item, printbuffer * const p) { - return print_string_ptr(item->valuestring, p); + return print_string_ptr((unsigned char*)item->valuestring, p); } /* Predeclare these prototypes. */ -static const char *parse_value(cJSON *item, const char *value, const char **ep); -static char *print_value(const cJSON *item, int depth, cjbool fmt, printbuffer *p); -static const char *parse_array(cJSON *item, const char *value, const char **ep); -static char *print_array(const cJSON *item, int depth, cjbool fmt, printbuffer *p); -static const char *parse_object(cJSON *item, const char *value, const char **ep); -static char *print_object(const cJSON *item, int depth, cjbool fmt, printbuffer *p); +static cJSON_bool parse_value(cJSON * const item, parse_buffer * const input_buffer); +static cJSON_bool print_value(const cJSON * const item, printbuffer * const output_buffer); +static cJSON_bool parse_array(cJSON * const item, parse_buffer * const input_buffer); +static cJSON_bool print_array(const cJSON * const item, printbuffer * const output_buffer); +static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_buffer); +static cJSON_bool print_object(const cJSON * const item, printbuffer * const output_buffer); /* Utility to jump whitespace and cr/lf */ -static const char *skip(const char *in) +static parse_buffer *buffer_skip_whitespace(parse_buffer * const buffer) { - while (in && *in && ((unsigned char)*in<=32)) + if ((buffer == NULL) || (buffer->content == NULL)) { - in++; + return NULL; + } + + while (can_access_at_index(buffer, 0) && (buffer_at_offset(buffer)[0] <= 32)) + { + buffer->offset++; } - return in; + if (buffer->offset == buffer->length) + { + buffer->offset--; + } + + return buffer; } -/* Parse an object - create a new root, and populate. */ -cJSON *cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cjbool require_null_terminated) +/* skip the UTF-8 BOM (byte order mark) if it is at the beginning of a buffer */ +static parse_buffer *skip_utf8_bom(parse_buffer * const buffer) { - const char *end = NULL; - /* use global error pointer if no specific one was given */ - const char **ep = return_parse_end ? return_parse_end : &global_ep; - cJSON *c = cJSON_New_Item(); - *ep = NULL; - if (!c) /* memory fail */ + if ((buffer == NULL) || (buffer->content == NULL) || (buffer->offset != 0)) { return NULL; } - end = parse_value(c, skip(value), ep); - if (!end) + if (can_access_at_index(buffer, 4) && (strncmp((const char*)buffer_at_offset(buffer), "\xEF\xBB\xBF", 3) == 0)) + { + buffer->offset += 3; + } + + return buffer; +} + +/* Parse an object - create a new root, and populate. */ +CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cJSON_bool require_null_terminated) +{ + parse_buffer buffer = { 0, 0, 0, 0, { 0, 0, 0 } }; + cJSON *item = NULL; + + /* reset error position */ + global_error.json = NULL; + global_error.position = 0; + + if (value == NULL) + { + goto fail; + } + + buffer.content = (const unsigned char*)value; + buffer.length = strlen((const char*)value) + sizeof(""); + buffer.offset = 0; + buffer.hooks = global_hooks; + + item = cJSON_New_Item(&global_hooks); + if (item == NULL) /* memory fail */ + { + goto fail; + } + + if (!parse_value(item, buffer_skip_whitespace(skip_utf8_bom(&buffer)))) { /* parse failure. ep is set. */ - cJSON_Delete(c); - return NULL; + goto fail; } /* if we require null-terminated JSON without appended garbage, skip and then check for a null terminator */ if (require_null_terminated) { - end = skip(end); - if (*end) + buffer_skip_whitespace(&buffer); + if ((buffer.offset >= buffer.length) || buffer_at_offset(&buffer)[0] != '\0') { - cJSON_Delete(c); - *ep = end; - return NULL; + goto fail; } } if (return_parse_end) { - *return_parse_end = end; + *return_parse_end = (const char*)buffer_at_offset(&buffer); + } + + return item; + +fail: + if (item != NULL) + { + cJSON_Delete(item); + } + + if (value != NULL) + { + error local_error; + local_error.json = (const unsigned char*)value; + local_error.position = 0; + + if (buffer.offset < buffer.length) + { + local_error.position = buffer.offset; + } + else if (buffer.length > 0) + { + local_error.position = buffer.length - 1; + } + + if (return_parse_end != NULL) + { + *return_parse_end = (const char*)local_error.json + local_error.position; + } + + global_error = local_error; } - return c; + return NULL; } /* Default options for cJSON_Parse */ -cJSON *cJSON_Parse(const char *value) +CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value) { return cJSON_ParseWithOpts(value, 0, 0); } +#define cjson_min(a, b) ((a < b) ? a : b) + +static unsigned char *print(const cJSON * const item, cJSON_bool format, const internal_hooks * const hooks) +{ + static const size_t default_buffer_size = 256; + printbuffer buffer[1]; + unsigned char *printed = NULL; + + memset(buffer, 0, sizeof(buffer)); + + /* create buffer */ + buffer->buffer = (unsigned char*) hooks->allocate(default_buffer_size); + buffer->length = default_buffer_size; + buffer->format = format; + buffer->hooks = *hooks; + if (buffer->buffer == NULL) + { + goto fail; + } + + /* print the value */ + if (!print_value(item, buffer)) + { + goto fail; + } + update_offset(buffer); + + /* check if reallocate is available */ + if (hooks->reallocate != NULL) + { + printed = (unsigned char*) hooks->reallocate(buffer->buffer, buffer->offset + 1); + if (printed == NULL) { + goto fail; + } + buffer->buffer = NULL; + } + else /* otherwise copy the JSON over to a new buffer */ + { + printed = (unsigned char*) hooks->allocate(buffer->offset + 1); + if (printed == NULL) + { + goto fail; + } + memcpy(printed, buffer->buffer, cjson_min(buffer->length, buffer->offset + 1)); + printed[buffer->offset] = '\0'; /* just to be sure */ + + /* free the buffer */ + hooks->deallocate(buffer->buffer); + } + + return printed; + +fail: + if (buffer->buffer != NULL) + { + hooks->deallocate(buffer->buffer); + } + + if (printed != NULL) + { + hooks->deallocate(printed); + } + + return NULL; +} + /* Render a cJSON item/entity/structure to text. */ -char *cJSON_Print(const cJSON *item) +CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item) { - return print_value(item, 0, 1, 0); + return (char*)print(item, true, &global_hooks); } -char *cJSON_PrintUnformatted(const cJSON *item) +CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item) { - return print_value(item, 0, 0, 0); + return (char*)print(item, false, &global_hooks); } -char *cJSON_PrintBuffered(const cJSON *item, int prebuffer, cjbool fmt) +CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt) { - printbuffer p; - p.buffer = (char*)cJSON_malloc(prebuffer); + printbuffer p = { 0, 0, 0, 0, 0, 0, { 0, 0, 0 } }; + + if (prebuffer < 0) + { + return NULL; + } + + p.buffer = (unsigned char*)global_hooks.allocate((size_t)prebuffer); if (!p.buffer) { return NULL; } - p.length = prebuffer; + + p.length = (size_t)prebuffer; p.offset = 0; p.noalloc = false; + p.format = fmt; + p.hooks = global_hooks; + + if (!print_value(item, &p)) + { + global_hooks.deallocate(p.buffer); + return NULL; + } - return print_value(item, 0, fmt, &p); + return (char*)p.buffer; } -int cJSON_PrintPreallocated(cJSON *item,char *buf, const int len, const cjbool fmt) +CJSON_PUBLIC(cJSON_bool) cJSON_PrintPreallocated(cJSON *item, char *buf, const int len, const cJSON_bool fmt) { - printbuffer p; - p.buffer = buf; - p.length = len; + printbuffer p = { 0, 0, 0, 0, 0, 0, { 0, 0, 0 } }; + + if ((len < 0) || (buf == NULL)) + { + return false; + } + + p.buffer = (unsigned char*)buf; + p.length = (size_t)len; p.offset = 0; p.noalloc = true; - return print_value(item,0,fmt,&p) != NULL; + p.format = fmt; + p.hooks = global_hooks; + + return print_value(item, &p); } /* Parser core - when encountering text, process appropriately. */ -static const char *parse_value(cJSON *item, const char *value, const char **ep) +static cJSON_bool parse_value(cJSON * const item, parse_buffer * const input_buffer) { - if (!value) + if ((input_buffer == NULL) || (input_buffer->content == NULL)) { - /* Fail on null. */ - return NULL; + return false; /* no input */ } /* parse the different types of values */ - if (!strncmp(value, "null", 4)) + /* null */ + if (can_read(input_buffer, 4) && (strncmp((const char*)buffer_at_offset(input_buffer), "null", 4) == 0)) { item->type = cJSON_NULL; - return value + 4; + input_buffer->offset += 4; + return true; } - if (!strncmp(value, "false", 5)) + /* false */ + if (can_read(input_buffer, 5) && (strncmp((const char*)buffer_at_offset(input_buffer), "false", 5) == 0)) { item->type = cJSON_False; - return value + 5; + input_buffer->offset += 5; + return true; } - if (!strncmp(value, "true", 4)) + /* true */ + if (can_read(input_buffer, 4) && (strncmp((const char*)buffer_at_offset(input_buffer), "true", 4) == 0)) { item->type = cJSON_True; item->valueint = 1; - return value + 4; + input_buffer->offset += 4; + return true; } - if (*value == '\"') + /* string */ + if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '\"')) { - return parse_string(item, value, ep); + return parse_string(item, input_buffer); } - if ((*value == '-') || ((*value >= '0') && (*value <= '9'))) + /* number */ + if (can_access_at_index(input_buffer, 0) && ((buffer_at_offset(input_buffer)[0] == '-') || ((buffer_at_offset(input_buffer)[0] >= '0') && (buffer_at_offset(input_buffer)[0] <= '9')))) { - return parse_number(item, value); + return parse_number(item, input_buffer); } - if (*value == '[') + /* array */ + if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '[')) { - return parse_array(item, value, ep); + return parse_array(item, input_buffer); } - if (*value == '{') + /* object */ + if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '{')) { - return parse_object(item, value, ep); + return parse_object(item, input_buffer); } - /* failure. */ - *ep = value; - return NULL; + return false; } /* Render a value to text. */ -static char *print_value(const cJSON *item, int depth, cjbool fmt, printbuffer *p) +static cJSON_bool print_value(const cJSON * const item, printbuffer * const output_buffer) { - char *out = NULL; + unsigned char *output = NULL; - if (!item) + if ((item == NULL) || (output_buffer == NULL)) { - return NULL; + return false; } - if (p) + + switch ((item->type) & 0xFF) { - switch ((item->type) & 0xFF) - { - case cJSON_NULL: - out = ensure(p, 5); - if (out) - { - strcpy(out, "null"); - } - break; - case cJSON_False: - out = ensure(p, 6); - if (out) - { - strcpy(out, "false"); - } - break; - case cJSON_True: - out = ensure(p, 5); - if (out) - { - strcpy(out, "true"); - } - break; - case cJSON_Number: - out = print_number(item, p); - break; - case cJSON_Raw: + case cJSON_NULL: + output = ensure(output_buffer, 5); + if (output == NULL) { - size_t raw_length = 0; - if (item->valuestring == NULL) - { - if (!p->noalloc) - { - cJSON_free(p->buffer); - } - out = NULL; - break; - } + return false; + } + strcpy((char*)output, "null"); + return true; - raw_length = strlen(item->valuestring) + sizeof('\0'); - out = ensure(p, raw_length); - if (out) - { - memcpy(out, item->valuestring, raw_length); - } - break; + case cJSON_False: + output = ensure(output_buffer, 6); + if (output == NULL) + { + return false; } - case cJSON_String: - out = print_string(item, p); - break; - case cJSON_Array: - out = print_array(item, depth, fmt, p); - break; - case cJSON_Object: - out = print_object(item, depth, fmt, p); - break; - } - } - else - { - switch ((item->type) & 0xFF) + strcpy((char*)output, "false"); + return true; + + case cJSON_True: + output = ensure(output_buffer, 5); + if (output == NULL) + { + return false; + } + strcpy((char*)output, "true"); + return true; + + case cJSON_Number: + return print_number(item, output_buffer); + + case cJSON_Raw: { - case cJSON_NULL: - out = cJSON_strdup("null"); - break; - case cJSON_False: - out = cJSON_strdup("false"); - break; - case cJSON_True: - out = cJSON_strdup("true"); - break; - case cJSON_Number: - out = print_number(item, 0); - break; - case cJSON_Raw: - out = cJSON_strdup(item->valuestring); - break; - case cJSON_String: - out = print_string(item, 0); - break; - case cJSON_Array: - out = print_array(item, depth, fmt, 0); - break; - case cJSON_Object: - out = print_object(item, depth, fmt, 0); - break; + size_t raw_length = 0; + if (item->valuestring == NULL) + { + return false; + } + + raw_length = strlen(item->valuestring) + sizeof(""); + output = ensure(output_buffer, raw_length); + if (output == NULL) + { + return false; + } + memcpy(output, item->valuestring, raw_length); + return true; } - } - return out; + case cJSON_String: + return print_string(item, output_buffer); + + case cJSON_Array: + return print_array(item, output_buffer); + + case cJSON_Object: + return print_object(item, output_buffer); + + default: + return false; + } } /* Build an array from input text. */ -static const char *parse_array(cJSON *item,const char *value,const char **ep) +static cJSON_bool parse_array(cJSON * const item, parse_buffer * const input_buffer) { - cJSON *child = NULL; - if (*value != '[') + cJSON *head = NULL; /* head of the linked list */ + cJSON *current_item = NULL; + + if (input_buffer->depth >= CJSON_NESTING_LIMIT) { - /* not an array! */ - *ep = value; - return NULL; + return false; /* to deeply nested */ } + input_buffer->depth++; - item->type = cJSON_Array; - value = skip(value + 1); - if (*value == ']') + if (buffer_at_offset(input_buffer)[0] != '[') { - /* empty array. */ - return value + 1; + /* not an array */ + goto fail; } - item->child = child = cJSON_New_Item(); - if (!item->child) + input_buffer->offset++; + buffer_skip_whitespace(input_buffer); + if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ']')) { - /* memory fail */ - return NULL; + /* empty array */ + goto success; } - /* skip any spacing, get the value. */ - value = skip(parse_value(child, skip(value), ep)); - if (!value) + + /* check if we skipped to the end of the buffer */ + if (cannot_access_at_index(input_buffer, 0)) { - return NULL; + input_buffer->offset--; + goto fail; } + /* step back to character in front of the first element */ + input_buffer->offset--; /* loop through the comma separated array elements */ - while (*value == ',') + do { - cJSON *new_item = NULL; - if (!(new_item = cJSON_New_Item())) + /* allocate next item */ + cJSON *new_item = cJSON_New_Item(&(input_buffer->hooks)); + if (new_item == NULL) { - /* memory fail */ - return NULL; + goto fail; /* allocation failure */ } - /* add new item to end of the linked list */ - child->next = new_item; - new_item->prev = child; - child = new_item; - /* go to the next comma */ - value = skip(parse_value(child, skip(value + 1), ep)); - if (!value) + /* attach next item to list */ + if (head == NULL) { - /* memory fail */ - return NULL; + /* start the linked list */ + current_item = head = new_item; + } + else + { + /* add to the end and advance */ + current_item->next = new_item; + new_item->prev = current_item; + current_item = new_item; } + + /* parse next value */ + input_buffer->offset++; + buffer_skip_whitespace(input_buffer); + if (!parse_value(current_item, input_buffer)) + { + goto fail; /* failed to parse value */ + } + buffer_skip_whitespace(input_buffer); } + while (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ',')); - if (*value == ']') + if (cannot_access_at_index(input_buffer, 0) || buffer_at_offset(input_buffer)[0] != ']') { - /* end of array */ - return value + 1; + goto fail; /* expected end of array */ } - /* malformed. */ - *ep = value; +success: + input_buffer->depth--; - return NULL; + item->type = cJSON_Array; + item->child = head; + + input_buffer->offset++; + + return true; + +fail: + if (head != NULL) + { + cJSON_Delete(head); + } + + return false; } /* Render an array to text */ -static char *print_array(const cJSON *item, int depth, cjbool fmt, printbuffer *p) -{ - char **entries; - char *out = NULL; - char *ptr = NULL; - char *ret = NULL; - int len = 5; - cJSON *child = item->child; - int numentries = 0; - int i = 0; - cjbool fail = false; - size_t tmplen = 0; - - /* How many entries in the array? */ - while (child) - { - numentries++; - child = child->next; - } +static cJSON_bool print_array(const cJSON * const item, printbuffer * const output_buffer) +{ + unsigned char *output_pointer = NULL; + size_t length = 0; + cJSON *current_element = item->child; - /* Explicitly handle numentries == 0 */ - if (!numentries) + if (output_buffer == NULL) { - if (p) - { - out = ensure(p, 3); - } - else - { - out = (char*)cJSON_malloc(3); - } - if (out) - { - strcpy(out,"[]"); - } - - return out; + return false; } - if (p) + /* Compose the output array. */ + /* opening square bracket */ + output_pointer = ensure(output_buffer, 1); + if (output_pointer == NULL) { - /* Compose the output array. */ - /* opening square bracket */ - i = p->offset; - ptr = ensure(p, 1); - if (!ptr) - { - return NULL; - } - *ptr = '['; - p->offset++; - - child = item->child; - while (child && !fail) - { - if (!print_value(child, depth + 1, fmt, p)) - { - return NULL; - } - p->offset = update(p); - if (child->next) - { - len = fmt ? 2 : 1; - ptr = ensure(p, len + 1); - if (!ptr) - { - return NULL; - } - *ptr++ = ','; - if(fmt) - { - *ptr++ = ' '; - } - *ptr = '\0'; - p->offset += len; - } - child = child->next; - } - ptr = ensure(p, 2); - if (!ptr) - { - return NULL; - } - *ptr++ = ']'; - *ptr = '\0'; - out = (p->buffer) + i; + return false; } - else - { - /* Allocate an array to hold the pointers to all printed values */ - entries = (char**)cJSON_malloc(numentries * sizeof(char*)); - if (!entries) - { - return NULL; - } - memset(entries, '\0', numentries * sizeof(char*)); - /* Retrieve all the results: */ - child = item->child; - while (child && !fail) - { - ret = print_value(child, depth + 1, fmt, 0); - entries[i++] = ret; - if (ret) - { - len += strlen(ret) + 2 + (fmt ? 1 : 0); - } - else - { - fail = true; - } - child = child->next; - } + *output_pointer = '['; + output_buffer->offset++; + output_buffer->depth++; - /* If we didn't fail, try to malloc the output string */ - if (!fail) - { - out = (char*)cJSON_malloc(len); - } - /* If that fails, we fail. */ - if (!out) + while (current_element != NULL) + { + if (!print_value(current_element, output_buffer)) { - fail = true; + return false; } - - /* Handle failure. */ - if (fail) + update_offset(output_buffer); + if (current_element->next) { - /* free all the entries in the array */ - for (i = 0; i < numentries; i++) + length = (size_t) (output_buffer->format ? 2 : 1); + output_pointer = ensure(output_buffer, length + 1); + if (output_pointer == NULL) { - if (entries[i]) - { - cJSON_free(entries[i]); - } + return false; } - cJSON_free(entries); - return NULL; - } - - /* Compose the output array. */ - *out='['; - ptr = out + 1; - *ptr = '\0'; - for (i = 0; i < numentries; i++) - { - tmplen = strlen(entries[i]); - memcpy(ptr, entries[i], tmplen); - ptr += tmplen; - if (i != (numentries - 1)) + *output_pointer++ = ','; + if(output_buffer->format) { - *ptr++ = ','; - if(fmt) - { - *ptr++ = ' '; - } - *ptr = '\0'; + *output_pointer++ = ' '; } - cJSON_free(entries[i]); + *output_pointer = '\0'; + output_buffer->offset += length; } - cJSON_free(entries); - *ptr++ = ']'; - *ptr++ = '\0'; + current_element = current_element->next; } - return out; + output_pointer = ensure(output_buffer, 2); + if (output_pointer == NULL) + { + return false; + } + *output_pointer++ = ']'; + *output_pointer = '\0'; + output_buffer->depth--; + + return true; } /* Build an object from the text. */ -static const char *parse_object(cJSON *item, const char *value, const char **ep) +static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_buffer) { - cJSON *child = NULL; - if (*value != '{') + cJSON *head = NULL; /* linked list head */ + cJSON *current_item = NULL; + + if (input_buffer->depth >= CJSON_NESTING_LIMIT) { - /* not an object! */ - *ep = value; - return NULL; + return false; /* to deeply nested */ } + input_buffer->depth++; - item->type = cJSON_Object; - value = skip(value + 1); - if (*value == '}') + if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != '{')) { - /* empty object. */ - return value + 1; + goto fail; /* not an object */ } - child = cJSON_New_Item(); - item->child = child; - if (!item->child) + input_buffer->offset++; + buffer_skip_whitespace(input_buffer); + if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '}')) { - return NULL; + goto success; /* empty object */ } - /* parse first key */ - value = skip(parse_string(child, skip(value), ep)); - if (!value) + + /* check if we skipped to the end of the buffer */ + if (cannot_access_at_index(input_buffer, 0)) { - return NULL; + input_buffer->offset--; + goto fail; } - /* use string as key, not value */ - child->string = child->valuestring; - child->valuestring = NULL; - if (*value != ':') + /* step back to character in front of the first element */ + input_buffer->offset--; + /* loop through the comma separated array elements */ + do { - /* invalid object. */ - *ep = value; - return NULL; - } - /* skip any spacing, get the value. */ - value = skip(parse_value(child, skip(value + 1), ep)); - if (!value) - { - return NULL; - } + /* allocate next item */ + cJSON *new_item = cJSON_New_Item(&(input_buffer->hooks)); + if (new_item == NULL) + { + goto fail; /* allocation failure */ + } - while (*value == ',') - { - cJSON *new_item = NULL; - if (!(new_item = cJSON_New_Item())) + /* attach next item to list */ + if (head == NULL) { - /* memory fail */ - return NULL; + /* start the linked list */ + current_item = head = new_item; + } + else + { + /* add to the end and advance */ + current_item->next = new_item; + new_item->prev = current_item; + current_item = new_item; } - /* add to linked list */ - child->next = new_item; - new_item->prev = child; - child = new_item; - value = skip(parse_string(child, skip(value + 1), ep)); - if (!value) + /* parse the name of the child */ + input_buffer->offset++; + buffer_skip_whitespace(input_buffer); + if (!parse_string(current_item, input_buffer)) { - return NULL; + goto fail; /* failed to parse name */ } + buffer_skip_whitespace(input_buffer); - /* use string as key, not value */ - child->string = child->valuestring; - child->valuestring = NULL; + /* swap valuestring and string, because we parsed the name */ + current_item->string = current_item->valuestring; + current_item->valuestring = NULL; - if (*value != ':') + if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != ':')) { - /* invalid object. */ - *ep = value; - return NULL; + goto fail; /* invalid object */ } - /* skip any spacing, get the value. */ - value = skip(parse_value(child, skip(value + 1), ep)); - if (!value) + + /* parse the value */ + input_buffer->offset++; + buffer_skip_whitespace(input_buffer); + if (!parse_value(current_item, input_buffer)) { - return NULL; + goto fail; /* failed to parse value */ } + buffer_skip_whitespace(input_buffer); } - /* end of object */ - if (*value == '}') + while (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ',')); + + if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != '}')) { - return value + 1; + goto fail; /* expected end of object */ } - /* malformed */ - *ep = value; - return NULL; +success: + input_buffer->depth--; + + item->type = cJSON_Object; + item->child = head; + + input_buffer->offset++; + return true; + +fail: + if (head != NULL) + { + cJSON_Delete(head); + } + + return false; } /* Render an object to text. */ -static char *print_object(const cJSON *item, int depth, cjbool fmt, printbuffer *p) -{ - char **entries = NULL; - char **names = NULL; - char *out = NULL; - char *ptr = NULL; - char *ret = NULL; - char *str = NULL; - int len = 7; - int i = 0; - int j = 0; - cJSON *child = item->child; - int numentries = 0; - cjbool fail = false; - size_t tmplen = 0; - - /* Count the number of entries. */ - while (child) - { - numentries++; - child = child->next; - } +static cJSON_bool print_object(const cJSON * const item, printbuffer * const output_buffer) +{ + unsigned char *output_pointer = NULL; + size_t length = 0; + cJSON *current_item = item->child; - /* Explicitly handle empty object case */ - if (!numentries) + if (output_buffer == NULL) { - if (p) - { - out = ensure(p, fmt ? depth + 4 : 3); - } - else - { - out = (char*)cJSON_malloc(fmt ? depth + 4 : 3); - } - if (!out) - { - return NULL; - } - ptr = out; - *ptr++ = '{'; - if (fmt) { - *ptr++ = '\n'; - for (i = 0; i < depth; i++) - { - *ptr++ = '\t'; - } - } - *ptr++ = '}'; - *ptr++ = '\0'; - - return out; + return false; } - if (p) + /* Compose the output: */ + length = (size_t) (output_buffer->format ? 2 : 1); /* fmt: {\n */ + output_pointer = ensure(output_buffer, length + 1); + if (output_pointer == NULL) { - /* Compose the output: */ - i = p->offset; - len = fmt ? 2 : 1; /* fmt: {\n */ - ptr = ensure(p, len + 1); - if (!ptr) - { - return NULL; - } + return false; + } - *ptr++ = '{'; - if (fmt) - { - *ptr++ = '\n'; - } - *ptr = '\0'; - p->offset += len; + *output_pointer++ = '{'; + output_buffer->depth++; + if (output_buffer->format) + { + *output_pointer++ = '\n'; + } + output_buffer->offset += length; - child = item->child; - depth++; - while (child) + while (current_item) + { + if (output_buffer->format) { - if (fmt) - { - ptr = ensure(p, depth); - if (!ptr) - { - return NULL; - } - for (j = 0; j < depth; j++) - { - *ptr++ = '\t'; - } - p->offset += depth; - } - - /* print key */ - if (!print_string_ptr(child->string, p)) - { - return NULL; - } - p->offset = update(p); - - len = fmt ? 2 : 1; - ptr = ensure(p, len); - if (!ptr) - { - return NULL; - } - *ptr++ = ':'; - if (fmt) - { - *ptr++ = '\t'; - } - p->offset+=len; - - /* print value */ - if (!print_value(child, depth, fmt, p)) + size_t i; + output_pointer = ensure(output_buffer, output_buffer->depth); + if (output_pointer == NULL) { - return NULL; - }; - p->offset = update(p); - - /* print comma if not last */ - len = (fmt ? 1 : 0) + (child->next ? 1 : 0); - ptr = ensure(p, len + 1); - if (!ptr) - { - return NULL; + return false; } - if (child->next) + for (i = 0; i < output_buffer->depth; i++) { - *ptr++ = ','; + *output_pointer++ = '\t'; } - - if (fmt) - { - *ptr++ = '\n'; - } - *ptr = '\0'; - p->offset += len; - - child = child->next; + output_buffer->offset += output_buffer->depth; } - ptr = ensure(p, fmt ? (depth + 1) : 2); - if (!ptr) + /* print key */ + if (!print_string_ptr((unsigned char*)current_item->string, output_buffer)) { - return NULL; + return false; } - if (fmt) - { - for (i = 0; i < (depth - 1); i++) - { - *ptr++ = '\t'; - } - } - *ptr++ = '}'; - *ptr = '\0'; - out = (p->buffer) + i; - } - else - { - /* Allocate space for the names and the objects */ - entries = (char**)cJSON_malloc(numentries * sizeof(char*)); - if (!entries) + update_offset(output_buffer); + + length = (size_t) (output_buffer->format ? 2 : 1); + output_pointer = ensure(output_buffer, length); + if (output_pointer == NULL) { - return NULL; + return false; } - names = (char**)cJSON_malloc(numentries * sizeof(char*)); - if (!names) + *output_pointer++ = ':'; + if (output_buffer->format) { - cJSON_free(entries); - return NULL; + *output_pointer++ = '\t'; } - memset(entries, '\0', sizeof(char*) * numentries); - memset(names, '\0', sizeof(char*) * numentries); + output_buffer->offset += length; - /* Collect all the results into our arrays: */ - child = item->child; - depth++; - if (fmt) - { - len += depth; - } - while (child && !fail) + /* print value */ + if (!print_value(current_item, output_buffer)) { - names[i] = str = print_string_ptr(child->string, 0); /* print key */ - entries[i++] = ret = print_value(child, depth, fmt, 0); - if (str && ret) - { - len += strlen(ret) + strlen(str) + 2 + (fmt ? 2 + depth : 0); - } - else - { - fail = true; - } - child = child->next; + return false; } + update_offset(output_buffer); - /* Try to allocate the output string */ - if (!fail) + /* print comma if not last */ + length = ((size_t)(output_buffer->format ? 1 : 0) + (size_t)(current_item->next ? 1 : 0)); + output_pointer = ensure(output_buffer, length + 1); + if (output_pointer == NULL) { - out = (char*)cJSON_malloc(len); + return false; } - if (!out) + if (current_item->next) { - fail = true; + *output_pointer++ = ','; } - /* Handle failure */ - if (fail) + if (output_buffer->format) { - /* free all the printed keys and values */ - for (i = 0; i < numentries; i++) - { - if (names[i]) - { - cJSON_free(names[i]); - } - if (entries[i]) - { - cJSON_free(entries[i]); - } - } - cJSON_free(names); - cJSON_free(entries); - return NULL; + *output_pointer++ = '\n'; } + *output_pointer = '\0'; + output_buffer->offset += length; - /* Compose the output: */ - *out = '{'; - ptr = out + 1; - if (fmt) - { - *ptr++ = '\n'; - } - *ptr = '\0'; - for (i = 0; i < numentries; i++) - { - if (fmt) - { - for (j = 0; j < depth; j++) - { - *ptr++='\t'; - } - } - tmplen = strlen(names[i]); - memcpy(ptr, names[i], tmplen); - ptr += tmplen; - *ptr++ = ':'; - if (fmt) - { - *ptr++ = '\t'; - } - strcpy(ptr, entries[i]); - ptr += strlen(entries[i]); - if (i != (numentries - 1)) - { - *ptr++ = ','; - } - if (fmt) - { - *ptr++ = '\n'; - } - *ptr = '\0'; - cJSON_free(names[i]); - cJSON_free(entries[i]); - } + current_item = current_item->next; + } - cJSON_free(names); - cJSON_free(entries); - if (fmt) + output_pointer = ensure(output_buffer, output_buffer->format ? (output_buffer->depth + 1) : 2); + if (output_pointer == NULL) + { + return false; + } + if (output_buffer->format) + { + size_t i; + for (i = 0; i < (output_buffer->depth - 1); i++) { - for (i = 0; i < (depth - 1); i++) - { - *ptr++ = '\t'; - } + *output_pointer++ = '\t'; } - *ptr++ = '}'; - *ptr++ = '\0'; } + *output_pointer++ = '}'; + *output_pointer = '\0'; + output_buffer->depth--; - return out; + return true; } /* Get Array size/item / object item. */ -int cJSON_GetArraySize(const cJSON *array) +CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array) { - cJSON *c = array->child; - int i = 0; - while(c) + cJSON *child = NULL; + size_t size = 0; + + if (array == NULL) + { + return 0; + } + + child = array->child; + + while(child != NULL) { - i++; - c = c->next; + size++; + child = child->next; } - return i; + + /* FIXME: Can overflow here. Cannot be fixed without breaking the API */ + + return (int)size; } -cJSON *cJSON_GetArrayItem(const cJSON *array, int item) +static cJSON* get_array_item(const cJSON *array, size_t index) { - cJSON *c = array ? array->child : NULL; - while (c && item > 0) + cJSON *current_child = NULL; + + if (array == NULL) { - item--; - c = c->next; + return NULL; } - return c; + current_child = array->child; + while ((current_child != NULL) && (index > 0)) + { + index--; + current_child = current_child->next; + } + + return current_child; } -cJSON *cJSON_GetObjectItem(const cJSON *object, const char *string) +CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index) { - cJSON *c = object ? object->child : NULL; - while (c && cJSON_strcasecmp(c->string, string)) + if (index < 0) + { + return NULL; + } + + return get_array_item(array, (size_t)index); +} + +static cJSON *get_object_item(const cJSON * const object, const char * const name, const cJSON_bool case_sensitive) +{ + cJSON *current_element = NULL; + + if ((object == NULL) || (name == NULL)) + { + return NULL; + } + + current_element = object->child; + if (case_sensitive) + { + while ((current_element != NULL) && (current_element->string != NULL) && (strcmp(name, current_element->string) != 0)) + { + current_element = current_element->next; + } + } + else { - c = c->next; + while ((current_element != NULL) && (case_insensitive_strcmp((const unsigned char*)name, (const unsigned char*)(current_element->string)) != 0)) + { + current_element = current_element->next; + } } - return c; + + if ((current_element == NULL) || (current_element->string == NULL)) { + return NULL; + } + + return current_element; } -cjbool cJSON_HasObjectItem(const cJSON *object,const char *string) +CJSON_PUBLIC(cJSON *) cJSON_GetObjectItem(const cJSON * const object, const char * const string) +{ + return get_object_item(object, string, false); +} + +CJSON_PUBLIC(cJSON *) cJSON_GetObjectItemCaseSensitive(const cJSON * const object, const char * const string) +{ + return get_object_item(object, string, true); +} + +CJSON_PUBLIC(cJSON_bool) cJSON_HasObjectItem(const cJSON *object, const char *string) { return cJSON_GetObjectItem(object, string) ? 1 : 0; } @@ -1731,29 +1834,39 @@ static void suffix_object(cJSON *prev, cJSON *item) } /* Utility for handling references. */ -static cJSON *create_reference(const cJSON *item) +static cJSON *create_reference(const cJSON *item, const internal_hooks * const hooks) { - cJSON *ref = cJSON_New_Item(); - if (!ref) + cJSON *reference = NULL; + if (item == NULL) + { + return NULL; + } + + reference = cJSON_New_Item(hooks); + if (reference == NULL) { return NULL; } - memcpy(ref, item, sizeof(cJSON)); - ref->string = NULL; - ref->type |= cJSON_IsReference; - ref->next = ref->prev = NULL; - return ref; + + memcpy(reference, item, sizeof(cJSON)); + reference->string = NULL; + reference->type |= cJSON_IsReference; + reference->next = reference->prev = NULL; + return reference; } -/* Add item to array/object. */ -void cJSON_AddItemToArray(cJSON *array, cJSON *item) +static cJSON_bool add_item_to_array(cJSON *array, cJSON *item) { - cJSON *c = array->child; - if (!item) + cJSON *child = NULL; + + if ((item == NULL) || (array == NULL)) { - return; + return false; } - if (!c) + + child = array->child; + + if (child == NULL) { /* list is empty, start new one */ array->child = item; @@ -1761,137 +1874,304 @@ void cJSON_AddItemToArray(cJSON *array, cJSON *item) else { /* append to the end */ - while (c->next) + while (child->next) { - c = c->next; + child = child->next; } - suffix_object(c, item); + suffix_object(child, item); } + + return true; } -void cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item) +/* Add item to array/object. */ +CJSON_PUBLIC(void) cJSON_AddItemToArray(cJSON *array, cJSON *item) { - if (!item) + add_item_to_array(array, item); +} + +#if defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))) + #pragma GCC diagnostic push +#endif +#ifdef __GNUC__ +#pragma GCC diagnostic ignored "-Wcast-qual" +#endif +/* helper function to cast away const */ +static void* cast_away_const(const void* string) +{ + return (void*)string; +} +#if defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))) + #pragma GCC diagnostic pop +#endif + + +static cJSON_bool add_item_to_object(cJSON * const object, const char * const string, cJSON * const item, const internal_hooks * const hooks, const cJSON_bool constant_key) +{ + char *new_key = NULL; + int new_type = cJSON_Invalid; + + if ((object == NULL) || (string == NULL) || (item == NULL)) { - return; + return false; } - /* free old key and set new one */ - if (item->string) + if (constant_key) + { + new_key = (char*)cast_away_const(string); + new_type = item->type | cJSON_StringIsConst; + } + else + { + new_key = (char*)cJSON_strdup((const unsigned char*)string, hooks); + if (new_key == NULL) + { + return false; + } + + new_type = item->type & ~cJSON_StringIsConst; + } + + if (!(item->type & cJSON_StringIsConst) && (item->string != NULL)) { - cJSON_free(item->string); + hooks->deallocate(item->string); } - item->string = cJSON_strdup(string); - cJSON_AddItemToArray(object,item); + item->string = new_key; + item->type = new_type; + + return add_item_to_array(object, item); +} + +CJSON_PUBLIC(void) cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item) +{ + add_item_to_object(object, string, item, &global_hooks, false); } /* Add an item to an object with constant string as key */ -void cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item) +CJSON_PUBLIC(void) cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item) { - if (!item) + add_item_to_object(object, string, item, &global_hooks, true); +} + +CJSON_PUBLIC(void) cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item) +{ + if (array == NULL) { return; } - if (!(item->type & cJSON_StringIsConst) && item->string) + + add_item_to_array(array, create_reference(item, &global_hooks)); +} + +CJSON_PUBLIC(void) cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item) +{ + if ((object == NULL) || (string == NULL)) { - cJSON_free(item->string); + return; } -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wcast-qual" - item->string = (char*)string; -#pragma GCC diagnostic pop - item->type |= cJSON_StringIsConst; - cJSON_AddItemToArray(object, item); + + add_item_to_object(object, string, create_reference(item, &global_hooks), &global_hooks, false); } -void cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item) +CJSON_PUBLIC(cJSON*) cJSON_AddNullToObject(cJSON * const object, const char * const name) { - cJSON_AddItemToArray(array, create_reference(item)); + cJSON *null = cJSON_CreateNull(); + if (add_item_to_object(object, name, null, &global_hooks, false)) + { + return null; + } + + cJSON_Delete(null); + return NULL; } -void cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item) +CJSON_PUBLIC(cJSON*) cJSON_AddTrueToObject(cJSON * const object, const char * const name) { - cJSON_AddItemToObject(object, string, create_reference(item)); + cJSON *true_item = cJSON_CreateTrue(); + if (add_item_to_object(object, name, true_item, &global_hooks, false)) + { + return true_item; + } + + cJSON_Delete(true_item); + return NULL; } -cJSON *cJSON_DetachItemFromArray(cJSON *array, int which) +CJSON_PUBLIC(cJSON*) cJSON_AddFalseToObject(cJSON * const object, const char * const name) { - cJSON *c = array->child; - while (c && (which > 0)) + cJSON *false_item = cJSON_CreateFalse(); + if (add_item_to_object(object, name, false_item, &global_hooks, false)) { - c = c->next; - which--; + return false_item; } - if (!c) + + cJSON_Delete(false_item); + return NULL; +} + +CJSON_PUBLIC(cJSON*) cJSON_AddBoolToObject(cJSON * const object, const char * const name, const cJSON_bool boolean) +{ + cJSON *bool_item = cJSON_CreateBool(boolean); + if (add_item_to_object(object, name, bool_item, &global_hooks, false)) { - /* item doesn't exist */ - return NULL; + return bool_item; } - if (c->prev) + + cJSON_Delete(bool_item); + return NULL; +} + +CJSON_PUBLIC(cJSON*) cJSON_AddNumberToObject(cJSON * const object, const char * const name, const double number) +{ + cJSON *number_item = cJSON_CreateNumber(number); + if (add_item_to_object(object, name, number_item, &global_hooks, false)) { - /* not the first element */ - c->prev->next = c->next; + return number_item; } - if (c->next) + + cJSON_Delete(number_item); + return NULL; +} + +CJSON_PUBLIC(cJSON*) cJSON_AddStringToObject(cJSON * const object, const char * const name, const char * const string) +{ + cJSON *string_item = cJSON_CreateString(string); + if (add_item_to_object(object, name, string_item, &global_hooks, false)) { - c->next->prev = c->prev; + return string_item; } - if (c==array->child) + + cJSON_Delete(string_item); + return NULL; +} + +CJSON_PUBLIC(cJSON*) cJSON_AddRawToObject(cJSON * const object, const char * const name, const char * const raw) +{ + cJSON *raw_item = cJSON_CreateRaw(raw); + if (add_item_to_object(object, name, raw_item, &global_hooks, false)) { - array->child = c->next; + return raw_item; } - /* make sure the detached item doesn't point anywhere anymore */ - c->prev = c->next = NULL; - return c; + cJSON_Delete(raw_item); + return NULL; } -void cJSON_DeleteItemFromArray(cJSON *array, int which) +CJSON_PUBLIC(cJSON*) cJSON_AddObjectToObject(cJSON * const object, const char * const name) { - cJSON_Delete(cJSON_DetachItemFromArray(array, which)); + cJSON *object_item = cJSON_CreateObject(); + if (add_item_to_object(object, name, object_item, &global_hooks, false)) + { + return object_item; + } + + cJSON_Delete(object_item); + return NULL; } -cJSON *cJSON_DetachItemFromObject(cJSON *object, const char *string) +CJSON_PUBLIC(cJSON*) cJSON_AddArrayToObject(cJSON * const object, const char * const name) { - int i = 0; - cJSON *c = object->child; - while (c && cJSON_strcasecmp(c->string,string)) + cJSON *array = cJSON_CreateArray(); + if (add_item_to_object(object, name, array, &global_hooks, false)) { - i++; - c = c->next; + return array; } - if (c) + + cJSON_Delete(array); + return NULL; +} + +CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item) +{ + if ((parent == NULL) || (item == NULL)) { - return cJSON_DetachItemFromArray(object, i); + return NULL; } - return NULL; + if (item->prev != NULL) + { + /* not the first element */ + item->prev->next = item->next; + } + if (item->next != NULL) + { + /* not the last element */ + item->next->prev = item->prev; + } + + if (item == parent->child) + { + /* first element */ + parent->child = item->next; + } + /* make sure the detached item doesn't point anywhere anymore */ + item->prev = NULL; + item->next = NULL; + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which) +{ + if (which < 0) + { + return NULL; + } + + return cJSON_DetachItemViaPointer(array, get_array_item(array, (size_t)which)); +} + +CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which) +{ + cJSON_Delete(cJSON_DetachItemFromArray(array, which)); +} + +CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObject(cJSON *object, const char *string) +{ + cJSON *to_detach = cJSON_GetObjectItem(object, string); + + return cJSON_DetachItemViaPointer(object, to_detach); +} + +CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string) +{ + cJSON *to_detach = cJSON_GetObjectItemCaseSensitive(object, string); + + return cJSON_DetachItemViaPointer(object, to_detach); } -void cJSON_DeleteItemFromObject(cJSON *object, const char *string) +CJSON_PUBLIC(void) cJSON_DeleteItemFromObject(cJSON *object, const char *string) { cJSON_Delete(cJSON_DetachItemFromObject(object, string)); } +CJSON_PUBLIC(void) cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string) +{ + cJSON_Delete(cJSON_DetachItemFromObjectCaseSensitive(object, string)); +} + /* Replace array/object items with new ones. */ -void cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem) +CJSON_PUBLIC(void) cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem) { - cJSON *c = array->child; - while (c && (which > 0)) + cJSON *after_inserted = NULL; + + if (which < 0) { - c = c->next; - which--; + return; } - if (!c) + + after_inserted = get_array_item(array, (size_t)which); + if (after_inserted == NULL) { - cJSON_AddItemToArray(array, newitem); + add_item_to_array(array, newitem); return; } - newitem->next = c; - newitem->prev = c->prev; - c->prev = newitem; - if (c == array->child) + + newitem->next = after_inserted; + newitem->prev = after_inserted->prev; + after_inserted->prev = newitem; + if (after_inserted == array->child) { array->child = newitem; } @@ -1901,62 +2181,85 @@ void cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem) } } -void cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem) +CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement) { - cJSON *c = array->child; - while (c && (which > 0)) + if ((parent == NULL) || (replacement == NULL) || (item == NULL)) { - c = c->next; - which--; + return false; } - if (!c) + + if (replacement == item) { - return; + return true; } - newitem->next = c->next; - newitem->prev = c->prev; - if (newitem->next) + + replacement->next = item->next; + replacement->prev = item->prev; + + if (replacement->next != NULL) { - newitem->next->prev = newitem; + replacement->next->prev = replacement; } - if (c == array->child) + if (replacement->prev != NULL) { - array->child = newitem; + replacement->prev->next = replacement; } - else + if (parent->child == item) { - newitem->prev->next = newitem; + parent->child = replacement; } - c->next = c->prev = NULL; - cJSON_Delete(c); + + item->next = NULL; + item->prev = NULL; + cJSON_Delete(item); + + return true; } -void cJSON_ReplaceItemInObject(cJSON *object, const char *string, cJSON *newitem) +CJSON_PUBLIC(void) cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem) { - int i = 0; - cJSON *c = object->child; - while(c && cJSON_strcasecmp(c->string, string)) + if (which < 0) { - i++; - c = c->next; + return; } - if(c) + + cJSON_ReplaceItemViaPointer(array, get_array_item(array, (size_t)which), newitem); +} + +static cJSON_bool replace_item_in_object(cJSON *object, const char *string, cJSON *replacement, cJSON_bool case_sensitive) +{ + if ((replacement == NULL) || (string == NULL)) { - /* free the old string if not const */ - if (!(newitem->type & cJSON_StringIsConst) && newitem->string) - { - cJSON_free(newitem->string); - } + return false; + } - newitem->string = cJSON_strdup(string); - cJSON_ReplaceItemInArray(object, i, newitem); + /* replace the name in the replacement */ + if (!(replacement->type & cJSON_StringIsConst) && (replacement->string != NULL)) + { + cJSON_free(replacement->string); } + replacement->string = (char*)cJSON_strdup((const unsigned char*)string, &global_hooks); + replacement->type &= ~cJSON_StringIsConst; + + cJSON_ReplaceItemViaPointer(object, get_object_item(object, string, case_sensitive), replacement); + + return true; +} + +CJSON_PUBLIC(void) cJSON_ReplaceItemInObject(cJSON *object, const char *string, cJSON *newitem) +{ + replace_item_in_object(object, string, newitem, false); +} + +CJSON_PUBLIC(void) cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object, const char *string, cJSON *newitem) +{ + replace_item_in_object(object, string, newitem, true); } /* Create basic types: */ -cJSON *cJSON_CreateNull(void) +CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void) { - cJSON *item = cJSON_New_Item(); + cJSON *item = cJSON_New_Item(&global_hooks); if(item) { item->type = cJSON_NULL; @@ -1965,9 +2268,9 @@ cJSON *cJSON_CreateNull(void) return item; } -cJSON *cJSON_CreateTrue(void) +CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void) { - cJSON *item = cJSON_New_Item(); + cJSON *item = cJSON_New_Item(&global_hooks); if(item) { item->type = cJSON_True; @@ -1976,9 +2279,9 @@ cJSON *cJSON_CreateTrue(void) return item; } -cJSON *cJSON_CreateFalse(void) +CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void) { - cJSON *item = cJSON_New_Item(); + cJSON *item = cJSON_New_Item(&global_hooks); if(item) { item->type = cJSON_False; @@ -1987,9 +2290,9 @@ cJSON *cJSON_CreateFalse(void) return item; } -cJSON *cJSON_CreateBool(cjbool b) +CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool b) { - cJSON *item = cJSON_New_Item(); + cJSON *item = cJSON_New_Item(&global_hooks); if(item) { item->type = b ? cJSON_True : cJSON_False; @@ -1998,26 +2301,39 @@ cJSON *cJSON_CreateBool(cjbool b) return item; } -cJSON *cJSON_CreateNumber(double num) +CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num) { - cJSON *item = cJSON_New_Item(); + cJSON *item = cJSON_New_Item(&global_hooks); if(item) { item->type = cJSON_Number; item->valuedouble = num; - item->valueint = (int)num; + + /* use saturation in case of overflow */ + if (num >= INT_MAX) + { + item->valueint = INT_MAX; + } + else if (num <= (double)INT_MIN) + { + item->valueint = INT_MIN; + } + else + { + item->valueint = (int)num; + } } return item; } -cJSON *cJSON_CreateString(const char *string) +CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string) { - cJSON *item = cJSON_New_Item(); + cJSON *item = cJSON_New_Item(&global_hooks); if(item) { item->type = cJSON_String; - item->valuestring = cJSON_strdup(string); + item->valuestring = (char*)cJSON_strdup((const unsigned char*)string, &global_hooks); if(!item->valuestring) { cJSON_Delete(item); @@ -2028,13 +2344,46 @@ cJSON *cJSON_CreateString(const char *string) return item; } -extern cJSON *cJSON_CreateRaw(const char *raw) +CJSON_PUBLIC(cJSON *) cJSON_CreateStringReference(const char *string) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if (item != NULL) + { + item->type = cJSON_String | cJSON_IsReference; + item->valuestring = (char*)cast_away_const(string); + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateObjectReference(const cJSON *child) { - cJSON *item = cJSON_New_Item(); + cJSON *item = cJSON_New_Item(&global_hooks); + if (item != NULL) { + item->type = cJSON_Object | cJSON_IsReference; + item->child = (cJSON*)cast_away_const(child); + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateArrayReference(const cJSON *child) { + cJSON *item = cJSON_New_Item(&global_hooks); + if (item != NULL) { + item->type = cJSON_Array | cJSON_IsReference; + item->child = (cJSON*)cast_away_const(child); + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw) +{ + cJSON *item = cJSON_New_Item(&global_hooks); if(item) { item->type = cJSON_Raw; - item->valuestring = cJSON_strdup(raw); + item->valuestring = (char*)cJSON_strdup((const unsigned char*)raw, &global_hooks); if(!item->valuestring) { cJSON_Delete(item); @@ -2045,9 +2394,9 @@ extern cJSON *cJSON_CreateRaw(const char *raw) return item; } -cJSON *cJSON_CreateArray(void) +CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void) { - cJSON *item = cJSON_New_Item(); + cJSON *item = cJSON_New_Item(&global_hooks); if(item) { item->type=cJSON_Array; @@ -2056,9 +2405,9 @@ cJSON *cJSON_CreateArray(void) return item; } -cJSON *cJSON_CreateObject(void) +CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void) { - cJSON *item = cJSON_New_Item(); + cJSON *item = cJSON_New_Item(&global_hooks); if (item) { item->type = cJSON_Object; @@ -2068,13 +2417,20 @@ cJSON *cJSON_CreateObject(void) } /* Create Arrays: */ -cJSON *cJSON_CreateIntArray(const int *numbers, int count) +CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count) { - int i = 0; + size_t i = 0; cJSON *n = NULL; cJSON *p = NULL; - cJSON *a = cJSON_CreateArray(); - for(i = 0; a && (i < count); i++) + cJSON *a = NULL; + + if ((count < 0) || (numbers == NULL)) + { + return NULL; + } + + a = cJSON_CreateArray(); + for(i = 0; a && (i < (size_t)count); i++) { n = cJSON_CreateNumber(numbers[i]); if (!n) @@ -2096,15 +2452,23 @@ cJSON *cJSON_CreateIntArray(const int *numbers, int count) return a; } -cJSON *cJSON_CreateFloatArray(const float *numbers, int count) +CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count) { - int i = 0; + size_t i = 0; cJSON *n = NULL; cJSON *p = NULL; - cJSON *a = cJSON_CreateArray(); - for(i = 0; a && (i < count); i++) + cJSON *a = NULL; + + if ((count < 0) || (numbers == NULL)) { - n = cJSON_CreateNumber(numbers[i]); + return NULL; + } + + a = cJSON_CreateArray(); + + for(i = 0; a && (i < (size_t)count); i++) + { + n = cJSON_CreateNumber((double)numbers[i]); if(!n) { cJSON_Delete(a); @@ -2124,13 +2488,21 @@ cJSON *cJSON_CreateFloatArray(const float *numbers, int count) return a; } -cJSON *cJSON_CreateDoubleArray(const double *numbers, int count) +CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count) { - int i = 0; + size_t i = 0; cJSON *n = NULL; cJSON *p = NULL; - cJSON *a = cJSON_CreateArray(); - for(i = 0;a && (i < count); i++) + cJSON *a = NULL; + + if ((count < 0) || (numbers == NULL)) + { + return NULL; + } + + a = cJSON_CreateArray(); + + for(i = 0;a && (i < (size_t)count); i++) { n = cJSON_CreateNumber(numbers[i]); if(!n) @@ -2152,13 +2524,21 @@ cJSON *cJSON_CreateDoubleArray(const double *numbers, int count) return a; } -cJSON *cJSON_CreateStringArray(const char **strings, int count) +CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char **strings, int count) { - int i = 0; + size_t i = 0; cJSON *n = NULL; cJSON *p = NULL; - cJSON *a = cJSON_CreateArray(); - for (i = 0; a && (i < count); i++) + cJSON *a = NULL; + + if ((count < 0) || (strings == NULL)) + { + return NULL; + } + + a = cJSON_CreateArray(); + + for (i = 0; a && (i < (size_t)count); i++) { n = cJSON_CreateString(strings[i]); if(!n) @@ -2181,23 +2561,23 @@ cJSON *cJSON_CreateStringArray(const char **strings, int count) } /* Duplication */ -cJSON *cJSON_Duplicate(const cJSON *item, cjbool recurse) +CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse) { cJSON *newitem = NULL; - cJSON *cptr = NULL; - cJSON *nptr = NULL; + cJSON *child = NULL; + cJSON *next = NULL; cJSON *newchild = NULL; /* Bail on bad ptr */ if (!item) { - return NULL; + goto fail; } /* Create new item */ - newitem = cJSON_New_Item(); + newitem = cJSON_New_Item(&global_hooks); if (!newitem) { - return NULL; + goto fail; } /* Copy over all vars */ newitem->type = item->type & (~cJSON_IsReference); @@ -2205,20 +2585,18 @@ cJSON *cJSON_Duplicate(const cJSON *item, cjbool recurse) newitem->valuedouble = item->valuedouble; if (item->valuestring) { - newitem->valuestring = cJSON_strdup(item->valuestring); + newitem->valuestring = (char*)cJSON_strdup((unsigned char*)item->valuestring, &global_hooks); if (!newitem->valuestring) { - cJSON_Delete(newitem); - return NULL; + goto fail; } } if (item->string) { - newitem->string = (item->type&cJSON_StringIsConst) ? item->string : cJSON_strdup(item->string); + newitem->string = (item->type&cJSON_StringIsConst) ? item->string : (char*)cJSON_strdup((unsigned char*)item->string, &global_hooks); if (!newitem->string) { - cJSON_Delete(newitem); - return NULL; + goto fail; } } /* If non-recursive, then we're done! */ @@ -2227,93 +2605,369 @@ cJSON *cJSON_Duplicate(const cJSON *item, cjbool recurse) return newitem; } /* Walk the ->next chain for the child. */ - cptr = item->child; - while (cptr) + child = item->child; + while (child != NULL) { - newchild = cJSON_Duplicate(cptr, 1); /* Duplicate (with recurse) each item in the ->next chain */ + newchild = cJSON_Duplicate(child, true); /* Duplicate (with recurse) each item in the ->next chain */ if (!newchild) { - cJSON_Delete(newitem); - return NULL; + goto fail; } - if (nptr) + if (next != NULL) { /* If newitem->child already set, then crosswire ->prev and ->next and move on */ - nptr->next = newchild; - newchild->prev = nptr; - nptr = newchild; + next->next = newchild; + newchild->prev = next; + next = newchild; } else { /* Set newitem->child and move to it */ - newitem->child = newchild; nptr = newchild; + newitem->child = newchild; + next = newchild; } - cptr = cptr->next; + child = child->next; } return newitem; + +fail: + if (newitem != NULL) + { + cJSON_Delete(newitem); + } + + return NULL; } -void cJSON_Minify(char *json) +static void skip_oneline_comment(char **input) { - char *into = json; - while (*json) + *input += static_strlen("//"); + + for (; (*input)[0] != '\0'; ++(*input)) { - if (*json == ' ') - { - json++; + if ((*input)[0] == '\n') { + *input += static_strlen("\n"); + return; } - else if (*json == '\t') + } +} + +static void skip_multiline_comment(char **input) +{ + *input += static_strlen("/*"); + + for (; (*input)[0] != '\0'; ++(*input)) + { + if (((*input)[0] == '*') && ((*input)[1] == '/')) { - /* Whitespace characters. */ - json++; + *input += static_strlen("*/"); + return; } - else if (*json == '\r') - { - json++; + } +} + +static void minify_string(char **input, char **output) { + (*output)[0] = (*input)[0]; + *input += static_strlen("\""); + *output += static_strlen("\""); + + + for (; (*input)[0] != '\0'; (void)++(*input), ++(*output)) { + (*output)[0] = (*input)[0]; + + if ((*input)[0] == '\"') { + (*output)[0] = '\"'; + *input += static_strlen("\""); + *output += static_strlen("\""); + return; + } else if (((*input)[0] == '\\') && ((*input)[1] == '\"')) { + (*output)[1] = (*input)[1]; + *input += static_strlen("\""); + *output += static_strlen("\""); } - else if (*json=='\n') + } +} + +CJSON_PUBLIC(void) cJSON_Minify(char *json) +{ + char *into = json; + + if (json == NULL) + { + return; + } + + while (json[0] != '\0') + { + switch (json[0]) { - json++; + case ' ': + case '\t': + case '\r': + case '\n': + json++; + break; + + case '/': + if (json[1] == '/') + { + skip_oneline_comment(&json); + } + else if (json[1] == '*') + { + skip_multiline_comment(&json); + } else { + json++; + } + break; + + case '\"': + minify_string(&json, (char**)&into); + break; + + default: + into[0] = json[0]; + json++; + into++; } - else if ((*json == '/') && (json[1] == '/')) - { - /* double-slash comments, to end of line. */ - while (*json && (*json != '\n')) + } + + /* and null-terminate. */ + *into = '\0'; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_Invalid; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_False; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xff) == cJSON_True; +} + + +CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & (cJSON_True | cJSON_False)) != 0; +} +CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_NULL; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_Number; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_String; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_Array; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_Object; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_Raw; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive) +{ + if ((a == NULL) || (b == NULL) || ((a->type & 0xFF) != (b->type & 0xFF)) || cJSON_IsInvalid(a)) + { + return false; + } + + /* check if type is valid */ + switch (a->type & 0xFF) + { + case cJSON_False: + case cJSON_True: + case cJSON_NULL: + case cJSON_Number: + case cJSON_String: + case cJSON_Raw: + case cJSON_Array: + case cJSON_Object: + break; + + default: + return false; + } + + /* identical objects are equal */ + if (a == b) + { + return true; + } + + switch (a->type & 0xFF) + { + /* in these cases and equal type is enough */ + case cJSON_False: + case cJSON_True: + case cJSON_NULL: + return true; + + case cJSON_Number: + if (a->valuedouble == b->valuedouble) { - json++; + return true; } - } - else if ((*json == '/') && (json[1] == '*')) - { - /* multiline comments. */ - while (*json && !((*json == '*') && (json[1] == '/'))) + return false; + + case cJSON_String: + case cJSON_Raw: + if ((a->valuestring == NULL) || (b->valuestring == NULL)) { - json++; + return false; } - json += 2; - } - else if (*json == '\"') + if (strcmp(a->valuestring, b->valuestring) == 0) + { + return true; + } + + return false; + + case cJSON_Array: { - /* string literals, which are \" sensitive. */ - *into++ = *json++; - while (*json && (*json != '\"')) + cJSON *a_element = a->child; + cJSON *b_element = b->child; + + for (; (a_element != NULL) && (b_element != NULL);) { - if (*json == '\\') + if (!cJSON_Compare(a_element, b_element, case_sensitive)) { - *into++=*json++; + return false; } - *into++ = *json++; + + a_element = a_element->next; + b_element = b_element->next; + } + + /* one of the arrays is longer than the other */ + if (a_element != b_element) { + return false; } - *into++ = *json++; + + return true; } - else + + case cJSON_Object: { - /* All other characters. */ - *into++ = *json++; + cJSON *a_element = NULL; + cJSON *b_element = NULL; + cJSON_ArrayForEach(a_element, a) + { + /* TODO This has O(n^2) runtime, which is horrible! */ + b_element = get_object_item(b, a_element->string, case_sensitive); + if (b_element == NULL) + { + return false; + } + + if (!cJSON_Compare(a_element, b_element, case_sensitive)) + { + return false; + } + } + + /* doing this twice, once on a and b to prevent true comparison if a subset of b + * TODO: Do this the proper way, this is just a fix for now */ + cJSON_ArrayForEach(b_element, b) + { + a_element = get_object_item(a, b_element->string, case_sensitive); + if (a_element == NULL) + { + return false; + } + + if (!cJSON_Compare(b_element, a_element, case_sensitive)) + { + return false; + } + } + + return true; } + + default: + return false; } +} - /* and null-terminate. */ - *into = '\0'; +CJSON_PUBLIC(void *) cJSON_malloc(size_t size) +{ + return global_hooks.allocate(size); +} + +CJSON_PUBLIC(void) cJSON_free(void *object) +{ + global_hooks.deallocate(object); } diff --git a/onvm/lib/cJSON.h b/onvm/lib/cJSON.h index 4ef0396b8..592986b86 100644 --- a/onvm/lib/cJSON.h +++ b/onvm/lib/cJSON.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2009 Dave Gamble + Copyright (c) 2009-2017 Dave Gamble and cJSON contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -28,17 +28,65 @@ extern "C" { #endif +#if !defined(__WINDOWS__) && (defined(WIN32) || defined(WIN64) || defined(_MSC_VER) || defined(_WIN32)) +#define __WINDOWS__ +#endif + +#ifdef __WINDOWS__ + +/* When compiling for windows, we specify a specific calling convention to avoid issues where we are being called from a project with a different default calling convention. For windows you have 3 define options: + +CJSON_HIDE_SYMBOLS - Define this in the case where you don't want to ever dllexport symbols +CJSON_EXPORT_SYMBOLS - Define this on library build when you want to dllexport symbols (default) +CJSON_IMPORT_SYMBOLS - Define this if you want to dllimport symbol + +For *nix builds that support visibility attribute, you can define similar behavior by + +setting default visibility to hidden by adding +-fvisibility=hidden (for gcc) +or +-xldscope=hidden (for sun cc) +to CFLAGS + +then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way CJSON_EXPORT_SYMBOLS does + +*/ + +#define CJSON_CDECL __cdecl +#define CJSON_STDCALL __stdcall + +/* export symbols by default, this is necessary for copy pasting the C and header file */ +#if !defined(CJSON_HIDE_SYMBOLS) && !defined(CJSON_IMPORT_SYMBOLS) && !defined(CJSON_EXPORT_SYMBOLS) +#define CJSON_EXPORT_SYMBOLS +#endif + +#if defined(CJSON_HIDE_SYMBOLS) +#define CJSON_PUBLIC(type) type CJSON_STDCALL +#elif defined(CJSON_EXPORT_SYMBOLS) +#define CJSON_PUBLIC(type) __declspec(dllexport) type CJSON_STDCALL +#elif defined(CJSON_IMPORT_SYMBOLS) +#define CJSON_PUBLIC(type) __declspec(dllimport) type CJSON_STDCALL +#endif +#else /* !__WINDOWS__ */ +#define CJSON_CDECL +#define CJSON_STDCALL + +#if (defined(__GNUC__) || defined(__SUNPRO_CC) || defined (__SUNPRO_C)) && defined(CJSON_API_VISIBILITY) +#define CJSON_PUBLIC(type) __attribute__((visibility("default"))) type +#else +#define CJSON_PUBLIC(type) type +#endif +#endif + /* project version */ #define CJSON_VERSION_MAJOR 1 -#define CJSON_VERSION_MINOR 2 -#define CJSON_VERSION_PATCH 0 - -/* returns the version of cJSON as a string */ -extern const char* cJSON_Version(void); +#define CJSON_VERSION_MINOR 7 +#define CJSON_VERSION_PATCH 12 #include /* cJSON Types: */ +#define cJSON_Invalid (0) #define cJSON_False (1 << 0) #define cJSON_True (1 << 1) #define cJSON_NULL (1 << 2) @@ -65,7 +113,7 @@ typedef struct cJSON /* The item's string, if type==cJSON_String and type == cJSON_Raw */ char *valuestring; - /* The item's number, if type==cJSON_Number */ + /* writing to valueint is DEPRECATED, use cJSON_SetNumberValue instead */ int valueint; /* The item's number, if type==cJSON_Number */ double valuedouble; @@ -76,104 +124,159 @@ typedef struct cJSON typedef struct cJSON_Hooks { - void *(*malloc_fn)(size_t sz); - void (*free_fn)(void *ptr); + /* malloc/free are CDECL on Windows regardless of the default calling convention of the compiler, so ensure the hooks allow passing those functions directly. */ + void *(CJSON_CDECL *malloc_fn)(size_t sz); + void (CJSON_CDECL *free_fn)(void *ptr); } cJSON_Hooks; +typedef int cJSON_bool; + +/* Limits how deeply nested arrays/objects can be before cJSON rejects to parse them. + * This is to prevent stack overflows. */ +#ifndef CJSON_NESTING_LIMIT +#define CJSON_NESTING_LIMIT 1000 +#endif + +/* returns the version of cJSON as a string */ +CJSON_PUBLIC(const char*) cJSON_Version(void); + /* Supply malloc, realloc and free functions to cJSON */ -extern void cJSON_InitHooks(cJSON_Hooks* hooks); +CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks); +/* Memory Management: the caller is always responsible to free the results from all variants of cJSON_Parse (with cJSON_Delete) and cJSON_Print (with stdlib free, cJSON_Hooks.free_fn, or cJSON_free as appropriate). The exception is cJSON_PrintPreallocated, where the caller has full responsibility of the buffer. */ +/* Supply a block of JSON, and this returns a cJSON object you can interrogate. */ +CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value); +/* ParseWithOpts allows you to require (and check) that the JSON is null terminated, and to retrieve the pointer to the final byte parsed. */ +/* If you supply a ptr in return_parse_end and parsing fails, then return_parse_end will contain a pointer to the error so will match cJSON_GetErrorPtr(). */ +CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cJSON_bool require_null_terminated); -/* Supply a block of JSON, and this returns a cJSON object you can interrogate. Call cJSON_Delete when finished. */ -extern cJSON *cJSON_Parse(const char *value); -/* Render a cJSON entity to text for transfer/storage. Free the char* when finished. */ -extern char *cJSON_Print(const cJSON *item); -/* Render a cJSON entity to text for transfer/storage without any formatting. Free the char* when finished. */ -extern char *cJSON_PrintUnformatted(const cJSON *item); +/* Render a cJSON entity to text for transfer/storage. */ +CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item); +/* Render a cJSON entity to text for transfer/storage without any formatting. */ +CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item); /* Render a cJSON entity to text using a buffered strategy. prebuffer is a guess at the final size. guessing well reduces reallocation. fmt=0 gives unformatted, =1 gives formatted */ -extern char *cJSON_PrintBuffered(const cJSON *item, int prebuffer, int fmt); -/* Render a cJSON entity to text using a buffer already allocated in memory with length buf_len. Returns 1 on success and 0 on failure. */ -extern int cJSON_PrintPreallocated(cJSON *item, char *buf, const int len, const int fmt); +CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt); +/* Render a cJSON entity to text using a buffer already allocated in memory with given length. Returns 1 on success and 0 on failure. */ +/* NOTE: cJSON is not always 100% accurate in estimating how much memory it will use, so to be safe allocate 5 bytes more than you actually need */ +CJSON_PUBLIC(cJSON_bool) cJSON_PrintPreallocated(cJSON *item, char *buffer, const int length, const cJSON_bool format); /* Delete a cJSON entity and all subentities. */ -extern void cJSON_Delete(cJSON *c); +CJSON_PUBLIC(void) cJSON_Delete(cJSON *c); /* Returns the number of items in an array (or object). */ -extern int cJSON_GetArraySize(const cJSON *array); -/* Retrieve item number "item" from array "array". Returns NULL if unsuccessful. */ -extern cJSON *cJSON_GetArrayItem(const cJSON *array, int item); +CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array); +/* Retrieve item number "index" from array "array". Returns NULL if unsuccessful. */ +CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index); /* Get item "string" from object. Case insensitive. */ -extern cJSON *cJSON_GetObjectItem(const cJSON *object, const char *string); -extern int cJSON_HasObjectItem(const cJSON *object, const char *string); +CJSON_PUBLIC(cJSON *) cJSON_GetObjectItem(const cJSON * const object, const char * const string); +CJSON_PUBLIC(cJSON *) cJSON_GetObjectItemCaseSensitive(const cJSON * const object, const char * const string); +CJSON_PUBLIC(cJSON_bool) cJSON_HasObjectItem(const cJSON *object, const char *string); /* For analysing failed parses. This returns a pointer to the parse error. You'll probably need to look a few chars back to make sense of it. Defined when cJSON_Parse() returns 0. 0 when cJSON_Parse() succeeds. */ -extern const char *cJSON_GetErrorPtr(void); +CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void); + +/* Check if the item is a string and return its valuestring */ +CJSON_PUBLIC(char *) cJSON_GetStringValue(cJSON *item); + +/* These functions check the type of an item */ +CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item); /* These calls create a cJSON item of the appropriate type. */ -extern cJSON *cJSON_CreateNull(void); -extern cJSON *cJSON_CreateTrue(void); -extern cJSON *cJSON_CreateFalse(void); -extern cJSON *cJSON_CreateBool(int b); -extern cJSON *cJSON_CreateNumber(double num); -extern cJSON *cJSON_CreateString(const char *string); +CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void); +CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void); +CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void); +CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool boolean); +CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num); +CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string); /* raw json */ -extern cJSON *cJSON_CreateRaw(const char *raw); -extern cJSON *cJSON_CreateArray(void); -extern cJSON *cJSON_CreateObject(void); +CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw); +CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void); +CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void); + +/* Create a string where valuestring references a string so + * it will not be freed by cJSON_Delete */ +CJSON_PUBLIC(cJSON *) cJSON_CreateStringReference(const char *string); +/* Create an object/arrray that only references it's elements so + * they will not be freed by cJSON_Delete */ +CJSON_PUBLIC(cJSON *) cJSON_CreateObjectReference(const cJSON *child); +CJSON_PUBLIC(cJSON *) cJSON_CreateArrayReference(const cJSON *child); /* These utilities create an Array of count items. */ -extern cJSON *cJSON_CreateIntArray(const int *numbers, int count); -extern cJSON *cJSON_CreateFloatArray(const float *numbers, int count); -extern cJSON *cJSON_CreateDoubleArray(const double *numbers, int count); -extern cJSON *cJSON_CreateStringArray(const char **strings, int count); +CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count); +CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count); +CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count); +CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char **strings, int count); /* Append item to the specified array/object. */ -extern void cJSON_AddItemToArray(cJSON *array, cJSON *item); -extern void cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item); +CJSON_PUBLIC(void) cJSON_AddItemToArray(cJSON *array, cJSON *item); +CJSON_PUBLIC(void) cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item); /* Use this when string is definitely const (i.e. a literal, or as good as), and will definitely survive the cJSON object. * WARNING: When this function was used, make sure to always check that (item->type & cJSON_StringIsConst) is zero before * writing to `item->string` */ -extern void cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item); +CJSON_PUBLIC(void) cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item); /* Append reference to item to the specified array/object. Use this when you want to add an existing cJSON to a new cJSON, but don't want to corrupt your existing cJSON. */ -extern void cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item); -extern void cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item); +CJSON_PUBLIC(void) cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item); +CJSON_PUBLIC(void) cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item); /* Remove/Detatch items from Arrays/Objects. */ -extern cJSON *cJSON_DetachItemFromArray(cJSON *array, int which); -extern void cJSON_DeleteItemFromArray(cJSON *array, int which); -extern cJSON *cJSON_DetachItemFromObject(cJSON *object, const char *string); -extern void cJSON_DeleteItemFromObject(cJSON *object, const char *string); +CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item); +CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which); +CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which); +CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObject(cJSON *object, const char *string); +CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string); +CJSON_PUBLIC(void) cJSON_DeleteItemFromObject(cJSON *object, const char *string); +CJSON_PUBLIC(void) cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string); /* Update array items. */ -extern void cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem); /* Shifts pre-existing items to the right. */ -extern void cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem); -extern void cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem); +CJSON_PUBLIC(void) cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem); /* Shifts pre-existing items to the right. */ +CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement); +CJSON_PUBLIC(void) cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem); +CJSON_PUBLIC(void) cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem); +CJSON_PUBLIC(void) cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object,const char *string,cJSON *newitem); /* Duplicate a cJSON item */ -extern cJSON *cJSON_Duplicate(const cJSON *item, int recurse); +CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse); /* Duplicate will create a new, identical cJSON item to the one you pass, in new memory that will need to be released. With recurse!=0, it will duplicate any children connected to the item. The item->next and ->prev pointers are always zero on return from Duplicate. */ +/* Recursively compare two cJSON items for equality. If either a or b is NULL or invalid, they will be considered unequal. + * case_sensitive determines if object keys are treated case sensitive (1) or case insensitive (0) */ +CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive); -/* ParseWithOpts allows you to require (and check) that the JSON is null terminated, and to retrieve the pointer to the final byte parsed. */ -/* If you supply a ptr in return_parse_end and parsing fails, then return_parse_end will contain a pointer to the error. If not, then cJSON_GetErrorPtr() does the job. */ -extern cJSON *cJSON_ParseWithOpts(const char *value, const char **return_parse_end, int require_null_terminated); -extern void cJSON_Minify(char *json); +CJSON_PUBLIC(void) cJSON_Minify(char *json); -/* Macros for creating things quickly. */ -#define cJSON_AddNullToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateNull()) -#define cJSON_AddTrueToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateTrue()) -#define cJSON_AddFalseToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateFalse()) -#define cJSON_AddBoolToObject(object,name,b) cJSON_AddItemToObject(object, name, cJSON_CreateBool(b)) -#define cJSON_AddNumberToObject(object,name,n) cJSON_AddItemToObject(object, name, cJSON_CreateNumber(n)) -#define cJSON_AddStringToObject(object,name,s) cJSON_AddItemToObject(object, name, cJSON_CreateString(s)) -#define cJSON_AddRawToObject(object,name,s) cJSON_AddItemToObject(object, name, cJSON_CreateRaw(s)) +/* Helper functions for creating and adding items to an object at the same time. + * They return the added item or NULL on failure. */ +CJSON_PUBLIC(cJSON*) cJSON_AddNullToObject(cJSON * const object, const char * const name); +CJSON_PUBLIC(cJSON*) cJSON_AddTrueToObject(cJSON * const object, const char * const name); +CJSON_PUBLIC(cJSON*) cJSON_AddFalseToObject(cJSON * const object, const char * const name); +CJSON_PUBLIC(cJSON*) cJSON_AddBoolToObject(cJSON * const object, const char * const name, const cJSON_bool boolean); +CJSON_PUBLIC(cJSON*) cJSON_AddNumberToObject(cJSON * const object, const char * const name, const double number); +CJSON_PUBLIC(cJSON*) cJSON_AddStringToObject(cJSON * const object, const char * const name, const char * const string); +CJSON_PUBLIC(cJSON*) cJSON_AddRawToObject(cJSON * const object, const char * const name, const char * const raw); +CJSON_PUBLIC(cJSON*) cJSON_AddObjectToObject(cJSON * const object, const char * const name); +CJSON_PUBLIC(cJSON*) cJSON_AddArrayToObject(cJSON * const object, const char * const name); /* When assigning an integer value, it needs to be propagated to valuedouble too. */ -#define cJSON_SetIntValue(object,val) ((object) ? (object)->valueint = (object)->valuedouble = (val) : (val)) -#define cJSON_SetNumberValue(object,val) ((object) ? (object)->valueint = (object)->valuedouble = (val) : (val)) +#define cJSON_SetIntValue(object, number) ((object) ? (object)->valueint = (object)->valuedouble = (number) : (number)) +/* helper for the cJSON_SetNumberValue macro */ +CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number); +#define cJSON_SetNumberValue(object, number) ((object != NULL) ? cJSON_SetNumberHelper(object, (double)number) : (number)) + +/* Macro for iterating over an array or object */ +#define cJSON_ArrayForEach(element, array) for(element = (array != NULL) ? (array)->child : NULL; element != NULL; element = element->next) -/* Macro for iterating over an array */ -#define cJSON_ArrayForEach(pos, head) for(pos = (head)->child; pos != NULL; pos = pos->next) +/* malloc/free objects using the malloc/free functions that have been set with cJSON_InitHooks */ +CJSON_PUBLIC(void *) cJSON_malloc(size_t size); +CJSON_PUBLIC(void) cJSON_free(void *object); #ifdef __cplusplus } From 60293fad1abdf5101c159eb595fc526dc0a25e4e Mon Sep 17 00:00:00 2001 From: Dennis Afanasev <32916582+dennisafa@users.noreply.github.com> Date: Sat, 27 Jul 2019 07:28:20 -0400 Subject: [PATCH 06/12] Extend Shared Core Functionality for Messages (#149) Adds functionality for NFs using shared core mode to sleep when no messages(and no packets) are enqueued onto an NF's message ring. Commit log: * Shared cores for messages * Rm line * nf context, not nf * Moved functionality to main thread * Adding macro for msg threshold * Update NF_Dev.md --- docs/NF_Dev.md | 4 ++-- onvm/onvm_nflib/onvm_common.h | 3 ++- onvm/onvm_nflib/onvm_nflib.c | 13 ++++++++----- 3 files changed, 12 insertions(+), 8 deletions(-) mode change 100644 => 100755 onvm/onvm_nflib/onvm_common.h mode change 100644 => 100755 onvm/onvm_nflib/onvm_nflib.c diff --git a/docs/NF_Dev.md b/docs/NF_Dev.md index 389080997..282711983 100644 --- a/docs/NF_Dev.md +++ b/docs/NF_Dev.md @@ -43,7 +43,7 @@ NFs can scale by running multiple threads. For launching more threads the main N Example use of Multithreading NF scaling functionality can be seen in the scaling_example NF. ### Shared core mode -This is an **EXPERIMENTAL** mode for OpenNetVM. It allows multiple NFs to run on a shared core. In "normal" OpenNetVM, each NF will poll its RX queue for packets, monopolizing the CPU even if it has a low load. This branch adds a semaphore-based communication system so that NFs will block when there are no packets available. The NF Manger will then signal the semaphore once one or more packets arrive. +This is an **EXPERIMENTAL** mode for OpenNetVM. It allows multiple NFs to run on a shared core. In "normal" OpenNetVM, each NF will poll its RX queue and message queue for packets and messages respectively, monopolizing the CPU even if it has a low load. This branch adds a semaphore-based communication system so that NFs will block when there are no packets and messages available. The NF Manger will then signal the semaphore once one or more packets or messages arrive. This code allows you to evaluate resource management techniques for NFs that share cores, however it has not been fully tested with complex NFs, therefore if you encounter any bugs please create an issue or a pull request with a proposed fix. @@ -52,7 +52,7 @@ The code is based on the hybrid-polling model proposed in [_Flurries: Countless Usage / Known Limitations: - To enable pass a `-c` flag to the onvm_mgr, and use a `-s` flag when starting a NF to specify that they want to share cores - All code for sharing CPUs is within `if (ONVM_NF_SHARE_CORES)` blocks - - When enabled, you can run multiple NFs on the same CPU core with much less interference than if they are polling for packets + - When enabled, you can run multiple NFs on the same CPU core with much less interference than if they are polling for packets and messages - This code does not provide any particular intelligence for how NFs are scheduled or when they wakeup/sleep - Note that the manager threads all still use polling diff --git a/onvm/onvm_nflib/onvm_common.h b/onvm/onvm_nflib/onvm_common.h old mode 100644 new mode 100755 index c2a947def..4368790d4 --- a/onvm/onvm_nflib/onvm_common.h +++ b/onvm/onvm_nflib/onvm_common.h @@ -75,6 +75,7 @@ #define ONVM_NF_ACTION_OUT 3 // send the packet out the NIC port set in the argument field #define PKT_WAKEUP_THRESHOLD 1 // for shared core mode, how many packets are required to wake up the NF +#define MSG_WAKEUP_THRESHOLD 1 // for shared core mode, how many messages on an NF's ring are required to wake up the NF /* Used in setting bit flags for core options */ #define MANUAL_CORE_ASSIGNMENT_BIT 0 @@ -469,7 +470,7 @@ get_sem_name(unsigned id) { static inline int whether_wakeup_client(struct onvm_nf *nf, struct nf_wakeup_info *nf_wakeup_info) { - if (rte_ring_count(nf->rx_q) < PKT_WAKEUP_THRESHOLD) + if (rte_ring_count(nf->rx_q) < PKT_WAKEUP_THRESHOLD && rte_ring_count(nf->msg_q) < MSG_WAKEUP_THRESHOLD) return 0; /* Check if its already woken up */ diff --git a/onvm/onvm_nflib/onvm_nflib.c b/onvm/onvm_nflib/onvm_nflib.c old mode 100644 new mode 100755 index 6ee08db17..2e3ecee9e --- a/onvm/onvm_nflib/onvm_nflib.c +++ b/onvm/onvm_nflib/onvm_nflib.c @@ -555,6 +555,14 @@ onvm_nflib_thread_main_loop(void *arg) { start_time = rte_get_tsc_cycles(); for (;rte_atomic16_read(&nf_local_ctx->keep_running) && rte_atomic16_read(&main_nf_local_ctx->keep_running);) { + /* Possibly sleep if in shared core mode, otherwise continue */ + if (ONVM_NF_SHARE_CORES) { + if (unlikely(rte_ring_count(nf->rx_q) == 0) && likely(rte_ring_count(nf->msg_q) == 0)) { + rte_atomic16_set(nf->shared_core.sleep_state, 1); + sem_wait(nf->shared_core.nf_mutex); + } + } + nb_pkts_added = onvm_nflib_dequeue_packets((void **)pkts, nf_local_ctx, nf->function_table->pkt_handler); @@ -908,12 +916,7 @@ onvm_nflib_dequeue_packets(void **pkts, struct onvm_nf_local_ctx *nf_local_ctx, /* Dequeue all packets in ring up to max possible. */ nb_pkts = rte_ring_dequeue_burst(nf->rx_q, pkts, PACKET_READ_SIZE, NULL); - /* Possibly sleep if in shared core mode, otherwise return */ if (unlikely(nb_pkts == 0)) { - if (ONVM_NF_SHARE_CORES) { - rte_atomic16_set(nf->shared_core.sleep_state, 1); - sem_wait(nf->shared_core.nf_mutex); - } return 0; } From 888c3961bc904e7bcd4ab479e31821ecb3d3ddb5 Mon Sep 17 00:00:00 2001 From: Evan Deng Date: Mon, 29 Jul 2019 06:08:06 +0800 Subject: [PATCH 07/12] Add NFD Library with Multiple Example NFs (#98) This PR adds a NFD library, a C++ based NF developing compiler designed by Wenfei's group (http://wenfei-wu.github.io/) from IIIS, Tsinghua University, China. NFD compiles the NF logic into common C++ program by using table-form language to model NFs' behavior. With NFD's help, NF developer, free from the problems that porting NFs into specific network environment or deploying platforms, can easily concentrate on designing NF's core logic. A list of provided example NFs: - Stateful Firewall - Stateless Firewall - DNS Amplification Mitigation - Heavy Hitter Detection - NAPT - SYN Flood Detection - UDP Flood Detection - Super Spread Detection Commit log: * replace direct assigning with memcpy function * add NFD-NF stateful_firewall in examples/ * update * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update stateful_firewall.cpp * add more NFs * update NFD-NFs' go.sh file and start_nf.sh file * delete simple_c++ * Added nf tag and core to stats * Made constants for the stats in the header file * Fixed verbose stat line-up * Fixed service id totals stats * clang formatting * Update README.md and `go.sh` file in NFD dir * add -march=native to makefile * remove some comments and change some places into actual NF name * add Makefile in NFD/ * Update README.md * Fixed style of output * Fixed shared cpu verbose * Patch for the new ONVM 19.05 API * Moved print formatters to onvm_stats.h * Fixed up stat lines * fixed some errors in HHD * fixed dpdk I/O interface used * Fixed macro styling * Fix console macros * Fix stats output macro consistency * Update APIs * fix some errors * update Readme * update readme and fixed some logic errors * update README and fixed some errors * fixed some parsing errors * update testing information * delete error NF * delete error NF * update README * fixed model * delete unnecessary code * add base IP * Update .gitignore --- examples/NFD/Makefile | 56 +++ examples/NFD/README.md | 30 ++ examples/NFD/architecture.txt | 8 + .../DNSAmplificationMitigation.cpp | 352 +++++++++++++++++ .../DNSAmplificationMitigationModel.txt | 41 ++ .../NFD/dns_amplification_mitigation/Makefile | 75 ++++ .../dns_amplification_mitigation/README.md | 80 ++++ .../NFD/dns_amplification_mitigation/go.sh | 1 + examples/NFD/go.sh | 17 + examples/NFD/heavy_hitter_detection/HHD.cpp | 359 +++++++++++++++++ .../NFD/heavy_hitter_detection/HHDmodel.txt | 40 ++ examples/NFD/heavy_hitter_detection/Makefile | 75 ++++ examples/NFD/heavy_hitter_detection/README.md | 71 ++++ examples/NFD/heavy_hitter_detection/go.sh | 1 + examples/NFD/include/basic_classes.h | 352 +++++++++++++++++ examples/NFD/include/decode.h | 61 +++ examples/NFD/lib/Makefile | 11 + examples/NFD/lib/basic_classes.cpp | 122 ++++++ examples/NFD/lib/basic_methods.cpp | 29 ++ examples/NFD/napt/.gitignore | 2 + examples/NFD/napt/Makefile | 75 ++++ examples/NFD/napt/NAPT.cpp | 365 +++++++++++++++++ examples/NFD/napt/README.md | 74 ++++ examples/NFD/napt/go.sh | 1 + examples/NFD/napt/model.txt | 46 +++ examples/NFD/stateful_firewall/.gitignore | 2 + examples/NFD/stateful_firewall/Makefile | 75 ++++ examples/NFD/stateful_firewall/README.md | 76 ++++ examples/NFD/stateful_firewall/go.sh | 1 + examples/NFD/stateful_firewall/model.txt | 31 ++ .../stateful_firewall/stateful_firewall.cpp | 332 ++++++++++++++++ examples/NFD/stateless_firewall/.gitignore | 2 + examples/NFD/stateless_firewall/Makefile | 75 ++++ examples/NFD/stateless_firewall/README.md | 77 ++++ examples/NFD/stateless_firewall/go.sh | 1 + examples/NFD/stateless_firewall/model.txt | 24 ++ .../stateless_firewall/stateless_firewall.cpp | 325 ++++++++++++++++ .../NFD/super_spreader_detection/Makefile | 75 ++++ .../NFD/super_spreader_detection/README.md | 72 ++++ examples/NFD/super_spreader_detection/SSD.cpp | 367 ++++++++++++++++++ .../NFD/super_spreader_detection/SSDmodel.txt | 53 +++ examples/NFD/super_spreader_detection/go.sh | 1 + examples/NFD/syn_flood_detection/Makefile | 75 ++++ examples/NFD/syn_flood_detection/README.md | 75 ++++ .../syn_flood_detection/SYNFloodDetection.cpp | 364 +++++++++++++++++ .../SYNFloodDetectionModel.txt | 45 +++ examples/NFD/syn_flood_detection/go.sh | 1 + examples/NFD/udp_flood_mitigation/Makefile | 75 ++++ examples/NFD/udp_flood_mitigation/README.md | 71 ++++ .../UDPFloodMitagation.cpp | 358 +++++++++++++++++ .../UDPFloodMitagationModel.txt | 37 ++ examples/NFD/udp_flood_mitigation/go.sh | 1 + examples/start_nf.sh | 2 + onvm/onvm_nflib/onvm_flow_table.h | 6 +- 54 files changed, 5041 insertions(+), 2 deletions(-) create mode 100644 examples/NFD/Makefile create mode 100644 examples/NFD/README.md create mode 100644 examples/NFD/architecture.txt create mode 100644 examples/NFD/dns_amplification_mitigation/DNSAmplificationMitigation.cpp create mode 100755 examples/NFD/dns_amplification_mitigation/DNSAmplificationMitigationModel.txt create mode 100644 examples/NFD/dns_amplification_mitigation/Makefile create mode 100644 examples/NFD/dns_amplification_mitigation/README.md create mode 120000 examples/NFD/dns_amplification_mitigation/go.sh create mode 100755 examples/NFD/go.sh create mode 100644 examples/NFD/heavy_hitter_detection/HHD.cpp create mode 100755 examples/NFD/heavy_hitter_detection/HHDmodel.txt create mode 100644 examples/NFD/heavy_hitter_detection/Makefile create mode 100644 examples/NFD/heavy_hitter_detection/README.md create mode 120000 examples/NFD/heavy_hitter_detection/go.sh create mode 100644 examples/NFD/include/basic_classes.h create mode 100644 examples/NFD/include/decode.h create mode 100644 examples/NFD/lib/Makefile create mode 100644 examples/NFD/lib/basic_classes.cpp create mode 100644 examples/NFD/lib/basic_methods.cpp create mode 100644 examples/NFD/napt/.gitignore create mode 100644 examples/NFD/napt/Makefile create mode 100644 examples/NFD/napt/NAPT.cpp create mode 100644 examples/NFD/napt/README.md create mode 120000 examples/NFD/napt/go.sh create mode 100755 examples/NFD/napt/model.txt create mode 100644 examples/NFD/stateful_firewall/.gitignore create mode 100644 examples/NFD/stateful_firewall/Makefile create mode 100644 examples/NFD/stateful_firewall/README.md create mode 120000 examples/NFD/stateful_firewall/go.sh create mode 100755 examples/NFD/stateful_firewall/model.txt create mode 100644 examples/NFD/stateful_firewall/stateful_firewall.cpp create mode 100644 examples/NFD/stateless_firewall/.gitignore create mode 100644 examples/NFD/stateless_firewall/Makefile create mode 100644 examples/NFD/stateless_firewall/README.md create mode 120000 examples/NFD/stateless_firewall/go.sh create mode 100755 examples/NFD/stateless_firewall/model.txt create mode 100644 examples/NFD/stateless_firewall/stateless_firewall.cpp create mode 100644 examples/NFD/super_spreader_detection/Makefile create mode 100644 examples/NFD/super_spreader_detection/README.md create mode 100644 examples/NFD/super_spreader_detection/SSD.cpp create mode 100755 examples/NFD/super_spreader_detection/SSDmodel.txt create mode 120000 examples/NFD/super_spreader_detection/go.sh create mode 100644 examples/NFD/syn_flood_detection/Makefile create mode 100644 examples/NFD/syn_flood_detection/README.md create mode 100644 examples/NFD/syn_flood_detection/SYNFloodDetection.cpp create mode 100755 examples/NFD/syn_flood_detection/SYNFloodDetectionModel.txt create mode 120000 examples/NFD/syn_flood_detection/go.sh create mode 100644 examples/NFD/udp_flood_mitigation/Makefile create mode 100644 examples/NFD/udp_flood_mitigation/README.md create mode 100644 examples/NFD/udp_flood_mitigation/UDPFloodMitagation.cpp create mode 100755 examples/NFD/udp_flood_mitigation/UDPFloodMitagationModel.txt create mode 120000 examples/NFD/udp_flood_mitigation/go.sh diff --git a/examples/NFD/Makefile b/examples/NFD/Makefile new file mode 100644 index 000000000..b71af9b42 --- /dev/null +++ b/examples/NFD/Makefile @@ -0,0 +1,56 @@ +# openNetVM +# https://github.com/sdnfv/openNetVM +# +# BSD LICENSE +# +# Copyright(c) +# 2015-2017 George Washington University +# 2015-2017 University of California Riverside +# 2016-2017 Hewlett Packard Enterprise Development LP +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# The name of the author may not be used to endorse or promote +# products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +# To add new examples, append the directory name to this variable +examples = dns_amplification_mitigation heavy_hitter_detection napt stateful_firewall stateless_firewall super_spreader_detection syn_flood_detection udp_flood_mitigation + +clean_examples=$(addprefix clean_,$(examples)) + +.PHONY: $(examples) $(clean_examples) + +all : $(examples) +clean: $(clean_examples) + +$(examples): + cd $@ && $(MAKE) + +$(clean_examples): + cd $(patsubst clean_%,%,$@) && $(MAKE) clean \ No newline at end of file diff --git a/examples/NFD/README.md b/examples/NFD/README.md new file mode 100644 index 000000000..dea9ebcc1 --- /dev/null +++ b/examples/NFD/README.md @@ -0,0 +1,30 @@ +NFD NF +== + +# About +NFD is a C++-based NF developing compiler designed by Wenfei's group (***http://wenfei-wu.github.io/***) from IIIS, Tsinghua University, China.
+ +NFD compiles the NF logic into common C++ program by using table-form language to model NFs' behavior . With NFD's help, NF developer, free from the problems that porting NFs into specific network environment or deploying platforms, can easily concentrate on designing NF's core logic. +
+
+Here shows the NFs developed by NFD compiler for OpenNetVM. + +# Using NFD-NF +To run NFs in NFD over OpenNetVM, first you should compile the NFD library: + +``` +cd lib +make + +``` + +then you can compile the NF: + +``` +cd ../ +make + +``` + +# Contact +If you are interested in NFD compiler or want to use the NFD NFs in your work, please ***[email us](mailto:hhy17@mails.tsinghua.edu.cn)*** in advance. diff --git a/examples/NFD/architecture.txt b/examples/NFD/architecture.txt new file mode 100644 index 000000000..458770b32 --- /dev/null +++ b/examples/NFD/architecture.txt @@ -0,0 +1,8 @@ +This file describes relative files of NFD. + +basic_classes.h && basic_classes.cpp: declare and define the fundamental classes used in NFD framework. + +basic_methods.cpp: defines some methods could be used. + +decode.h: define some basic network data stuctures. + diff --git a/examples/NFD/dns_amplification_mitigation/DNSAmplificationMitigation.cpp b/examples/NFD/dns_amplification_mitigation/DNSAmplificationMitigation.cpp new file mode 100644 index 000000000..a3cdaa8c7 --- /dev/null +++ b/examples/NFD/dns_amplification_mitigation/DNSAmplificationMitigation.cpp @@ -0,0 +1,352 @@ +/********************************************************************* + * openNetVM + * https://sdnfv.github.io + * + * BSD LICENSE + * + * Copyright(c) + * 2015-2017 George Washington University + * 2015-2017 University of California Riverside + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * The name of the author may not be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ********************************************************************/ + +/************************************************************************************ +* Filename: DNSAmplificationMitigation.cpp +* Author: Hongyi Huang(hhy17 AT mails.tsinghua.edu.cn), Bangwen Deng, Wenfei Wu +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: This code is the implementation of DNSAmplificationMitigation from NFD project, + a C++ based NF developing framework designed by Wenfei's group + from IIIS, Tsinghua University, China. +*************************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#include "onvm_nflib.h" +#include "onvm_pkt_helper.h" + +#ifdef __cplusplus +} +#endif + +// NFD ADD +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "basic_classes.h" +#include "decode.h" + +using namespace std; + +#define NF_TAG "DNSAmplificationMitigation" + +/* number of package between each print */ +static uint32_t print_delay = 1000000; + +static uint32_t destination; + +/*******************************NFD features********************************/ +Flow::Flow(u_char *packet, int totallength) { + /*Decoding*/ + this->pkt = packet; + struct ether_header *eth_header; + eth_header = (struct ether_header *)packet; + + /* Pointers to start point of various headers */ + u_char *ip_header; + /* Header lengths in bytes */ + int ethernet_header_length = 14; /* Doesn't change */ + int ip_header_length; + ip_header = packet + ethernet_header_length; + ip_header_length = ((*ip_header) & 0x0F); + ip_header_length = ip_header_length * 4; + int src_addr, des_addr; + src_addr = ntohl(*((int *)(ip_header + 12))); + des_addr = ntohl(*((int *)(ip_header + 16))); + this->field_value["sip"] = new IP(src_addr, 32); + this->field_value["dip"] = new IP(des_addr, 32); + this->field_value["tag"] = new int(0); + this->field_value["iplen"] = new int(totallength); + uint8_t protocol = (*(uint8_t *)(ip_header + 9)); + this->field_value["UDP"] = new int((protocol == IPPROTO_UDP) ? 1 : 0); + + /*TCP layer*/ + TCPHdr *tcph = (TCPHdr *)(ip_header + ip_header_length); + this->field_value["sport"] = new int(ntohs(tcph->th_sport)); + this->field_value["dport"] = new int(ntohs(tcph->th_dport)); + /*URG ACK PSH RST SYN FIN*/ + this->field_value["flag_fin"] = new int(tcph->th_flags & 0x1); + this->field_value["flag_syn"] = new int((tcph->th_flags >> 1) & 0x1); + this->field_value["flag_ack"] = new int((tcph->th_flags >> 4) & 0x1); +} +void +Flow::clean() { +} + +int +process(Flow &f); +Flow f_glb; +long int _counter = 0; +long int _drop = 0; +unordered_map F_Type::MAP = unordered_map(); +unordered_map F_Type::MAP2 = unordered_map(); + +struct timeval begin_time; +struct timeval end_time; + +void +_init_() { + (new F_Type())->init(); +} + +State>> bq(*(new unordered_map>())); + +int +process(Flow &f) { + if ((*(int *)f["dport"]) == *(new int(53))) { + bq[f][(*(IP *)f["sip"])][(*(IP *)f["dip"])] = *(new int(1)); + } + if (((*(int *)f["dport"]) != *(new int(53)) && (*(int *)f["sport"]) == *(new int(53))) && + (bq[f][(*(IP *)f["dip"])][(*(IP *)f["sip"])] != *(new int(1)))) { + (*(IP *)f["dip"]) = *(new IP("0.0.0.0/0")); + } + if ((*(int *)f["dport"]) != *(new int(53)) && (*(int *)f["sport"]) != *(new int(53))) { + } + if (((*(int *)f["dport"]) != *(new int(53))) && + (bq[f][(*(IP *)f["dip"])][(*(IP *)f["sip"])] == *(new int(1)))) { + } + if (*(IP *)f["dip"] == *(new IP("0.0.0.0/0"))) { + return -1; + } + f.clean(); + return 0; +} + +int +DNSAM(u_char *pkt, int totallength) { + f_glb = Flow(pkt, totallength); + return process(f_glb); +} + +void +stop() { + gettimeofday(&end_time, NULL); + + double total = end_time.tv_sec - begin_time.tv_sec + (end_time.tv_usec - begin_time.tv_usec) / 1000000.0; + + printf("\n\n**************************************************\n"); + printf("%ld packets are processed\n", _counter); + printf("NF runs for %f seconds\n", total); + printf("**************************************************\n\n"); +} + +/**********************************************************************/ + +/* + * Print a usage message + */ +static void +usage(const char *progname) { + printf("Usage: %s [EAL args] -- [NF_LIB args] -- -d -p \n\n", progname); +} + +/* + * Parse the application arguments. + */ +static int +parse_app_args(int argc, char *argv[], const char *progname) { + int c, dst_flag = 0; + + while ((c = getopt(argc, argv, "d:p:")) != -1) { + switch (c) { + case 'd': + destination = strtoul(optarg, NULL, 10); + dst_flag = 1; + break; + case 'p': + print_delay = strtoul(optarg, NULL, 10); + break; + case '?': + usage(progname); + if (optopt == 'd') + RTE_LOG(INFO, APP, "Option -%c requires an argument.\n", optopt); + else if (optopt == 'p') + RTE_LOG(INFO, APP, "Option -%c requires an argument.\n", optopt); + else if (isprint(optopt)) + RTE_LOG(INFO, APP, "Unknown option `-%c'.\n", optopt); + else + RTE_LOG(INFO, APP, "Unknown option character `\\x%x'.\n", optopt); + return -1; + default: + usage(progname); + return -1; + } + } + + if (!dst_flag) { + RTE_LOG(INFO, APP, "DNS Amplification Mitigation NF requires destination flag -d.\n"); + return -1; + } + + return optind; +} + +/* + * This function displays stats. It uses ANSI terminal codes to clear + * screen when called. It is called from a single non-master + * thread in the server process, when the process is run with more + * than one lcore enabled. + */ +static void +do_stats_display(struct rte_mbuf *pkt) { + const char clr[] = {27, '[', '2', 'J', '\0'}; + const char topLeft[] = {27, '[', '1', ';', '1', 'H', '\0'}; + static uint64_t pkt_process = 0; + struct ipv4_hdr *ip; + + pkt_process += print_delay; + + /* Clear screen and move to top left */ + printf("%s%s", clr, topLeft); + + printf("PACKETS\n"); + printf("-----\n"); + printf("Port : %d\n", pkt->port); + printf("Size : %d\n", pkt->pkt_len); + printf("N° : %" PRIu64 "\n", pkt_process); + printf("\n\n"); + + ip = onvm_pkt_ipv4_hdr(pkt); + if (ip != NULL) { + onvm_pkt_print(pkt); + } else { + printf("No IP4 header found\n"); + } +} + +static int +packet_handler(struct rte_mbuf *buf, struct onvm_pkt_meta *meta, + __attribute__((unused)) struct onvm_nf_local_ctx *nf_local_ctx) { + static uint32_t counter = 0; + if (++counter == print_delay) { + do_stats_display(buf); + counter = 0; + } + ////////add NFD NF process here!//////////////// + _counter++; + u_char * pkt = rte_pktmbuf_mtod(buf, u_char*); + int length = (int)buf->pkt_len; + int ok; + + ok = DNSAM(pkt, length); + + if (ok == -1) { + _drop++; + meta->action = ONVM_NF_ACTION_DROP; + } else { + meta->action = ONVM_NF_ACTION_TONF; + meta->destination = destination; + } + //////////////////////////////////////////////// + return 0; +} + +int +main(int argc, char *argv[]) { + struct onvm_nf_local_ctx *nf_local_ctx; + struct onvm_nf_function_table *nf_function_table; + int arg_offset; + + const char *progname = argv[0]; + + nf_local_ctx = onvm_nflib_init_nf_local_ctx(); + onvm_nflib_start_signal_handler(nf_local_ctx, NULL); + + nf_function_table = onvm_nflib_init_nf_function_table(); + nf_function_table->pkt_handler = &packet_handler; + + if ((arg_offset = onvm_nflib_init(argc, argv, NF_TAG, nf_local_ctx, nf_function_table)) < 0) { + onvm_nflib_stop(nf_local_ctx); + if (arg_offset == ONVM_SIGNAL_TERMINATION) { + printf("Exiting due to user termination\n"); + return 0; + } else { + rte_exit(EXIT_FAILURE, "Failed ONVM init\n"); + } + } + + argc -= arg_offset; + argv += arg_offset; + + // NFD begin + _init_(); + // NFD end + + if (parse_app_args(argc, argv, progname) < 0) { + onvm_nflib_stop(nf_local_ctx); + rte_exit(EXIT_FAILURE, "Invalid command-line arguments\n"); + } + + // NFD begin + gettimeofday(&begin_time, NULL); + // NFD end + + onvm_nflib_run(nf_local_ctx); + + onvm_nflib_stop(nf_local_ctx); + stop(); + printf("If we reach here, program is ending\n"); + + return 0; +} diff --git a/examples/NFD/dns_amplification_mitigation/DNSAmplificationMitigationModel.txt b/examples/NFD/dns_amplification_mitigation/DNSAmplificationMitigationModel.txt new file mode 100755 index 000000000..47b820867 --- /dev/null +++ b/examples/NFD/dns_amplification_mitigation/DNSAmplificationMitigationModel.txt @@ -0,0 +1,41 @@ +/********************************************************************************** + NFD project + A C++ based NF developing framework designed by Wenfei's group + from IIIS, Tsinghua University, China. +******************************************************************************/ + +/************************************************************************************ +* Filename: DNSAmplificationMitigationModel.txt +* Author: Hongyi Huang(hhy17 AT mails.tsinghua.edu.cn), Bangwen Deng, Wenfei Wu +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: This file is a model file for NFD project, used to be compiled into NFD-NF +*************************************************************************************/ + + +program DNSAM{ + map> bq; + + entry{ + match_flow{f[dport]==53} + action_state{bq[f[sip]][f[dip]]=1;} + } + entry{ + match_flow{f[dport]!=53 && f[sport]==53 } + match_state{bq[f[dip]][f[sip]]!=1} + action_flow{f[dip]=DROP;} + } + entry{ + match_flow{f[dport]!=53 && f[sport]!=53 } + action_flow{pass;} + action_state{pass;} + } + + entry{ + match_flow{f[dport]!=53 } + match_state{ bq[f[dip]][f[sip]]==1} + action_flow{pass;} + action_state{pass;} + } + +} \ No newline at end of file diff --git a/examples/NFD/dns_amplification_mitigation/Makefile b/examples/NFD/dns_amplification_mitigation/Makefile new file mode 100644 index 000000000..69ed340d1 --- /dev/null +++ b/examples/NFD/dns_amplification_mitigation/Makefile @@ -0,0 +1,75 @@ +# openNetVM +# https://github.com/sdnfv/openNetVM +# +# BSD LICENSE +# +# Copyright(c) +# 2015-2017 George Washington University +# 2015-2017 University of California Riverside +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# The name of the author may not be used to endorse or promote +# products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +RTE_TARGET ?= x86_64-native-linuxapp-gcc + +# Default target, can be overriden by command line or environment +include $(RTE_SDK)/mk/rte.vars.mk + +# binary name +APP = dns_amplification_mitigation + +# all source are stored in SRCS-y +SRCS-y := DNSAmplificationMitigation.c + +# OpenNetVM, DPDK and NFD path +ONVM= $(SRCDIR)/../../../onvm +DPDK= $(SRCDIR)/../../../dpdk +NFD= $(SRCDIR)/.. + +CC=g++ + +CPPFLAGS = -O3 $(USER_FLAGS) -lpcap -std=c++11 -march=native +CPPFLAGS += -I$(ONVM)/onvm_nflib +CPPFLAGS += -I$(ONVM)/lib +CPPFLAGS += -I$(DPDK)/build/include +CPPFLAGS += -I$(NFD)/include/ + +LDFLAGS += $(ONVM)/onvm_nflib/$(RTE_TARGET)/libonvm.a +LDFLAGS += $(ONVM)/lib/$(RTE_TARGET)/lib/libonvmhelper.a +LDFLAGS += $(NFD)/lib/libNFD.a -lm -lstdc++ -lpcap + +# workaround for a gcc bug with noreturn attribute +# http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603 +ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +CFLAGS_main.o += -Wno-return-type +endif + +include $(RTE_SDK)/mk/rte.extapp.mk \ No newline at end of file diff --git a/examples/NFD/dns_amplification_mitigation/README.md b/examples/NFD/dns_amplification_mitigation/README.md new file mode 100644 index 000000000..15af9c87f --- /dev/null +++ b/examples/NFD/dns_amplification_mitigation/README.md @@ -0,0 +1,80 @@ +DNSAmplificationMitigation +== +DNS Amplification Mitigation developed by the NFD framework + + +NFD is a NF developing framework, consisting two parts, NFD language and NFD compiler. NFD compiler translate
+the NF model file wiiten in NFD language, a table-form language, into standard runtime environment(such as C++).

+ + +DNS Amplification Mitigation is translated from the `DNSAmplificationMitigationModel.txt` to C++ environment.
+ + +
+This NF is designed to be deployed in the user's end. An attacker uses a spoofed server IP to request many DNS queries that result in large answers to that DoS the server with the spoofed IP. Mitigation is by tracking if the server actually committed this request. In this program, we use a map to track all the real DNS requests from the server with the spoofed IP and pass the real DNS responses while block the responces due to attacker. + + +Testing +-- + +The DNS Amplification mitigation NF will drop all DNS response packets(TCP source port is 53) whose source IP is the user never send a DNS request to. To verify this run these 2 NFs: + +Run the DNS Amplification mitigation NF with: + +``` +./go.sh 1 -d 2 + +``` + +Run Speed Tester NF(to replay pcap file) with: + +``` +./go.sh 2 -d 1 -o pcap/pktgen_large.pcap + +``` +Above command will not trigger the dropping process. + +In order to trigger the dropping process, we need to filter the packets whose source port is 53 only(in wireshark apply `dns and dns and udp.srcport==53` filter the packets) from pktgen_large.pcap. + +And run: +``` +./go.sh 2 -d 1 -o pcap/udpResponseOnly.pcap + +``` + + +Compilation and Executionthe +-- + +To run this NF, you should use either clang++ or g++ to compile this NF developed by C++. + +``` +cd dns_amplification_mitigation +make + +``` + +Then, you can run this code. + +``` +./go.sh CORELIST SERVICE_ID DST [PRINT_DELAY] + +OR + +./go.sh -F CONFIG_FILE -- -- -d DST [-p PRINT_DELAY] + +OR + +sudo ./build/dns_amplification_mitigation -l CORELIST -n 3 --proc-type=secondary -- -r SERVICE_ID -- -d DST [-p PRINT_DELAY] +``` + +App Specific Arguments +-- + - `-d `: destination service ID to foward to + - `-p `: number of packets between each print, e.g. `-p 1` prints every packets. + +Config File Support +-- +This NF supports the NF generating arguments from a config file. For additional reading, see [Examples.md](../../docs/Examples.md) + +See `../example_config.json` for all possible options that can be set. diff --git a/examples/NFD/dns_amplification_mitigation/go.sh b/examples/NFD/dns_amplification_mitigation/go.sh new file mode 120000 index 000000000..40babd806 --- /dev/null +++ b/examples/NFD/dns_amplification_mitigation/go.sh @@ -0,0 +1 @@ +../go.sh \ No newline at end of file diff --git a/examples/NFD/go.sh b/examples/NFD/go.sh new file mode 100755 index 000000000..5bce8c94e --- /dev/null +++ b/examples/NFD/go.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +#The go.sh script is a convinient way to run start_nf.sh without specifying NF_NAME + +NF_DIR=${PWD##*/} + +NF_DIR=NFD/${NF_DIR} + +# echo ${NF_DIR} + +if [ ! -f ../../start_nf.sh ]; then + echo "ERROR: The ./go.sh script can only be used from the NF folder" + echo "If running from other directory use examples/start_nf.sh" + exit 1 +fi + +../../start_nf.sh $NF_DIR "$@" diff --git a/examples/NFD/heavy_hitter_detection/HHD.cpp b/examples/NFD/heavy_hitter_detection/HHD.cpp new file mode 100644 index 000000000..c97a5567a --- /dev/null +++ b/examples/NFD/heavy_hitter_detection/HHD.cpp @@ -0,0 +1,359 @@ +/********************************************************************* + * openNetVM + * https://sdnfv.github.io + * + * BSD LICENSE + * + * Copyright(c) + * 2015-2017 George Washington University + * 2015-2017 University of California Riverside + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * The name of the author may not be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ********************************************************************/ + +/************************************************************************************ +* Filename: HHD.cpp +* Author: Hongyi Huang(hhy17 AT mails.tsinghua.edu.cn), Bangwen Deng, Wenfei Wu +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: This code is the implementation of HeavyHitterDetection from NFD project, + a C++ based NF developing framework designed by Wenfei's group + from IIIS, Tsinghua University, China. +*************************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#include "onvm_nflib.h" +#include "onvm_pkt_helper.h" + +#ifdef __cplusplus +} +#endif + +// NFD ADD +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "basic_classes.h" +#include "decode.h" + +using namespace std; + +#define NF_TAG "HeavyHitterDetection" + +/* number of package between each print */ +static uint32_t print_delay = 1000000; + +static uint32_t destination; + +/*******************************NFD features********************************/ +Flow::Flow(u_char *packet, int totallength) { + /*Decoding*/ + this->pkt = packet; + struct ether_header *eth_header; + eth_header = (struct ether_header *)packet; + + /* Pointers to start point of various headers */ + u_char *ip_header; + /* Header lengths in bytes */ + int ethernet_header_length = 14; /* Doesn't change */ + int ip_header_length; + ip_header = packet + ethernet_header_length; + ip_header_length = ((*ip_header) & 0x0F); + ip_header_length = ip_header_length * 4; + int src_addr, des_addr; + src_addr = ntohl(*((int *)(ip_header + 12))); + des_addr = ntohl(*((int *)(ip_header + 16))); + this->field_value["sip"] = new IP(src_addr, 32); + this->field_value["dip"] = new IP(des_addr, 32); + this->field_value["tag"] = new int(0); + this->field_value["iplen"] = new int(totallength); + uint8_t protocol = (*(uint8_t *)(ip_header + 9)); + this->field_value["UDP"] = new int((protocol == IPPROTO_UDP) ? 1 : 0); + + /*TCP layer*/ + TCPHdr *tcph = (TCPHdr *)(ip_header + ip_header_length); + this->field_value["sport"] = new int(ntohs(tcph->th_sport)); + this->field_value["dport"] = new int(ntohs(tcph->th_dport)); + /*URG ACK PSH RST SYN FIN*/ + this->field_value["flag_fin"] = new int(tcph->th_flags & 0x1); + this->field_value["flag_syn"] = new int((tcph->th_flags >> 1) & 0x1); + this->field_value["flag_ack"] = new int((tcph->th_flags >> 4) & 0x1); +} +void +Flow::clean() { +} + +int +process(Flow &f); +Flow f_glb; +long int _counter = 0; +unordered_map F_Type::MAP = unordered_map(); +unordered_map F_Type::MAP2 = unordered_map(); + +int _t1 = 100; +int _t2 = 1; +int _t3 = 1; +int _t4 = 1; +int _t5 = 1; +int _t6 = 1; +int _t7 = 1; +int _t8 = 1; +int _t9 = 1; +int _t10 = 1; + +struct timeval begin_time; +struct timeval end_time; + +State> hh(*(new unordered_map())); +State> hh_counter(*(new unordered_map())); +State threshold(_t1); + +void +_init_() { + (new F_Type())->init(); +} + +int +process(Flow &f) { + if (((*(int *)f["flag_syn"]) == _t2) && + (hh[f][(*(IP *)f["sip"])] != _t3 && hh_counter[f][(*(IP *)f["sip"])] != threshold[f])) { + hh_counter[f][(*(IP *)f["sip"])] = hh_counter[f][(*(IP *)f["sip"])] + _t4; + } else if (((*(int *)f["flag_syn"]) == _t5) && + (hh[f][(*(IP *)f["sip"])] != _t6 && hh_counter[f][(*(IP *)f["sip"])] == threshold[f])) { + hh[f][(*(IP *)f["sip"])] = _t7; + } else if (((*(int *)f["flag_syn"]) == _t8) && (hh[f][(*(IP *)f["sip"])] == _t9)) { + return -1; + } else if ((*(int *)f["flag_syn"]) != _t10) { + } + f.clean(); + return 0; +} + +int +HHD(u_char *pkt, int totallength) { + f_glb = Flow(pkt, totallength); + return process(f_glb); +} + +void +stop() { + gettimeofday(&end_time, NULL); + + double total = end_time.tv_sec - begin_time.tv_sec + (end_time.tv_usec - begin_time.tv_usec) / 1000000.0; + + printf("\n\n**************************************************\n"); + printf("%ld packets are processed\n", _counter); + printf("NF runs for %f seconds\n", total); + printf("**************************************************\n\n"); +} + +/**********************************************************************/ + +/* + * Print a usage message + */ +static void +usage(const char *progname) { + printf("Usage: %s [EAL args] -- [NF_LIB args] -- -d -p \n\n", progname); +} + +/* + * Parse the application arguments. + */ +static int +parse_app_args(int argc, char *argv[], const char *progname) { + int c, dst_flag = 0; + + while ((c = getopt(argc, argv, "d:p:")) != -1) { + switch (c) { + case 'd': + destination = strtoul(optarg, NULL, 10); + dst_flag = 1; + break; + case 'p': + print_delay = strtoul(optarg, NULL, 10); + break; + case '?': + usage(progname); + if (optopt == 'd') + RTE_LOG(INFO, APP, "Option -%c requires an argument.\n", optopt); + else if (optopt == 'p') + RTE_LOG(INFO, APP, "Option -%c requires an argument.\n", optopt); + else if (isprint(optopt)) + RTE_LOG(INFO, APP, "Unknown option `-%c'.\n", optopt); + else + RTE_LOG(INFO, APP, "Unknown option character `\\x%x'.\n", optopt); + return -1; + default: + usage(progname); + return -1; + } + } + + if (!dst_flag) { + RTE_LOG(INFO, APP, "Heavy Hitter Detection NF requires destination flag -d.\n"); + return -1; + } + + return optind; +} + +/* + * This function displays stats. It uses ANSI terminal codes to clear + * screen when called. It is called from a single non-master + * thread in the server process, when the process is run with more + * than one lcore enabled. + */ +static void +do_stats_display(struct rte_mbuf *pkt) { + const char clr[] = {27, '[', '2', 'J', '\0'}; + const char topLeft[] = {27, '[', '1', ';', '1', 'H', '\0'}; + static uint64_t pkt_process = 0; + struct ipv4_hdr *ip; + + pkt_process += print_delay; + + /* Clear screen and move to top left */ + printf("%s%s", clr, topLeft); + + printf("PACKETS\n"); + printf("-----\n"); + printf("Port : %d\n", pkt->port); + printf("Size : %d\n", pkt->pkt_len); + printf("N° : %" PRIu64 "\n", pkt_process); + printf("\n\n"); + + ip = onvm_pkt_ipv4_hdr(pkt); + if (ip != NULL) { + onvm_pkt_print(pkt); + } else { + printf("No IP4 header found\n"); + } +} + +static int +packet_handler(struct rte_mbuf *buf, struct onvm_pkt_meta *meta, + __attribute__((unused)) struct onvm_nf_local_ctx *nf_local_ctx) { + static uint32_t counter = 0; + if (++counter == print_delay) { + do_stats_display(buf); + counter = 0; + } + ////////add NFD NF process here!//////////////// + _counter++; + u_char * pkt = rte_pktmbuf_mtod(buf, u_char*); + int length = (int)buf->pkt_len; + int ok; + + ok = HHD(pkt, length); + + if (ok == -1) { + meta->action = ONVM_NF_ACTION_DROP; + } else { + meta->action = ONVM_NF_ACTION_TONF; + meta->destination = destination; + } + + //////////////////////////////////////////////// + return 0; +} + +int +main(int argc, char *argv[]) { + struct onvm_nf_local_ctx *nf_local_ctx; + struct onvm_nf_function_table *nf_function_table; + int arg_offset; + + const char *progname = argv[0]; + + nf_local_ctx = onvm_nflib_init_nf_local_ctx(); + onvm_nflib_start_signal_handler(nf_local_ctx, NULL); + + nf_function_table = onvm_nflib_init_nf_function_table(); + nf_function_table->pkt_handler = &packet_handler; + + if ((arg_offset = onvm_nflib_init(argc, argv, NF_TAG, nf_local_ctx, nf_function_table)) < 0) { + onvm_nflib_stop(nf_local_ctx); + if (arg_offset == ONVM_SIGNAL_TERMINATION) { + printf("Exiting due to user termination\n"); + return 0; + } else { + rte_exit(EXIT_FAILURE, "Failed ONVM init\n"); + } + } + + argc -= arg_offset; + argv += arg_offset; + + // NFD begin + _init_(); + // NFD end + + if (parse_app_args(argc, argv, progname) < 0) { + onvm_nflib_stop(nf_local_ctx); + rte_exit(EXIT_FAILURE, "Invalid command-line arguments\n"); + } + + // NFD begin + gettimeofday(&begin_time, NULL); + // NFD end + + onvm_nflib_run(nf_local_ctx); + + onvm_nflib_stop(nf_local_ctx); + stop(); + printf("If we reach here, program is ending\n"); + + return 0; +} diff --git a/examples/NFD/heavy_hitter_detection/HHDmodel.txt b/examples/NFD/heavy_hitter_detection/HHDmodel.txt new file mode 100755 index 000000000..626f1bc26 --- /dev/null +++ b/examples/NFD/heavy_hitter_detection/HHDmodel.txt @@ -0,0 +1,40 @@ +/********************************************************************************** + NFD project + A C++ based NF developing framework designed by Wenfei's group + from IIIS, Tsinghua University, China. +******************************************************************************/ + +/************************************************************************************ +* Filename: HHDmodel.txt +* Author: Hongyi Huang(hhy17 AT mails.tsinghua.edu.cn), Bangwen Deng, Wenfei Wu +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: This file is a model file for NFD project, used to be compiled into NFD-NF +*************************************************************************************/ + +program HHD{ + map hh; + map hh_counter; + int threshold=100; + + entry{ + match_flow{f[flag_syn]==1 } + match_state{hh[f[sip]]!=1 && hh_counter[f[sip]]!=threshold} + action_state{hh_counter[f[sip]]=hh_counter[f[sip]]+1;} + } + entry{ + match_flow{f[flag_syn]==1} + match_state{hh[f[sip]]!=1 && hh_counter[f[sip]]==threshold} + action_state{hh[f[sip]]=1;} + } + entry{ + match_flow{f[flag_syn]==1} + match_state{hh[f[sip]]==1} + action_flow{f[sip]=DROP;} + } + entry{ + match_flow{f[flag_syn]!=1} + action_flow{pass;} + action_state{pass;} + } +} \ No newline at end of file diff --git a/examples/NFD/heavy_hitter_detection/Makefile b/examples/NFD/heavy_hitter_detection/Makefile new file mode 100644 index 000000000..3f5b214a2 --- /dev/null +++ b/examples/NFD/heavy_hitter_detection/Makefile @@ -0,0 +1,75 @@ +# openNetVM +# https://github.com/sdnfv/openNetVM +# +# BSD LICENSE +# +# Copyright(c) +# 2015-2017 George Washington University +# 2015-2017 University of California Riverside +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# The name of the author may not be used to endorse or promote +# products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +RTE_TARGET ?= x86_64-native-linuxapp-gcc + +# Default target, can be overriden by command line or environment +include $(RTE_SDK)/mk/rte.vars.mk + +# binary name +APP = heavy_hitter_detection + +# all source are stored in SRCS-y +SRCS-y := HHD.c + +# OpenNetVM, DPDK and NFD path +ONVM= $(SRCDIR)/../../../onvm +DPDK= $(SRCDIR)/../../../dpdk +NFD= $(SRCDIR)/.. + +CC=g++ + +CPPFLAGS = -O3 $(USER_FLAGS) -lpcap -std=c++11 -march=native +CPPFLAGS += -I$(ONVM)/onvm_nflib +CPPFLAGS += -I$(ONVM)/lib +CPPFLAGS += -I$(DPDK)/build/include +CPPFLAGS += -I$(NFD)/include/ + +LDFLAGS += $(ONVM)/onvm_nflib/$(RTE_TARGET)/libonvm.a +LDFLAGS += $(ONVM)/lib/$(RTE_TARGET)/lib/libonvmhelper.a +LDFLAGS += $(NFD)/lib/libNFD.a -lm -lstdc++ -lpcap + +# workaround for a gcc bug with noreturn attribute +# http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603 +ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +CFLAGS_main.o += -Wno-return-type +endif + +include $(RTE_SDK)/mk/rte.extapp.mk diff --git a/examples/NFD/heavy_hitter_detection/README.md b/examples/NFD/heavy_hitter_detection/README.md new file mode 100644 index 000000000..48335b64d --- /dev/null +++ b/examples/NFD/heavy_hitter_detection/README.md @@ -0,0 +1,71 @@ +HeavyHitterDetection +== +Heavy Hitter Detection developed by the NFD framework + + +NFD is a NF developing framework, consisting two parts, NFD language and NFD compiler. NFD compiler translate
+the NF model file wiiten in NFD language, a table-form language, into standard runtime environment(such as C++).

+ + +Heavy Hitter Detection is translated from the `HHDmodel.txt` to C++ environment.
+ +
+ +Heavy Hitter Detection keeps a counter for per flow and detects which flows consume most bandwidth by comparing the counters with a threshold. (threshold is set by user, in this program, we set it 100.) If the count reaches threshold, we will drop the following packets. + + +Testing +-- + +The Heavy Hitter Detection NF will start dropping SYN packets(from the same source IP address) after a certain threshold is reached, to verify this run these 2 NFs: + +Run the Heavy Hitter Detection NF with: + +``` +./go.sh 1 -d 2 + +``` + +Run Speed Tester NF(to replay pcap file) with: + +``` +./go.sh 2 -d 1 -o pcap/64B_download.pcap + +``` + + +Compilation and Execution +-- + +To run this NF, you should use either clang++ or g++ to compile this NF developed by C++. + +``` +cd heavy_hitter_detection +make + +``` + +Then, you can run this code. + +``` +./go.sh CORELIST SERVICE_ID DST [PRINT_DELAY] + +OR + +./go.sh -F CONFIG_FILE -- -- -d DST [-p PRINT_DELAY] + +OR + +sudo ./build/heavy_hitter_detection -l CORELIST -n 3 --proc-type=secondary -- -r SERVICE_ID -- -d DST [-p PRINT_DELAY] +``` + +App Specific Arguments +-- + - `-d `: destination service ID to foward to + - `-p `: number of packets between each print, e.g. `-p 1` prints every packets. + +Config File Support +-- +This NF supports the NF generating arguments from a config file. For additional reading, see [Examples.md](../../docs/Examples.md) + +See `../example_config.json` for all possible options that can be set. diff --git a/examples/NFD/heavy_hitter_detection/go.sh b/examples/NFD/heavy_hitter_detection/go.sh new file mode 120000 index 000000000..40babd806 --- /dev/null +++ b/examples/NFD/heavy_hitter_detection/go.sh @@ -0,0 +1 @@ +../go.sh \ No newline at end of file diff --git a/examples/NFD/include/basic_classes.h b/examples/NFD/include/basic_classes.h new file mode 100644 index 000000000..d4f644b32 --- /dev/null +++ b/examples/NFD/include/basic_classes.h @@ -0,0 +1,352 @@ +/********************************************************************************** + NFD project + A C++ based NF developing framework designed by Wenfei's group + from IIIS, Tsinghua University, China. +******************************************************************************/ + +/************************************************************************************ +* Filename: basic_classes.h +* Author: Hongyi Huang(hhy17 AT mails.tsinghua.edu.cn), Bangwen Deng, Wenfei Wu +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: This file is a supprot file for NFD project, defining the types maybe + used in NFD NF. +*************************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +using namespace std; +class IP; +typedef unordered_set ipset; +enum header { Iplen = 0, Sport = 1, Dport = 2, Tcp = 3, Udp = 4, Sip = 10, Dip = 11, Tag = 20 }; + +#define ERROR_HANDLE(x) std::cout << "Error Information: " << x << endl; +std::vector +split(const std::string& text, char sep); + +template +T +union_set(T& s1, T& s2) { + T result = s1; + result.insert(s2.cbegin(), s2.cend()); + return result; +} + +template +unordered_set& +create_set(unordered_set& ns, int count, ...) { + va_list ap; + va_start(ap, count); + for (int i = 0; i < count; i++) { + T item = *((T*)va_arg(ap, void*)); + ns.insert(item); + } + return ns; +} + +template +unordered_map& +create_map(unordered_map& ns, int count, ...) { + va_list ap; + va_start(ap, count); + for (int i = 0; i < count; i = i + 2) { + A key = *((A*)va_arg(ap, void*)); + B value = *((B*)va_arg(ap, void*)); + ns[key] = value; + } + return ns; +} + +class F_Type { + public: + static unordered_map MAP; + static unordered_map MAP2; + static void + init() { + /* TYPE int == 1 */ + MAP["dport"] = 1; + MAP["sport"] = 1; + + /* TYPE IP == 2 */ + MAP["sip"] = 2; + MAP["dip"] = 2; + + MAP2["sip"] = 2; + MAP2["dip"] = 3; + + /* TYPE others == 3 */ + MAP["tag"] = 3; + MAP2["tag"] = 1; + } + + static int + type_id(string& field, int* ret2) { + int ret1; + auto res = MAP.find(field); + if (res != MAP.end()) { + ret1 = res->second; + } else { + std::cout << "type_id SEARCHING " << field << " ERROR 12390" << endl; + ret1 = 0; + } + auto res2 = MAP2.find(field); + if (res2 != MAP2.end()) { + *ret2 = res2->second; + } else { + std::cout << "type_id SEARCHING " << field << " ERROR 12390" << endl; + *ret2 = 0; + } + return ret1; + } +}; + +class Flow { + u_char* pkt; + void* q = new string("error"); + unordered_map field_value; + friend struct FlowCmp; + + public: + void* headers[30]; + // Flow(int count, ...); + Flow(){}; + Flow(int* tag); + Flow(u_char* pkt, int totallength); + void*& operator[](const string& field); + int + matches(const string& field, const void* p); + void + clean(); +}; + +/*IP class for reserving IP*/ +class IP { + private: + public: + uint32_t ip; + uint32_t mask; + IP(const string& raw_ip, int raw_mask); + IP(int ip, int mask); + IP(const string& raw_ip); + IP() { + } + char* + showAddr(); + // bool contains(const IP& ip2) const; + // bool operator>=(const IP& other); + bool + operator<=(const IP& other); + bool + operator==(const IP& other) const; + bool + operator!=(const IP& other); +}; + +class Tuple { + private: + public: + vector ints; + vector ips; + Tuple(const vector& ins, const vector& is) { + this->ints = ins; + this->ips = is; + } +}; + +template +class State { + private: + vector
keywords; + unordered_map states; + T gl_state; + bool global = false; + + public: + T init; + int + getSize() { + if (this->global == true) + return 1; + return this->states.size(); + } + Tuple + create_tuple(Flow& f) { + vector v_int; + vector v_ip; + auto it = this->keywords.begin(); + for (; it != this->keywords.end(); it++) { + if (*it < 10) { + v_int.push_back(*((int*)f.headers[*it])); + continue; + } else if (*it >= 10 && *it < 20) { + v_ip.push_back(*((IP*)f.headers[*it])); + continue; + } else { + continue; + } + } + Tuple tp(v_int, v_ip); + return tp; + } + /*Initial*/ + /* count is number of fields/keywords to distinct two state instances*/ + State(T ini, string input) { + this->init = ini; + std::vector fields = split(input, '&'); + std::vector::iterator it = fields.begin(); + for (; it != fields.end(); it++) { + header h1; + if (*it == "iplen") { + h1 = Iplen; + } else if (*it == "sport") { + h1 = Sport; + } else if (*it == "dport") { + h1 = Dport; + } else if (*it == "sip") { + h1 = Sip; + } else if (*it == "dip") { + h1 = Dip; + } else if (*it == "tag") { + h1 = Tag; + } + this->keywords.push_back(h1); + } + if (input == "") + this->global = true; + return; + } + /* Globally shared state */ + State(T ini) { + this->gl_state = ini; + this->global = true; + return; + } + + /* [] return states of type T belonging to f*/ + T& operator[](Flow& f) { + /* + 1. new a Tuple + 2. see if f is among keys + a. yes, push_back a new pair + b. no, create new pair, initialize it + */ + if (this->global == false) { + auto tp = create_tuple(f); + auto it = states.find(tp); + if (it == states.end()) { + // not exist in keys of map + this->states[tp] = init; + } + return this->states[tp]; + } else { + return this->gl_state; + } + } +}; + +namespace std { +template <> +struct hash { + std::size_t + operator()(const IP& ip) const { + using std::hash; + using std::size_t; + + // Compute individual hash values for first, + // second and third and combine them using XOR + // and bit shifting: + + return ((hash()(ip.ip) ^ (hash()(ip.mask) << 1)) >> 1); + } +}; +template <> +struct hash> { + std::size_t + operator()(const vector ins) const { + using std::hash; + using std::size_t; + size_t ret = 1; + + auto lp = ins.begin(); + for (; lp != ins.end(); lp++) { + ret = ret ^ (hash()(*lp) << 1) >> 1; + } + return ret; + } +}; +template <> +struct hash> { + std::size_t + operator()(const vector ips) const { + using std::hash; + using std::size_t; + size_t ret = 1; + + auto lp = ips.begin(); + for (; lp != ips.end(); lp++) { + ret = ret ^ ((hash()((*lp).ip) ^ (hash()((*lp).mask) << 1)) >> 1); + } + return ret; + } +}; + +template <> +struct equal_to { + bool + operator()(const IP& lhs, const IP& rhs) const { + return (lhs.ip == rhs.ip) && (lhs.mask == rhs.mask); + } +}; +template <> +struct hash { + std::size_t + operator()(const Tuple& tp) const { + using std::hash; + using std::size_t; + + // Compute individual hash values for first, + // second and third and combine them using XOR + // and bit shifting: + + return ((hash>()(tp.ints) ^ (hash>()(tp.ips) << 1)) >> 1); + } +}; + +template <> +struct equal_to { + bool + operator()(const Tuple& lhs, const Tuple& rhs) const { + if (lhs.ints.size() == rhs.ints.size()) { + auto lp = lhs.ints.begin(); + auto rp = rhs.ints.begin(); + for (; lp != lhs.ints.end(); lp++, rp++) { + if (*lp != *rp) { + return false; + } + } + } else + return false; + if (lhs.ips.size() == rhs.ips.size()) { + auto lp = lhs.ips.begin(); + auto rp = rhs.ips.begin(); + for (; lp != lhs.ips.end(); lp++, rp++) { + if (!(*lp == *rp)) { + return false; + } + } + } else + return false; + return true; + } +}; +} // namespace std diff --git a/examples/NFD/include/decode.h b/examples/NFD/include/decode.h new file mode 100644 index 000000000..235b91c5a --- /dev/null +++ b/examples/NFD/include/decode.h @@ -0,0 +1,61 @@ +/********************************************************************************** + NFD project + A C++ based NF developing framework designed by Wenfei's group + from IIIS, Tsinghua University, China. +******************************************************************************/ + +/************************************************************************************ +* Filename: decode.h +* Author: Hongyi Huang(hhy17 AT mails.tsinghua.edu.cn), Bangwen Deng, Wenfei Wu +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: This file is a supprot file for NFD project, containing the methods maybe + used in NFD NF. +*************************************************************************************/ + +#define TH_FIN 0x01 +#define TH_SYN 0x02 +#define TH_RST 0x04 +#define TH_PUSH 0x08 +#define TH_ACK 0x10 +#define TH_URG 0x20 +typedef struct _EtherHdr { + unsigned char ether_dst[6]; + unsigned char ether_src[6]; + unsigned short ether_type; +} EtherHdr; + +typedef struct _IPHdr { +#if defined(WORDS_BIGENDIAN) + u_char ip_ver : 4, /* IP version */ + ip_hlen : 4; /* IP header length */ +#else + u_char ip_hlen : 4, ip_ver : 4; +#endif + u_char ip_tos; /* type of service */ + u_short ip_len; /* datagram length */ + u_short ip_id; /* identification */ + u_short ip_off; + u_char ip_ttl; /* time to live field */ + u_char ip_proto; /* datagram protocol */ + u_short ip_csum; /* checksum */ + struct in_addr ip_src; /* source IP */ + struct in_addr ip_dst; /* dest IP */ +} IPHdr; +typedef struct _TCPHdr { + u_short th_sport; /* source port */ + u_short th_dport; /* destination port */ + uint32_t th_seq; /* sequence number */ + uint32_t th_ack; /* acknowledgement number */ +#ifdef WORDS_BIGENDIAN + u_char th_off : 4, /* data offset */ + th_x2 : 4; /* (unused) */ +#else + u_char th_x2 : 4, /* (unused) */ + th_off : 4; /* data offset */ +#endif + u_char th_flags; + u_short th_win; /* window */ + u_short th_sum; /* checksum */ + u_short th_urp; /* urgent pointer */ +} TCPHdr; diff --git a/examples/NFD/lib/Makefile b/examples/NFD/lib/Makefile new file mode 100644 index 000000000..ddd80df60 --- /dev/null +++ b/examples/NFD/lib/Makefile @@ -0,0 +1,11 @@ + +# NFD= $(SRCDIR)/.. + +CC=g++ +CURRENTPATH= $(shell pwd) +# $(info $(CURRENTPATH)) +CPPFLAGS += -I $(CURRENTPATH)/../include -std=c++11 + + +all: basic_classes.o basic_methods.o + ar rcs libNFD.a basic_methods.o basic_classes.o diff --git a/examples/NFD/lib/basic_classes.cpp b/examples/NFD/lib/basic_classes.cpp new file mode 100644 index 000000000..4f3da09d0 --- /dev/null +++ b/examples/NFD/lib/basic_classes.cpp @@ -0,0 +1,122 @@ +/********************************************************************************** + NFD project + A C++ based NF developing framework designed by Wenfei's group + from IIIS, Tsinghua University, China. +******************************************************************************/ + +/************************************************************************************ +* Filename: basic_classes.cpp +* Author: Hongyi Huang(hhy17 AT mails.tsinghua.edu.cn), Bangwen Deng, Wenfei Wu +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: This file is a supprot file for NFD project, containing the types maybe + used in NFD NF. +*************************************************************************************/ + +#include "basic_classes.h" +#include +#include +#include +#include +#include +#include +#include "decode.h" + +using namespace std; + +// constructor 1 +IP::IP(const string& raw_ip) { + std::vector vec = split(raw_ip, '/'); + std::vector::iterator it = vec.begin(); + + string raw_ip1 = *it; + int raw_mask1 = std::stoi(*(++it)); + + std::vector vec1 = split(raw_ip1, '.'); + std::vector::iterator it1 = vec1.begin(); + this->ip = 0; + this->mask = 0; + for (; it1 != vec1.end(); it1++) { + uint8_t t = (uint8_t)std::stoi(*it1); + this->ip = (this->ip << 8) + t; + } + // UINT32_MAX included in stdint.h + this->mask = UINT32_MAX << (32 - raw_mask1); + return; +} +// constructor 2 +IP::IP(const string& raw_ip, int raw_mask) { + std::vector vec = split(raw_ip, '.'); + std::vector::iterator it = vec.begin(); + this->ip = 0; + this->mask = 0; + for (; it != vec.end(); it++) { + uint8_t t = (uint8_t)std::stoi(*it); + this->ip = (this->ip << 8) + t; + } + // UINT32_MAX included in stdint.h + this->mask = UINT32_MAX << (32 - raw_mask); + return; +} +// constructor 3, mask should be 0~32 +IP::IP(int ip, int raw_mask) { + this->ip = ip; + this->mask = UINT32_MAX << (32 - raw_mask); + return; +} + +char* +IP::showAddr() { + struct in_addr ip_addr; + ip_addr.s_addr = htonl(this->ip); + return inet_ntoa(ip_addr); +} + +bool +IP::operator<=(const IP& other) { + if ((other.mask <= this->mask) && ((other.mask & other.ip) == (other.mask & this->ip))) { + return true; + } else { + return false; + } +} +/*for type IP, two IPs are equivalent only if they share them same ip, as well as mask*/ +bool +IP::operator==(const IP& other) const { + return ((*this).ip == other.ip) && ((*this).mask == other.mask); +} +// reverse result of <= +bool +IP::operator!=(const IP& other) { + if ((other.mask <= this->mask) && ((other.mask & other.ip) == (other.mask & this->ip))) { + return false; + } else { + return true; + } +} + +Flow::Flow(int* tag) { + this->headers[Tag] = tag; + this->headers[Sip] = new IP(0, 0); + this->headers[Dip] = new IP(0, 0); + this->headers[Iplen] = new int(0); +} + +void*& Flow::operator[](const string& field) { + unordered_map::iterator it = field_value.find(field); + if (it == field_value.end()) { + ERROR_HANDLE("field " + field + " not in flow, now create a new entry, its tag is " + + to_string(*((int*)this->field_value["tag"]))); + // void * q = new string("error"); + Flow::field_value[field] = q; + return q; + } + return it->second; +} + +int +Flow::matches(const string& field, const void* p) { + if ((*this)[field] != NULL) { + } + return 1; +} diff --git a/examples/NFD/lib/basic_methods.cpp b/examples/NFD/lib/basic_methods.cpp new file mode 100644 index 000000000..47b25da11 --- /dev/null +++ b/examples/NFD/lib/basic_methods.cpp @@ -0,0 +1,29 @@ +/********************************************************************************** + NFD project + A C++ based NF developing framework designed by Wenfei's group + from IIIS, Tsinghua University, China. +******************************************************************************/ + +/************************************************************************************ +* Filename: basic_methods.cpp +* Author: Hongyi Huang(hhy17 AT mails.tsinghua.edu.cn), Bangwen Deng, Wenfei Wu +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: This file is a supprot file for NFD project, containing the methods maybe + used in NFD NF. +*************************************************************************************/ + +#include "basic_classes.h" + +/* function for spliting string by seperator*/ +std::vector +split(const std::string &text, char sep) { + std::vector tokens; + std::size_t start = 0, end = 0; + while ((end = text.find(sep, start)) != std::string::npos) { + tokens.push_back(text.substr(start, end - start)); + start = end + 1; + } + tokens.push_back(text.substr(start)); + return tokens; +} diff --git a/examples/NFD/napt/.gitignore b/examples/NFD/napt/.gitignore new file mode 100644 index 000000000..a5b24e36e --- /dev/null +++ b/examples/NFD/napt/.gitignore @@ -0,0 +1,2 @@ +napt/ +build/ diff --git a/examples/NFD/napt/Makefile b/examples/NFD/napt/Makefile new file mode 100644 index 000000000..95f2dd530 --- /dev/null +++ b/examples/NFD/napt/Makefile @@ -0,0 +1,75 @@ +# openNetVM +# https://github.com/sdnfv/openNetVM +# +# BSD LICENSE +# +# Copyright(c) +# 2015-2017 George Washington University +# 2015-2017 University of California Riverside +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# The name of the author may not be used to endorse or promote +# products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +RTE_TARGET ?= x86_64-native-linuxapp-gcc + +# Default target, can be overriden by command line or environment +include $(RTE_SDK)/mk/rte.vars.mk + +# binary name +APP = napt + +# all source are stored in SRCS-y +SRCS-y := NAPT.c + +# OpenNetVM, DPDK and NFD path +ONVM= $(SRCDIR)/../../../onvm +DPDK= $(SRCDIR)/../../../dpdk +NFD= $(SRCDIR)/.. + +CC=g++ + +CPPFLAGS = -O3 $(USER_FLAGS) -lpcap -std=c++11 -march=native +CPPFLAGS += -I$(ONVM)/onvm_nflib +CPPFLAGS += -I$(ONVM)/lib +CPPFLAGS += -I$(DPDK)/build/include +CPPFLAGS += -I$(NFD)/include/ + +LDFLAGS += $(ONVM)/onvm_nflib/$(RTE_TARGET)/libonvm.a +LDFLAGS += $(ONVM)/lib/$(RTE_TARGET)/lib/libonvmhelper.a +LDFLAGS += $(NFD)/lib/libNFD.a -lm -lstdc++ -lpcap + +# workaround for a gcc bug with noreturn attribute +# http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603 +ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +CFLAGS_main.o += -Wno-return-type +endif + +include $(RTE_SDK)/mk/rte.extapp.mk \ No newline at end of file diff --git a/examples/NFD/napt/NAPT.cpp b/examples/NFD/napt/NAPT.cpp new file mode 100644 index 000000000..521a111db --- /dev/null +++ b/examples/NFD/napt/NAPT.cpp @@ -0,0 +1,365 @@ +/********************************************************************* + * openNetVM + * https://sdnfv.github.io + * + * BSD LICENSE + * + * Copyright(c) + * 2015-2017 George Washington University + * 2015-2017 University of California Riverside + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * The name of the author may not be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ********************************************************************/ + +/************************************************************************************ +* Filename: NAPT.cpp +* Author: Hongyi Huang(hhy17 AT mails.tsinghua.edu.cn), Bangwen Deng, Wenfei Wu +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: This code is the implementation of state_firewall from NFD project, + a C++ based NF developing framework designed by Wenfei's group + from IIIS, Tsinghua University, China. +*************************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#include "onvm_nflib.h" +#include "onvm_pkt_helper.h" + +#ifdef __cplusplus +} +#endif + +// NFD ADD +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "basic_classes.h" +#include "decode.h" + +using namespace std; + +#define NF_TAG "NAPT" + +/* number of package between each print */ +static uint32_t print_delay = 1000000; + +static uint32_t destination; + +/*******************************NFD features********************************/ +Flow::Flow(u_char *packet, int totallength) { + this->pkt = packet; + int ethernet_header_length = 14; + + EtherHdr *e_hdr = (EtherHdr *)packet; + if (ntohs(e_hdr->ether_type) == 0x8100) + ethernet_header_length = 14 + 4; + else + ethernet_header_length = 14; + IPHdr *ip_hdr = (IPHdr *)(packet + ethernet_header_length); + int src_addr = ntohl(ip_hdr->ip_src.s_addr); + this->headers[Sip] = new IP(src_addr, 32); + int des_addr = ntohl(ip_hdr->ip_dst.s_addr); + this->headers[Dip] = new IP(des_addr, 32); + + int ip_header_length = ((*(packet + ethernet_header_length)) & 0x0F); + ip_header_length = ip_header_length * 4; + TCPHdr *tcph = (TCPHdr *)(packet + ethernet_header_length + ip_header_length); + this->headers[Sport] = new int(ntohs(tcph->th_sport)); + this->headers[Dport] = new int(ntohs(tcph->th_dport)); +} +void +Flow::clean() { + int ethernet_header_length = 14; + u_char *packet = this->pkt; + EtherHdr *e_hdr = (EtherHdr *)this->pkt; + if (ntohs(e_hdr->ether_type) == 0x8100) + ethernet_header_length = 14 + 4; + else + ethernet_header_length = 14; + + IPHdr *ip_hdr = (IPHdr *)(packet + ethernet_header_length); + + ip_hdr->ip_src.s_addr = htonl(((IP *)this->headers[Sip])->ip); + ip_hdr->ip_dst.s_addr = htonl(((IP *)this->headers[Dip])->ip); + + int ip_header_length = ((*(packet + ethernet_header_length)) & 0x0F); + ip_header_length = ip_header_length * 4; + TCPHdr *tcph = (TCPHdr *)(packet + ethernet_header_length + ip_header_length); + tcph->th_sport = htons(u_short(*((int *)this->headers[Sport]))); + tcph->th_dport = htons(u_short(*((int *)this->headers[Dport]))); +} + +long int _counter = 0; +unordered_map F_Type::MAP = unordered_map(); +unordered_map F_Type::MAP2 = unordered_map(); +int +process(Flow &f); +Flow f_glb; + +/////time//// +struct timeval begin_time; +struct timeval end_time; + +void +_init_() { + (new F_Type())->init(); +} + +IP _t1("192.168.0.0/16"); +IP _t2("219.168.135.100/32"); +int _t3=8; +int _t4=1; +State base(_t2); +State port(_t3); +State> listIP(*(new unordered_map())); +State> listPORT(*(new unordered_map())); + +int +process(Flow &f) { + if (*((IP *)f.headers[Sip]) <= _t1) { + listIP[f][port[f]] = (*(IP *)f.headers[Sip]); + listPORT[f][port[f]] = (*(int *)f.headers[Sport]); + (*(IP *)f.headers[Sip]) = base[f]; + (*(int *)f.headers[Sport]) = port[f]; + port[f] = port[f] + _t4; + } else if ((*((IP *)f.headers[Sip]) != _t1 && (*(IP *)f.headers[Dip]) == base[f]) && + (listIP[f].find((*(int *)f.headers[Dport])) != listIP[f].end())) { + (*(IP *)f.headers[Dip]) = listIP[f][(*(int *)f.headers[Dport])]; + (*(int *)f.headers[Dport]) = listPORT[f][(*(int *)f.headers[Dport])]; + } else if (((*((IP *)f.headers[Sip]) != _t1) && (*(IP *)f.headers[Dip]) == base[f]) && + (~(listIP[f].find((*(int *)f.headers[Dport])) != listIP[f].end()))) { + return -1; + } else if (*((IP *)f.headers[Sip]) != _t1 && (*(IP *)f.headers[Dip]) != base[f]) { + return -1; + } + f.clean(); + return 0; +} + +int +NAPT(u_char *pkt, int totallength) { + f_glb = Flow(pkt, totallength); + return process(f_glb); +} + +void +stop() { + gettimeofday(&end_time, NULL); + + double total = end_time.tv_sec - begin_time.tv_sec + (end_time.tv_usec - begin_time.tv_usec) / 1000000.0; + + printf("\n\n**************************************************\n"); + printf("%ld packets are processed\n", _counter); + printf("NF runs for %f seconds\n", total); + printf("**************************************************\n\n"); +} + +/**********************************************************************/ + +/* + * Print a usage message + */ +static void +usage(const char *progname) { + printf("Usage: %s [EAL args] -- [NF_LIB args] -- -d -p \n\n", progname); +} + +/* + * Parse the application arguments. + */ +static int +parse_app_args(int argc, char *argv[], const char *progname) { + int c, dst_flag = 0; + + while ((c = getopt(argc, argv, "d:p:")) != -1) { + switch (c) { + case 'd': + destination = strtoul(optarg, NULL, 10); + dst_flag = 1; + break; + case 'p': + print_delay = strtoul(optarg, NULL, 10); + break; + case '?': + usage(progname); + if (optopt == 'd') + RTE_LOG(INFO, APP, "Option -%c requires an argument.\n", optopt); + else if (optopt == 'p') + RTE_LOG(INFO, APP, "Option -%c requires an argument.\n", optopt); + else if (isprint(optopt)) + RTE_LOG(INFO, APP, "Unknown option `-%c'.\n", optopt); + else + RTE_LOG(INFO, APP, "Unknown option character `\\x%x'.\n", optopt); + return -1; + default: + usage(progname); + return -1; + } + } + + if (!dst_flag) { + RTE_LOG(INFO, APP, "NAPT NF requires destination flag -d.\n"); + return -1; + } + + return optind; +} + +/* + * This function displays stats. It uses ANSI terminal codes to clear + * screen when called. It is called from a single non-master + * thread in the server process, when the process is run with more + * than one lcore enabled. + */ +static void +do_stats_display(struct rte_mbuf *pkt) { + const char clr[] = {27, '[', '2', 'J', '\0'}; + const char topLeft[] = {27, '[', '1', ';', '1', 'H', '\0'}; + static uint64_t pkt_process = 0; + struct ipv4_hdr *ip; + + pkt_process += print_delay; + + /* Clear screen and move to top left */ + printf("%s%s", clr, topLeft); + + printf("PACKETS\n"); + printf("-----\n"); + printf("Port : %d\n", pkt->port); + printf("Size : %d\n", pkt->pkt_len); + printf("N° : %" PRIu64 "\n", pkt_process); + printf("\n\n"); + + ip = onvm_pkt_ipv4_hdr(pkt); + if (ip != NULL) { + onvm_pkt_print(pkt); + } else { + printf("No IP4 header found\n"); + } +} + +static int +packet_handler(struct rte_mbuf *buf, struct onvm_pkt_meta *meta, + __attribute__((unused)) struct onvm_nf_local_ctx *nf_local_ctx) { + static uint32_t counter = 0; + if (++counter == print_delay) { + do_stats_display(buf); + counter = 0; + } + ////////add NFD NF process here!//////////////// + _counter++; + u_char * pkt = rte_pktmbuf_mtod(buf, u_char*); + int length = (int)buf->pkt_len; + int ok; + + ok = NAPT(pkt, length); + + if (ok == -1) { + meta->action = ONVM_NF_ACTION_DROP; + } else { + meta->action = ONVM_NF_ACTION_TONF; + meta->destination = destination; + } + //////////////////////////////////////////////// + return 0; +} + +int +main(int argc, char *argv[]) { + struct onvm_nf_local_ctx *nf_local_ctx; + struct onvm_nf_function_table *nf_function_table; + int arg_offset; + + const char *progname = argv[0]; + + nf_local_ctx = onvm_nflib_init_nf_local_ctx(); + onvm_nflib_start_signal_handler(nf_local_ctx, NULL); + + nf_function_table = onvm_nflib_init_nf_function_table(); + nf_function_table->pkt_handler = &packet_handler; + + if ((arg_offset = onvm_nflib_init(argc, argv, NF_TAG, nf_local_ctx, nf_function_table)) < 0) { + onvm_nflib_stop(nf_local_ctx); + if (arg_offset == ONVM_SIGNAL_TERMINATION) { + printf("Exiting due to user termination\n"); + return 0; + } else { + rte_exit(EXIT_FAILURE, "Failed ONVM init\n"); + } + } + + argc -= arg_offset; + argv += arg_offset; + + // NFD begin + _init_(); + // NFD end + + if (parse_app_args(argc, argv, progname) < 0) { + onvm_nflib_stop(nf_local_ctx); + rte_exit(EXIT_FAILURE, "Invalid command-line arguments\n"); + } + + // NFD begin + gettimeofday(&begin_time, NULL); + // NFD end + + onvm_nflib_run(nf_local_ctx); + + onvm_nflib_stop(nf_local_ctx); + stop(); + printf("If we reach here, program is ending\n"); + + return 0; +} diff --git a/examples/NFD/napt/README.md b/examples/NFD/napt/README.md new file mode 100644 index 000000000..7fc7efb72 --- /dev/null +++ b/examples/NFD/napt/README.md @@ -0,0 +1,74 @@ +NAPT +== +NAPT developed by the NFD framework + + +NFD is a NF developing framework, consisting two parts, NFD language and NFD compiler. NFD compiler translate
+the NF model file wiiten in NFD language, a table-form language, into standard runtime environment(such as C++).

+ + +NAPT is translated from the `model.txt` to C++ environment.
+ +
+ +NAPT is short for Network Address Port Translation. It enables mappings from tuples(address, L4 port number) to tuples(registered address and assigned port number) to complete address translation. +
+ + +Testing +-- + +The NAPT NF will do the simple network address translation jobs. To verify it, first you need to generate two kinds of packets:1) srcIP is in the range(192.168.0.0/16 in the program); 2)dstIP is the NAT's IP (219.168.135.100 in program). + +Run these 2 NFs: + +Run the NAPT NF with: + +``` +./go.sh 1 -d 2 + +``` + +Run Speed Tester NF(to replay pcap file) with: + +``` +./go.sh 2 -d 1 -o pcap/trace.pcap + +``` + + +Compilation and Execution +-- + +To run this NF, you should use either clang++ or g++ to compile this NF developed by C++. + +``` +cd examples/NFD/napt +make + +``` + +Then, you can run this code. + +``` +./go.sh CORELIST SERVICE_ID DST [PRINT_DELAY] + +OR + +./go.sh -F CONFIG_FILE -- -- -d DST [-p PRINT_DELAY] + +OR + +sudo ./build/napt -l CORELIST -n 3 --proc-type=secondary -- -r SERVICE_ID -- -d DST [-p PRINT_DELAY] +``` + +App Specific Arguments +-- + - `-d `: destination service ID to foward to + - `-p `: number of packets between each print, e.g. `-p 1` prints every packets. + +Config File Support +-- +This NF supports the NF generating arguments from a config file. For additional reading, see [Examples.md](../../docs/Examples.md) + +See `../example_config.json` for all possible options that can be set. diff --git a/examples/NFD/napt/go.sh b/examples/NFD/napt/go.sh new file mode 120000 index 000000000..40babd806 --- /dev/null +++ b/examples/NFD/napt/go.sh @@ -0,0 +1 @@ +../go.sh \ No newline at end of file diff --git a/examples/NFD/napt/model.txt b/examples/NFD/napt/model.txt new file mode 100755 index 000000000..736c8f041 --- /dev/null +++ b/examples/NFD/napt/model.txt @@ -0,0 +1,46 @@ +/********************************************************************************** + NFD project + A C++ based NF developing framework designed by Wenfei's group + from IIIS, Tsinghua University, China. +******************************************************************************/ + +/************************************************************************************ +* Filename: model.txt +* Author: Hongyi Huang(hhy17 AT mails.tsinghua.edu.cn), Bangwen Deng, Wenfei Wu +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: This file is a model file for NFD project, used to be compiled into NFD-NF +*************************************************************************************/ + +program NAPT{ + rule R=sip:192.168.0.0/16; + IP base=219.168.135.100; + int port=8; + map listIP; + map listPORT; + + entry{ + match_flow{f matches R } + action_state { listIP[port]=f[sip]; listPORT[port]=f[sport];} + action_flow{f[sip]=base; f[sport]=port; port=port+1;} + + } + + entry{ + match_flow{f mismatches R && f[dip] == base} + match_state{ f[dport] in listIP } + action_flow{f[dip]=listIP[f[dport]]; f[dport]=listPORT[f[dport]];} + + } + entry{ + match_flow{f mismatches R && f[dip] == base} + match_state{ ~(f[dport] in listIP) } + action_flow{f[dip]=DROP; } + } + + entry{ + match_flow{f mismatches R && f[dip] != base} + action_flow{f[dip]=DROP; } + } +} + diff --git a/examples/NFD/stateful_firewall/.gitignore b/examples/NFD/stateful_firewall/.gitignore new file mode 100644 index 000000000..6338c1427 --- /dev/null +++ b/examples/NFD/stateful_firewall/.gitignore @@ -0,0 +1,2 @@ +stateful_firewall/ +build/ diff --git a/examples/NFD/stateful_firewall/Makefile b/examples/NFD/stateful_firewall/Makefile new file mode 100644 index 000000000..106c621f4 --- /dev/null +++ b/examples/NFD/stateful_firewall/Makefile @@ -0,0 +1,75 @@ +# openNetVM +# https://github.com/sdnfv/openNetVM +# +# BSD LICENSE +# +# Copyright(c) +# 2015-2017 George Washington University +# 2015-2017 University of California Riverside +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# The name of the author may not be used to endorse or promote +# products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +RTE_TARGET ?= x86_64-native-linuxapp-gcc + +# Default target, can be overriden by command line or environment +include $(RTE_SDK)/mk/rte.vars.mk + +# binary name +APP = stateful_firewall + +# all source are stored in SRCS-y +SRCS-y := stateful_firewall.c + +# OpenNetVM, DPDK and NFD path +ONVM= $(SRCDIR)/../../../onvm +DPDK= $(SRCDIR)/../../../dpdk +NFD= $(SRCDIR)/.. + +CC=g++ + +CPPFLAGS = -O3 $(USER_FLAGS) -lpcap -std=c++11 -march=native +CPPFLAGS += -I$(ONVM)/onvm_nflib +CPPFLAGS += -I$(ONVM)/lib +CPPFLAGS += -I$(DPDK)/build/include +CPPFLAGS += -I$(NFD)/include/ + +LDFLAGS += $(ONVM)/onvm_nflib/$(RTE_TARGET)/libonvm.a +LDFLAGS += $(ONVM)/lib/$(RTE_TARGET)/lib/libonvmhelper.a +LDFLAGS += $(NFD)/lib/libNFD.a -lm -lstdc++ -lpcap + +# workaround for a gcc bug with noreturn attribute +# http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603 +ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +CFLAGS_main.o += -Wno-return-type +endif + +include $(RTE_SDK)/mk/rte.extapp.mk \ No newline at end of file diff --git a/examples/NFD/stateful_firewall/README.md b/examples/NFD/stateful_firewall/README.md new file mode 100644 index 000000000..8f06026d1 --- /dev/null +++ b/examples/NFD/stateful_firewall/README.md @@ -0,0 +1,76 @@ +Stateful firewall +== +Stateful firewall developed by the NFD framework + + +NFD is a NF developing framework, consisting two parts, NFD language and NFD compiler. NFD compiler translate
+the NF model file wiiten in NFD language, a table-form language, into standard runtime environment(such as C++).

+ + +Stateful firewall is translated from the `model.txt` to C++ environment and describes the following table-form logic.
+ + +**Match flow** |**Match state** | **Action flow** | **Action state** + --------- | ----------- | ----------- |---------- + outgoing | * | pass | record as seen + incoming | seen | pass | - + incoming | not seen | drop | - + +
+ +All outgoing flows are allowed and recorded, all incoming flows initiated by an outgoing flow are also allowed, all incoming flows without initiation are dropped.
+ + +Testing +-- + +To test stateful firewall NF functionality, we need some traces which have packets with source IP of `192.168.22.0/24` or just modify the ALLOW(`line 40: IP _t1()` in source code) into your ALLOW nwetwork. Run the stateful firewall NF: + +``` +./go.sh 1 -d 2 + +``` + +Run Speed Tester NF(to replay pcap file) with: + +``` +./go.sh 2 -d 1 -o pcap/trace.pcap + +``` + + +Compilation and Execution +-- + +To run this NF, you should use either clang++ or g++ to compile this NF developed by C++. + +``` +cd stateful_firewall +make + +``` + +Then, you can run this code. + +``` +./go.sh CORELIST SERVICE_ID DST [PRINT_DELAY] + +OR + +./go.sh -F CONFIG_FILE -- -- -d DST [-p PRINT_DELAY] + +OR + +sudo ./build/stateful_firewall -l CORELIST -n 3 --proc-type=secondary -- -r SERVICE_ID -- -d DST [-p PRINT_DELAY] +``` + +App Specific Arguments +-- + - `-d `: destination service ID to foward to + - `-p `: number of packets between each print, e.g. `-p 1` prints every packets. + +Config File Support +-- +This NF supports the NF generating arguments from a config file. For additional reading, see [Examples.md](../../docs/Examples.md) + +See `../example_config.json` for all possible options that can be set. diff --git a/examples/NFD/stateful_firewall/go.sh b/examples/NFD/stateful_firewall/go.sh new file mode 120000 index 000000000..40babd806 --- /dev/null +++ b/examples/NFD/stateful_firewall/go.sh @@ -0,0 +1 @@ +../go.sh \ No newline at end of file diff --git a/examples/NFD/stateful_firewall/model.txt b/examples/NFD/stateful_firewall/model.txt new file mode 100755 index 000000000..0c00d5786 --- /dev/null +++ b/examples/NFD/stateful_firewall/model.txt @@ -0,0 +1,31 @@ +/********************************************************************************** + NFD project + A C++ based NF developing framework designed by Wenfei's group + from IIIS, Tsinghua University, China. +******************************************************************************/ + +/************************************************************************************ +* Filename: model.txt +* Author: Hongyi Huang(hhy17 AT mails.tsinghua.edu.cn), Bangwen Deng, Wenfei Wu +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: This file is a model file for NFD project, used to be compiled into NFD-NF +*************************************************************************************/ + +program IDS{ + rule ALLOW = sip:192.168.22.0/24; + set seen; + entry { + match_flow { f matches ALLOW } + action_state { seen= seen | {f[dip]} ; } + } + entry { + match_flow { f mismatches ALLOW } + match_state { f[sip] in seen } + } + entry { + match_flow { f mismatches ALLOW } + match_state { ~ (f[sip] in seen) } + action_flow { f[dip]= DROP; } + } +} diff --git a/examples/NFD/stateful_firewall/stateful_firewall.cpp b/examples/NFD/stateful_firewall/stateful_firewall.cpp new file mode 100644 index 000000000..64a665cd0 --- /dev/null +++ b/examples/NFD/stateful_firewall/stateful_firewall.cpp @@ -0,0 +1,332 @@ +/********************************************************************* + * openNetVM + * https://sdnfv.github.io + * + * BSD LICENSE + * + * Copyright(c) + * 2015-2017 George Washington University + * 2015-2017 University of California Riverside + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * The name of the author may not be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ********************************************************************/ + +/************************************************************************************ +* Filename: stateful_firewall.cpp +* Author: Hongyi Huang(hhy17 AT mails.tsinghua.edu.cn), Bangwen Deng, Wenfei Wu +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: This code is the implementation of state_firewall from NFD project, + a C++ based NF developing framework designed by Wenfei's group + from IIIS, Tsinghua University, China. +*************************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#include "onvm_nflib.h" +#include "onvm_pkt_helper.h" + +#ifdef __cplusplus +} +#endif + +// NFD ADD +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "basic_classes.h" +#include "decode.h" + +using namespace std; + +#define NF_TAG "stateful_firewall" + +/* number of package between each print */ +static uint32_t print_delay = 1000000; + +static uint32_t destination; + +/*******************************NFD features********************************/ +Flow::Flow(u_char *packet, int totallength) { + /*Decoding*/ + this->pkt = packet; + int ethernet_header_length = 14; + /* Header lengths in bytes */ + EtherHdr *e_hdr = (EtherHdr *)packet; + if (ntohs(e_hdr->ether_type) == 0x8100) + ethernet_header_length = 14 + 4; /* For 802.1Q Virtual LAN */ + else + ethernet_header_length = 14; /* For general wired */ + + IPHdr *ip_hdr = (IPHdr *)(packet + ethernet_header_length); + int src_addr = ntohl(ip_hdr->ip_src.s_addr); + this->headers[Sip] = new IP(src_addr, 32); + int dst_addr = ntohl(ip_hdr->ip_dst.s_addr); + this->headers[Dip] = new IP(dst_addr, 32); +} +void +Flow::clean() { + /*Encoding*/ +} + +long int _counter = 0; +long int _drop = 0; +unordered_map F_Type::MAP = unordered_map(); +unordered_map F_Type::MAP2 = unordered_map(); +int +process(Flow &f); +Flow f_glb; + +/////time//// +struct timeval begin_time; +struct timeval end_time; + +void +_init_() { + (new F_Type())->init(); +} + +// this setting was set by the model.txt +IP _t1("192.168.22.0/24"); + +State> seen(*(new unordered_set())); + +int +process(Flow &f) { + if (*((IP *)f.headers[Sip]) <= _t1) { + seen[f] = union_set>( + seen[f], create_set(*(new unordered_set()), 1, &((*(IP *)f.headers[Dip])))); + } else if ((*((IP *)f.headers[Sip]) != _t1) && (seen[f].find((*(IP *)f.headers[Sip])) != seen[f].end())) { + } else if ((*((IP *)f.headers[Sip]) != _t1) && (~(seen[f].find((*(IP *)f.headers[Sip])) != seen[f].end()))) { + return -1; + } + + f.clean(); + return 0; +} +int +stateful_firewall(u_char *pkt, int totallength) { + f_glb = Flow(pkt, totallength); + return process(f_glb); +} + +void +stop() { + gettimeofday(&end_time, NULL); + + double total = end_time.tv_sec - begin_time.tv_sec + (end_time.tv_usec - begin_time.tv_usec) / 1000000.0; + + printf("\n\n**************************************************\n"); + printf("%ld packets are processed, %ld packets are dropped\n", _counter, _drop); + printf("NF runs for %f seconds\n", total); + printf("**************************************************\n\n"); +} + +/**********************************************************************/ + +/* + * Print a usage message + */ +static void +usage(const char *progname) { + printf("Usage: %s [EAL args] -- [NF_LIB args] -- -d -p \n\n", progname); +} + +/* + * Parse the application arguments. + */ +static int +parse_app_args(int argc, char *argv[], const char *progname) { + int c, dst_flag = 0; + + while ((c = getopt(argc, argv, "d:p:")) != -1) { + switch (c) { + case 'd': + destination = strtoul(optarg, NULL, 10); + dst_flag = 1; + break; + case 'p': + print_delay = strtoul(optarg, NULL, 10); + break; + case '?': + usage(progname); + if (optopt == 'd') + RTE_LOG(INFO, APP, "Option -%c requires an argument.\n", optopt); + else if (optopt == 'p') + RTE_LOG(INFO, APP, "Option -%c requires an argument.\n", optopt); + else if (isprint(optopt)) + RTE_LOG(INFO, APP, "Unknown option `-%c'.\n", optopt); + else + RTE_LOG(INFO, APP, "Unknown option character `\\x%x'.\n", optopt); + return -1; + default: + usage(progname); + return -1; + } + } + + if (!dst_flag) { + RTE_LOG(INFO, APP, "Stateful Firewall NF requires destination flag -d.\n"); + return -1; + } + + return optind; +} + +/* + * This function displays stats. It uses ANSI terminal codes to clear + * screen when called. It is called from a single non-master + * thread in the server process, when the process is run with more + * than one lcore enabled. + */ +static void +do_stats_display(struct rte_mbuf *pkt) { + const char clr[] = {27, '[', '2', 'J', '\0'}; + const char topLeft[] = {27, '[', '1', ';', '1', 'H', '\0'}; + static uint64_t pkt_process = 0; + struct ipv4_hdr *ip; + + pkt_process += print_delay; + + /* Clear screen and move to top left */ + printf("%s%s", clr, topLeft); + + printf("PACKETS\n"); + printf("-----\n"); + printf("Port : %d\n", pkt->port); + printf("Size : %d\n", pkt->pkt_len); + printf("N° : %" PRIu64 "\n", pkt_process); + printf("\n\n"); + + ip = onvm_pkt_ipv4_hdr(pkt); + if (ip != NULL) { + onvm_pkt_print(pkt); + } else { + printf("No IP4 header found\n"); + } +} + +static int +packet_handler(struct rte_mbuf *buf, struct onvm_pkt_meta *meta, + __attribute__((unused)) struct onvm_nf_local_ctx *nf_local_ctx) { + static uint32_t counter = 0; + if (++counter == print_delay) { + do_stats_display(buf); + counter = 0; + } + ////////add NFD NF process here!//////////////// + _counter++; + u_char * pkt = rte_pktmbuf_mtod(buf, u_char*); + int length = (int)buf->pkt_len; + int ok; + + ok = stateful_firewall(pkt, length); + if (ok == -1) { + _drop++; + meta->action = ONVM_NF_ACTION_DROP; + } else { + meta->action = ONVM_NF_ACTION_TONF; + meta->destination = destination; + } + //////////////////////////////////////////////// + return 0; +} + +int +main(int argc, char *argv[]) { + struct onvm_nf_local_ctx *nf_local_ctx; + struct onvm_nf_function_table *nf_function_table; + int arg_offset; + + const char *progname = argv[0]; + + nf_local_ctx = onvm_nflib_init_nf_local_ctx(); + onvm_nflib_start_signal_handler(nf_local_ctx, NULL); + + nf_function_table = onvm_nflib_init_nf_function_table(); + nf_function_table->pkt_handler = &packet_handler; + + if ((arg_offset = onvm_nflib_init(argc, argv, NF_TAG, nf_local_ctx, nf_function_table)) < 0) { + onvm_nflib_stop(nf_local_ctx); + if (arg_offset == ONVM_SIGNAL_TERMINATION) { + printf("Exiting due to user termination\n"); + return 0; + } else { + rte_exit(EXIT_FAILURE, "Failed ONVM init\n"); + } + } + + argc -= arg_offset; + argv += arg_offset; + + // NFD begin + _init_(); + // NFD end + + if (parse_app_args(argc, argv, progname) < 0) { + onvm_nflib_stop(nf_local_ctx); + rte_exit(EXIT_FAILURE, "Invalid command-line arguments\n"); + } + + // NFD begin + gettimeofday(&begin_time, NULL); + // NFD end + + onvm_nflib_run(nf_local_ctx); + + onvm_nflib_stop(nf_local_ctx); + stop(); + printf("If we reach here, program is ending\n"); + + return 0; +} diff --git a/examples/NFD/stateless_firewall/.gitignore b/examples/NFD/stateless_firewall/.gitignore new file mode 100644 index 000000000..aec14776e --- /dev/null +++ b/examples/NFD/stateless_firewall/.gitignore @@ -0,0 +1,2 @@ +stateless_firewall/ +build/ diff --git a/examples/NFD/stateless_firewall/Makefile b/examples/NFD/stateless_firewall/Makefile new file mode 100644 index 000000000..d4a68313a --- /dev/null +++ b/examples/NFD/stateless_firewall/Makefile @@ -0,0 +1,75 @@ +# openNetVM +# https://github.com/sdnfv/openNetVM +# +# BSD LICENSE +# +# Copyright(c) +# 2015-2017 George Washington University +# 2015-2017 University of California Riverside +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# The name of the author may not be used to endorse or promote +# products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +RTE_TARGET ?= x86_64-native-linuxapp-gcc + +# Default target, can be overriden by command line or environment +include $(RTE_SDK)/mk/rte.vars.mk + +# binary name +APP = stateless_firewall + +# all source are stored in SRCS-y +SRCS-y := stateless_firewall.c + +# OpenNetVM, DPDK and NFD path +ONVM= $(SRCDIR)/../../../onvm +DPDK= $(SRCDIR)/../../../dpdk +NFD= $(SRCDIR)/.. + +CC=g++ + +CPPFLAGS = -O3 $(USER_FLAGS) -lpcap -std=c++11 -march=native +CPPFLAGS += -I$(ONVM)/onvm_nflib +CPPFLAGS += -I$(ONVM)/lib +CPPFLAGS += -I$(DPDK)/build/include +CPPFLAGS += -I$(NFD)/include/ + +LDFLAGS += $(ONVM)/onvm_nflib/$(RTE_TARGET)/libonvm.a +LDFLAGS += $(ONVM)/lib/$(RTE_TARGET)/lib/libonvmhelper.a +LDFLAGS += $(NFD)/lib/libNFD.a -lm -lstdc++ -lpcap + +# workaround for a gcc bug with noreturn attribute +# http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603 +ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +CFLAGS_main.o += -Wno-return-type +endif + +include $(RTE_SDK)/mk/rte.extapp.mk \ No newline at end of file diff --git a/examples/NFD/stateless_firewall/README.md b/examples/NFD/stateless_firewall/README.md new file mode 100644 index 000000000..b2a2a1df3 --- /dev/null +++ b/examples/NFD/stateless_firewall/README.md @@ -0,0 +1,77 @@ +Stateless Firewall +== +Stateless Firewall developed by the NFD framework + + +NFD is a NF developing framework, consisting two parts, NFD language and NFD compiler. NFD compiler translate
+the NF model file wiiten in NFD language, a table-form language, into standard runtime environment(such as C++).

+ + +Stateless firewall is translated from the `model.txt` to C++ environment and describes the following table-form logic.
+ + +**Match flow** |**Match state** | **Action flow** | **Action state** + --------- | ----------- | ----------- |---------- + allow | * | pass | - + not allow | * | drop | - + + +
+ Stateless Firewall allows connections satisfying the whitelist or blocks those hitting the blacklist. + +
+ + +Testing +-- + +To test stateless firewall NF functionality, we need some traces which have packets with source IP of `192.168.22.0/24` or just modify the ALLOW(`line 133: IP _t1()` in source code) into your ALLOW nwetwork. It wiil drop all packets not belong to ALLOW. Run the stateless firewall NF: + +``` +./go.sh 1 -d 2 + +``` + +Run Speed Tester NF(to replay pcap file) with: + +``` +./go.sh 2 -d 1 -o pcap/trace.pcap + +``` + + +Compilation and Execution +-- + +To run this NF, you should use either clang++ or g++ to compile this NF developed by C++. + +``` +cd stateless_firewall +make + +``` + +Then, you can run this code. + +``` +./go.sh CORELIST SERVICE_ID DST [PRINT_DELAY] + +OR + +./go.sh -F CONFIG_FILE -- -- -d DST [-p PRINT_DELAY] + +OR + +sudo ./build/stateless_firewall -l CORELIST -n 3 --proc-type=secondary -- -r SERVICE_ID -- -d DST [-p PRINT_DELAY] +``` + +App Specific Arguments +-- + - `-d `: destination service ID to foward to + - `-p `: number of packets between each print, e.g. `-p 1` prints every packets. + +Config File Support +-- +This NF supports the NF generating arguments from a config file. For additional reading, see [Examples.md](../../docs/Examples.md) + +See `../example_config.json` for all possible options that can be set. diff --git a/examples/NFD/stateless_firewall/go.sh b/examples/NFD/stateless_firewall/go.sh new file mode 120000 index 000000000..40babd806 --- /dev/null +++ b/examples/NFD/stateless_firewall/go.sh @@ -0,0 +1 @@ +../go.sh \ No newline at end of file diff --git a/examples/NFD/stateless_firewall/model.txt b/examples/NFD/stateless_firewall/model.txt new file mode 100755 index 000000000..0e16d1fce --- /dev/null +++ b/examples/NFD/stateless_firewall/model.txt @@ -0,0 +1,24 @@ +/********************************************************************************** + NFD project + A C++ based NF developing framework designed by Wenfei's group + from IIIS, Tsinghua University, China. +******************************************************************************/ + +/************************************************************************************ +* Filename: model.txt +* Author: Hongyi Huang(hhy17 AT mails.tsinghua.edu.cn), Bangwen Deng, Wenfei Wu +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: This file is a model file for NFD project, used to be compiled into NFD-NF +*************************************************************************************/ + +program IDS{ + rule ALLOW = sip:192.168.22.0/24; + entry { + match_flow { f matches ALLOW } + } + entry { + match_flow { f mismatches ALLOW } + action_flow { f[dip]= DROP; } + } +} \ No newline at end of file diff --git a/examples/NFD/stateless_firewall/stateless_firewall.cpp b/examples/NFD/stateless_firewall/stateless_firewall.cpp new file mode 100644 index 000000000..d58188e64 --- /dev/null +++ b/examples/NFD/stateless_firewall/stateless_firewall.cpp @@ -0,0 +1,325 @@ +/********************************************************************* + * openNetVM + * https://sdnfv.github.io + * + * BSD LICENSE + * + * Copyright(c) + * 2015-2017 George Washington University + * 2015-2017 University of California Riverside + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * The name of the author may not be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ********************************************************************/ + +/************************************************************************************ +* Filename: stateful_firewall.cpp +* Author: Hongyi Huang(hhy17 AT mails.tsinghua.edu.cn), Bangwen Deng, Wenfei Wu +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: This code is the implementation of state_firewall from NFD project, + a C++ based NF developing framework designed by Wenfei's group + from IIIS, Tsinghua University, China. +*************************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#include "onvm_nflib.h" +#include "onvm_pkt_helper.h" + +#ifdef __cplusplus +} +#endif + +// NFD ADD +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "basic_classes.h" +#include "decode.h" + +using namespace std; + +#define NF_TAG "stateless_firewall" + +/* number of package between each print */ +static uint32_t print_delay = 1000000; + +static uint32_t destination; + +/*******************************NFD features********************************/ +Flow::Flow(u_char *packet, int totallength) { + /*Decoding*/ + this->pkt = packet; + int ethernet_header_length = 14; + /* Header lengths in bytes */ + EtherHdr *e_hdr = (EtherHdr *)packet; + if (ntohs(e_hdr->ether_type) == 0x8100) + ethernet_header_length = 14 + 4; /* For 802.1Q Virtual LAN */ + else + ethernet_header_length = 14; /* For general wired */ + + IPHdr *ip_hdr = (IPHdr *)(packet + ethernet_header_length); + int src_addr = ntohl(ip_hdr->ip_src.s_addr); + this->headers[Sip] = new IP(src_addr, 32); + short protocol = (short)(ip_hdr->ip_proto); + this->headers[Tcp] = new int((protocol == IPPROTO_TCP) ? 1 : 0); +} +void +Flow::clean() { +} + +long int _counter = 0; +long int _drop = 0; +unordered_map F_Type::MAP = unordered_map(); +unordered_map F_Type::MAP2 = unordered_map(); +int +process(Flow &f); +Flow f_glb; + +/////time//// +struct timeval begin_time; +struct timeval end_time; + +IP ip1("192.168.22.0/24"); + +void +_init_() { + (new F_Type())->init(); +} + +int +process(Flow &f) { + if (*((IP *)f.headers[Sip]) != ip1) { + return -1; + } + else if (*((IP *)f.headers[Sip]) <= ip1 && *((int *)f.headers[Tcp])) { + } + f.clean(); + return 0; +} +int +stateless_firewall(u_char *pkt, int totallength) { + f_glb = Flow(pkt, totallength); + return process(f_glb); +} + +void +stop() { + gettimeofday(&end_time, NULL); + + double total = end_time.tv_sec - begin_time.tv_sec + (end_time.tv_usec - begin_time.tv_usec) / 1000000.0; + + printf("\n\n**************************************************\n"); + printf("%ld packets are processed, %ld packets are dropped\n", _counter, _drop); + printf("NF runs for %f seconds\n", total); + printf("**************************************************\n\n"); +} + +/**********************************************************************/ + +/* + * Print a usage message + */ +static void +usage(const char *progname) { + printf("Usage: %s [EAL args] -- [NF_LIB args] -- -d -p \n\n", progname); +} + +/* + * Parse the application arguments. + */ +static int +parse_app_args(int argc, char *argv[], const char *progname) { + int c, dst_flag = 0; + + while ((c = getopt(argc, argv, "d:p:")) != -1) { + switch (c) { + case 'd': + destination = strtoul(optarg, NULL, 10); + dst_flag = 1; + break; + case 'p': + print_delay = strtoul(optarg, NULL, 10); + break; + case '?': + usage(progname); + if (optopt == 'd') + RTE_LOG(INFO, APP, "Option -%c requires an argument.\n", optopt); + else if (optopt == 'p') + RTE_LOG(INFO, APP, "Option -%c requires an argument.\n", optopt); + else if (isprint(optopt)) + RTE_LOG(INFO, APP, "Unknown option `-%c'.\n", optopt); + else + RTE_LOG(INFO, APP, "Unknown option character `\\x%x'.\n", optopt); + return -1; + default: + usage(progname); + return -1; + } + } + + if (!dst_flag) { + RTE_LOG(INFO, APP, "Stateless Firewall NF requires destination flag -d.\n"); + return -1; + } + + return optind; +} + +/* + * This function displays stats. It uses ANSI terminal codes to clear + * screen when called. It is called from a single non-master + * thread in the server process, when the process is run with more + * than one lcore enabled. + */ +static void +do_stats_display(struct rte_mbuf *pkt) { + const char clr[] = {27, '[', '2', 'J', '\0'}; + const char topLeft[] = {27, '[', '1', ';', '1', 'H', '\0'}; + static uint64_t pkt_process = 0; + struct ipv4_hdr *ip; + + pkt_process += print_delay; + + /* Clear screen and move to top left */ + printf("%s%s", clr, topLeft); + + printf("PACKETS\n"); + printf("-----\n"); + printf("Port : %d\n", pkt->port); + printf("Size : %d\n", pkt->pkt_len); + printf("N° : %" PRIu64 "\n", pkt_process); + printf("\n\n"); + + ip = onvm_pkt_ipv4_hdr(pkt); + if (ip != NULL) { + onvm_pkt_print(pkt); + } else { + printf("No IP4 header found\n"); + } +} + +static int +packet_handler(struct rte_mbuf *buf, struct onvm_pkt_meta *meta, + __attribute__((unused)) struct onvm_nf_local_ctx *nf_local_ctx) { + static uint32_t counter = 0; + if (++counter == print_delay) { + do_stats_display(buf); + counter = 0; + } + ////////add NFD NF process here!//////////////// + _counter++; + u_char * pkt = rte_pktmbuf_mtod(buf, u_char*); + int length = (int)buf->pkt_len; + int ok; + + ok = stateless_firewall(pkt, length); + if (ok == -1) { + _drop++; + meta->action = ONVM_NF_ACTION_DROP; + } else { + meta->action = ONVM_NF_ACTION_TONF; + meta->destination = destination; + } + //////////////////////////////////////////////// + return 0; +} + +int +main(int argc, char *argv[]) { + struct onvm_nf_local_ctx *nf_local_ctx; + struct onvm_nf_function_table *nf_function_table; + int arg_offset; + + const char *progname = argv[0]; + + nf_local_ctx = onvm_nflib_init_nf_local_ctx(); + onvm_nflib_start_signal_handler(nf_local_ctx, NULL); + + nf_function_table = onvm_nflib_init_nf_function_table(); + nf_function_table->pkt_handler = &packet_handler; + + if ((arg_offset = onvm_nflib_init(argc, argv, NF_TAG, nf_local_ctx, nf_function_table)) < 0) { + onvm_nflib_stop(nf_local_ctx); + if (arg_offset == ONVM_SIGNAL_TERMINATION) { + printf("Exiting due to user termination\n"); + return 0; + } else { + rte_exit(EXIT_FAILURE, "Failed ONVM init\n"); + } + } + + argc -= arg_offset; + argv += arg_offset; + + // NFD begin + _init_(); + // NFD end + + if (parse_app_args(argc, argv, progname) < 0) { + onvm_nflib_stop(nf_local_ctx); + rte_exit(EXIT_FAILURE, "Invalid command-line arguments\n"); + } + + // NFD begin + gettimeofday(&begin_time, NULL); + // NFD end + + onvm_nflib_run(nf_local_ctx); + + onvm_nflib_stop(nf_local_ctx); + stop(); + printf("If we reach here, program is ending\n"); + + return 0; +} diff --git a/examples/NFD/super_spreader_detection/Makefile b/examples/NFD/super_spreader_detection/Makefile new file mode 100644 index 000000000..a5ea8b2a5 --- /dev/null +++ b/examples/NFD/super_spreader_detection/Makefile @@ -0,0 +1,75 @@ +# openNetVM +# https://github.com/sdnfv/openNetVM +# +# BSD LICENSE +# +# Copyright(c) +# 2015-2017 George Washington University +# 2015-2017 University of California Riverside +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# The name of the author may not be used to endorse or promote +# products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +RTE_TARGET ?= x86_64-native-linuxapp-gcc + +# Default target, can be overriden by command line or environment +include $(RTE_SDK)/mk/rte.vars.mk + +# binary name +APP = super_spreader_detection + +# all source are stored in SRCS-y +SRCS-y := SSD.c + +# OpenNetVM, DPDK and NFD path +ONVM= $(SRCDIR)/../../../onvm +DPDK= $(SRCDIR)/../../../dpdk +NFD= $(SRCDIR)/.. + +CC=g++ + +CPPFLAGS = -O3 $(USER_FLAGS) -lpcap -std=c++11 -march=native +CPPFLAGS += -I$(ONVM)/onvm_nflib +CPPFLAGS += -I$(ONVM)/lib +CPPFLAGS += -I$(DPDK)/build/include +CPPFLAGS += -I$(NFD)/include/ + +LDFLAGS += $(ONVM)/onvm_nflib/$(RTE_TARGET)/libonvm.a +LDFLAGS += $(ONVM)/lib/$(RTE_TARGET)/lib/libonvmhelper.a +LDFLAGS += $(NFD)/lib/libNFD.a -lm -lstdc++ -lpcap + +# workaround for a gcc bug with noreturn attribute +# http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603 +ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +CFLAGS_main.o += -Wno-return-type +endif + +include $(RTE_SDK)/mk/rte.extapp.mk \ No newline at end of file diff --git a/examples/NFD/super_spreader_detection/README.md b/examples/NFD/super_spreader_detection/README.md new file mode 100644 index 000000000..a79c1844e --- /dev/null +++ b/examples/NFD/super_spreader_detection/README.md @@ -0,0 +1,72 @@ +Super Spreader Detection +== +Super Spreader Detection developed by the NFD framework + + +NFD is a NF developing framework, consisting two parts, NFD language and NFD compiler. NFD compiler translate
+the NF model file wiiten in NFD language, a table-form language, into standard runtime environment(such as C++).

+ + +Super Spreader Detection is translated from the `SSDmodel.txt` to C++ environment.
+ +
+ Super Spreader Detection detects and identifies super spreaders to preempt port scan attacks or DDoS attacks by increasing the counter on SYNs and decrease it on FINs. The count will be compared to threshold set by user. In this program, we set threshold 100. +
+ + +Testing +-- + +The Super Spreader Detection NF will drop the flow packets(denoted by source IP) whose accumulated SYN num (SYN packets from a source packet will be accumulated the number while the FIN packets from same source will decrease the number) reaches to threshold. To verify the dropping process, you should filter all SYN packets out without its corresponding FIN packets. Then you can trigger the dropping process. Run these 2 NFs: + +Run the Super Spreader Detection NF with: + +``` +./go.sh 1 -d 2 + +``` + +Run Speed Tester NF(to replay pcap file) with: + +``` +./go.sh 2 -d 1 -o pcap/SYNWithoutFIN.pcap + +``` + + + +Compilation and Execution +-- + +To run this NF, you should use either clang++ or g++ to compile this NF developed by C++. + +``` +cd super_spreader_detection +make + +``` + +Then, you can run this code. + +``` +./go.sh CORELIST SERVICE_ID DST [PRINT_DELAY] + +OR + +./go.sh -F CONFIG_FILE -- -- -d DST [-p PRINT_DELAY] + +OR + +sudo ./build/super_spreader_detection -l CORELIST -n 3 --proc-type=secondary -- -r SERVICE_ID -- -d DST [-p PRINT_DELAY] +``` + +App Specific Arguments +-- + - `-d `: destination service ID to foward to + - `-p `: number of packets between each print, e.g. `-p 1` prints every packets. + +Config File Support +-- +This NF supports the NF generating arguments from a config file. For additional reading, see [Examples.md](../../docs/Examples.md) + +See `../example_config.json` for all possible options that can be set. diff --git a/examples/NFD/super_spreader_detection/SSD.cpp b/examples/NFD/super_spreader_detection/SSD.cpp new file mode 100644 index 000000000..359c94a1e --- /dev/null +++ b/examples/NFD/super_spreader_detection/SSD.cpp @@ -0,0 +1,367 @@ +/********************************************************************* + * openNetVM + * https://sdnfv.github.io + * + * BSD LICENSE + * + * Copyright(c) + * 2015-2017 George Washington University + * 2015-2017 University of California Riverside + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * The name of the author may not be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ********************************************************************/ + +/************************************************************************************ +* Filename: SSD.cpp +* Author: Hongyi Huang(hhy17 AT mails.tsinghua.edu.cn), Bangwen Deng, Wenfei Wu +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: This code is the implementation of SuperSpreaderDetection from NFD project, + a C++ based NF developing framework designed by Wenfei's group + from IIIS, Tsinghua University, China. +*************************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#include "onvm_nflib.h" +#include "onvm_pkt_helper.h" + +#ifdef __cplusplus +} +#endif + +// NFD ADD +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "basic_classes.h" +#include "decode.h" + +using namespace std; + +#define NF_TAG "SuperSpreaderDetection" + +/* number of package between each print */ +static uint32_t print_delay = 1000000; + +static uint32_t destination; + +/*******************************NFD features********************************/ +Flow::Flow(u_char *packet, int totallength) { + /*Decoding*/ + this->pkt = packet; + struct ether_header *eth_header; + eth_header = (struct ether_header *)packet; + + /* Pointers to start point of various headers */ + u_char *ip_header; + /* Header lengths in bytes */ + int ethernet_header_length = 14; /* Doesn't change */ + int ip_header_length; + ip_header = packet + ethernet_header_length; + ip_header_length = ((*ip_header) & 0x0F); + ip_header_length = ip_header_length * 4; + int src_addr, des_addr; + src_addr = ntohl(*((int *)(ip_header + 12))); + des_addr = ntohl(*((int *)(ip_header + 16))); + this->field_value["sip"] = new IP(src_addr, 32); + this->field_value["dip"] = new IP(des_addr, 32); + this->field_value["tag"] = new int(0); + this->field_value["iplen"] = new int(totallength); + uint8_t protocol = (*(uint8_t *)(ip_header + 9)); + this->field_value["UDP"] = new int((protocol == IPPROTO_UDP) ? 1 : 0); + + /*TCP layer*/ + TCPHdr *tcph = (TCPHdr *)(ip_header + ip_header_length); + this->field_value["sport"] = new int(ntohs(tcph->th_sport)); + this->field_value["dport"] = new int(ntohs(tcph->th_dport)); + /*URG ACK PSH RST SYN FIN*/ + this->field_value["flag_fin"] = new int(tcph->th_flags & 0x1); + this->field_value["flag_syn"] = new int((tcph->th_flags >> 1) & 0x1); + this->field_value["flag_ack"] = new int((tcph->th_flags >> 4) & 0x1); +} +void +Flow::clean() { +} + +int +process(Flow &f); +Flow f_glb; +long int _counter = 0; +unordered_map F_Type::MAP = unordered_map(); +unordered_map F_Type::MAP2 = unordered_map(); + +struct timeval begin_time; +struct timeval end_time; + +void +_init_() { + (new F_Type())->init(); +} +int _t1 = 100; +int _t2 = 1; +int _t3 = 1; +int _t4 = 1; +int _t5 = 1; +int _t6 = 1; +int _t7 = 1; +int _t8 = 1; +int _t9 = 1; +int _t10 = 1; +int _t11 = 1; +State> list(*(new unordered_map())); +State> tlist(*(new unordered_map())); +State threshold(_t1); + +int +process(Flow &f) { + if (((*(int *)f["flag_syn"]) == 1) && + tlist[f][(*(IP *)f["sip"])] == 1){ + return -1; + } else if (((*(int *)f["flag_syn"]) == _t2) && + (tlist[f][(*(IP *)f["sip"])] != _t3 && list[f][(*(IP *)f["sip"])] != threshold[f])) { + list[f][(*(IP *)f["sip"])] = list[f][(*(IP *)f["sip"])] + _t4; + } else if (((*(int *)f["flag_syn"]) == _t5) && + (tlist[f][(*(IP *)f["sip"])] != _t6 && list[f][(*(IP *)f["sip"])] == threshold[f])) { + tlist[f][(*(IP *)f["sip"])] = _t7; + } else if ((*(int *)f["flag_fin"]) == _t8 && + tlist[f][(*(IP *)f["sip"])] == 1){ + list[f][(*(IP *)f["sip"])] = list[f][(*(IP *)f["sip"])] - 1; + tlist[f][(*(IP *)f["sip"])] = 0; + } else if ((*(int *)f["flag_fin"]) == _t8) { + list[f][(*(IP *)f["sip"])] = list[f][(*(IP *)f["sip"])] - _t9; + } else if ((*(int *)f["flag_syn"]) != _t10 && (*(int *)f["flag_fin"]) == _t11) { + } + f.clean(); + return 0; +} + +int +SSD(u_char *pkt, int totallength) { + f_glb = Flow(pkt, totallength); + return process(f_glb); +} + +void +stop() { + gettimeofday(&end_time, NULL); + + double total = end_time.tv_sec - begin_time.tv_sec + (end_time.tv_usec - begin_time.tv_usec) / 1000000.0; + + printf("\n\n**************************************************\n"); + printf("%ld packets are processed\n", _counter); + printf("NF runs for %f seconds\n", total); + printf("**************************************************\n\n"); +} + +/**********************************************************************/ + +/* + * Print a usage message + */ +static void +usage(const char *progname) { + printf("Usage: %s [EAL args] -- [NF_LIB args] -- -d -p \n\n", progname); +} + +/* + * Parse the application arguments. + */ +static int +parse_app_args(int argc, char *argv[], const char *progname) { + int c, dst_flag = 0; + + while ((c = getopt(argc, argv, "d:p:")) != -1) { + switch (c) { + case 'd': + destination = strtoul(optarg, NULL, 10); + dst_flag = 1; + break; + case 'p': + print_delay = strtoul(optarg, NULL, 10); + break; + case '?': + usage(progname); + if (optopt == 'd') + RTE_LOG(INFO, APP, "Option -%c requires an argument.\n", optopt); + else if (optopt == 'p') + RTE_LOG(INFO, APP, "Option -%c requires an argument.\n", optopt); + else if (isprint(optopt)) + RTE_LOG(INFO, APP, "Unknown option `-%c'.\n", optopt); + else + RTE_LOG(INFO, APP, "Unknown option character `\\x%x'.\n", optopt); + return -1; + default: + usage(progname); + return -1; + } + } + + if (!dst_flag) { + RTE_LOG(INFO, APP, "Super Spreader Detection NF requires destination flag -d.\n"); + return -1; + } + + return optind; +} + +/* + * This function displays stats. It uses ANSI terminal codes to clear + * screen when called. It is called from a single non-master + * thread in the server process, when the process is run with more + * than one lcore enabled. + */ +static void +do_stats_display(struct rte_mbuf *pkt) { + const char clr[] = {27, '[', '2', 'J', '\0'}; + const char topLeft[] = {27, '[', '1', ';', '1', 'H', '\0'}; + static uint64_t pkt_process = 0; + struct ipv4_hdr *ip; + + pkt_process += print_delay; + + /* Clear screen and move to top left */ + printf("%s%s", clr, topLeft); + + printf("PACKETS\n"); + printf("-----\n"); + printf("Port : %d\n", pkt->port); + printf("Size : %d\n", pkt->pkt_len); + printf("N° : %" PRIu64 "\n", pkt_process); + printf("\n\n"); + + ip = onvm_pkt_ipv4_hdr(pkt); + if (ip != NULL) { + onvm_pkt_print(pkt); + } else { + printf("No IP4 header found\n"); + } +} + +static int +packet_handler(struct rte_mbuf *buf, struct onvm_pkt_meta *meta, + __attribute__((unused)) struct onvm_nf_local_ctx *nf_local_ctx) { + static uint32_t counter = 0; + if (++counter == print_delay) { + do_stats_display(buf); + counter = 0; + } + ////////add NFD NF process here!//////////////// + _counter++; + u_char * pkt = rte_pktmbuf_mtod(buf, u_char*); + int length = (int)buf->pkt_len; + int ok; + + ok = SSD(pkt, length); + + if (ok == -1) { + meta->action = ONVM_NF_ACTION_DROP; + } else { + meta->action = ONVM_NF_ACTION_TONF; + meta->destination = destination; + } + + meta->action = ONVM_NF_ACTION_TONF; + meta->destination = destination; + //////////////////////////////////////////////// + return 0; +} + +int +main(int argc, char *argv[]) { + struct onvm_nf_local_ctx *nf_local_ctx; + struct onvm_nf_function_table *nf_function_table; + int arg_offset; + + const char *progname = argv[0]; + + nf_local_ctx = onvm_nflib_init_nf_local_ctx(); + onvm_nflib_start_signal_handler(nf_local_ctx, NULL); + + nf_function_table = onvm_nflib_init_nf_function_table(); + nf_function_table->pkt_handler = &packet_handler; + + if ((arg_offset = onvm_nflib_init(argc, argv, NF_TAG, nf_local_ctx, nf_function_table)) < 0) { + onvm_nflib_stop(nf_local_ctx); + if (arg_offset == ONVM_SIGNAL_TERMINATION) { + printf("Exiting due to user termination\n"); + return 0; + } else { + rte_exit(EXIT_FAILURE, "Failed ONVM init\n"); + } + } + + argc -= arg_offset; + argv += arg_offset; + + // NFD begin + _init_(); + // NFD end + + if (parse_app_args(argc, argv, progname) < 0) { + onvm_nflib_stop(nf_local_ctx); + rte_exit(EXIT_FAILURE, "Invalid command-line arguments\n"); + } + + // NFD begin + gettimeofday(&begin_time, NULL); + // NFD end + + onvm_nflib_run(nf_local_ctx); + + onvm_nflib_stop(nf_local_ctx); + stop(); + printf("If we reach here, program is ending\n"); + + return 0; +} diff --git a/examples/NFD/super_spreader_detection/SSDmodel.txt b/examples/NFD/super_spreader_detection/SSDmodel.txt new file mode 100755 index 000000000..d4304a8c5 --- /dev/null +++ b/examples/NFD/super_spreader_detection/SSDmodel.txt @@ -0,0 +1,53 @@ +/********************************************************************************** + NFD project + A C++ based NF developing framework designed by Wenfei's group + from IIIS, Tsinghua University, China. +******************************************************************************/ + +/************************************************************************************ +* Filename: SSDmodel.txt +* Author: Hongyi Huang(hhy17 AT mails.tsinghua.edu.cn), Bangwen Deng, Wenfei Wu +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: This file is a model file for NFD project, used to be compiled into NFD-NF +*************************************************************************************/ + + +program SSD{ + map list; + map tlist; + int threshold=100; + + entry{ + match_flow{f[flag_syn]==1} + match_state{tlist[f[sip]]==1} + action_flow{f[dip]==DROP;} + } + entry{ + match_flow{f[flag_syn]==1 } + match_state{tlist[f[sip]]!=1 && list[f[sip]]!= threshold} + action_state{list[f[sip]]=list[f[sip]]+1;} + } + entry{ + match_flow{f[flag_syn]==1} + match_state{tlist[f[sip]]!=1 && list[f[sip]]== threshold} + action_state{tlist[f[sip]]=1;} + action_flow{f[dip]=DROP;} + } + entry{ + match_flow{f[flag_fin]==1} + match_state{tlist[f[sip]]==1} + action_state{list[f[sip]]=list[f[sip]]-1;} + action_state{tlist[f[sip]]=0;} + } + entry{ + match_flow{f[flag_fin]==1} + action_state{list[f[sip]]=list[f[sip]]-1;} + } + entry{ + match_flow{f[flag_syn]!=1 && f[flag_fin]==1} + action_flow{pass;} + action_state{pass;} + } + +} \ No newline at end of file diff --git a/examples/NFD/super_spreader_detection/go.sh b/examples/NFD/super_spreader_detection/go.sh new file mode 120000 index 000000000..40babd806 --- /dev/null +++ b/examples/NFD/super_spreader_detection/go.sh @@ -0,0 +1 @@ +../go.sh \ No newline at end of file diff --git a/examples/NFD/syn_flood_detection/Makefile b/examples/NFD/syn_flood_detection/Makefile new file mode 100644 index 000000000..fa7ecd2b3 --- /dev/null +++ b/examples/NFD/syn_flood_detection/Makefile @@ -0,0 +1,75 @@ +# openNetVM +# https://github.com/sdnfv/openNetVM +# +# BSD LICENSE +# +# Copyright(c) +# 2015-2017 George Washington University +# 2015-2017 University of California Riverside +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# The name of the author may not be used to endorse or promote +# products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +RTE_TARGET ?= x86_64-native-linuxapp-gcc + +# Default target, can be overriden by command line or environment +include $(RTE_SDK)/mk/rte.vars.mk + +# binary name +APP = syn_flood_detection + +# all source are stored in SRCS-y +SRCS-y := SYNFloodDetection.c + +# OpenNetVM, DPDK and NFD path +ONVM= $(SRCDIR)/../../../onvm +DPDK= $(SRCDIR)/../../../dpdk +NFD= $(SRCDIR)/.. + +CC=g++ + +CPPFLAGS = -O3 $(USER_FLAGS) -lpcap -std=c++11 -march=native +CPPFLAGS += -I$(ONVM)/onvm_nflib +CPPFLAGS += -I$(ONVM)/lib +CPPFLAGS += -I$(DPDK)/build/include +CPPFLAGS += -I$(NFD)/include/ + +LDFLAGS += $(ONVM)/onvm_nflib/$(RTE_TARGET)/libonvm.a +LDFLAGS += $(ONVM)/lib/$(RTE_TARGET)/lib/libonvmhelper.a +LDFLAGS += $(NFD)/lib/libNFD.a -lm -lstdc++ -lpcap + +# workaround for a gcc bug with noreturn attribute +# http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603 +ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +CFLAGS_main.o += -Wno-return-type +endif + +include $(RTE_SDK)/mk/rte.extapp.mk \ No newline at end of file diff --git a/examples/NFD/syn_flood_detection/README.md b/examples/NFD/syn_flood_detection/README.md new file mode 100644 index 000000000..86cf7a894 --- /dev/null +++ b/examples/NFD/syn_flood_detection/README.md @@ -0,0 +1,75 @@ +SYNFloodDetection +== +SYN Flood Detection developed by the NFD framework + + +NFD is a NF developing framework, consisting two parts, NFD language and NFD compiler. NFD compiler translate
+the NF model file wiiten in NFD language, a table-form language, into standard runtime environment(such as C++).

+ + +SYN Flood Detection is translated from the `SYNFloodDetectionModel.txt` to C++ environment.
+ +
+ SYN Flood Detection counts the number of SYNs without any matching ACK from the sender side. If one sender exceeds a certain threshold X (set by user. In this program, we set threshold 100.), it should be blocked. + +
+ + + +Testing +-- + +The SYN Flood Detection NF counts the number of SYNs without any matching ACK from the sender side. If one sender(denoted by IP) exceeds a certain threshold X (set by user. In this program, we set threshold 100.), it should be blocked. To trigger the dropping process, you need to just send the SYN packets without ACK (from sender to server) and accumulate it to the threshold. You can filter from the `pktgen_large.pcap` by `tcp.flags.syn==1 and tcp.flags.ack!=1` to get the test trace. Run these 2 NFs: + +Run the SYN Flood Detection NF with: + +``` +./go.sh 1 -d 2 + +``` + +Run Speed Tester NF(to replay pcap file) with: + +``` +./go.sh 2 -d 1 -o pcap/SYNWithoutACK.pcap + +``` + + + + +Compilation and Execution +-- + +To run this NF, you should use either clang++ or g++ to compile this NF developed by C++. + +``` +cd syn_flood_detection +make + +``` + +Then, you can run this code. + +``` +./go.sh CORELIST SERVICE_ID DST [PRINT_DELAY] + +OR + +./go.sh -F CONFIG_FILE -- -- -d DST [-p PRINT_DELAY] + +OR + +sudo ./build/syn_flood_detection -l CORELIST -n 3 --proc-type=secondary -- -r SERVICE_ID -- -d DST [-p PRINT_DELAY] +``` + +App Specific Arguments +-- + - `-d `: destination service ID to foward to + - `-p `: number of packets between each print, e.g. `-p 1` prints every packets. + +Config File Support +-- +This NF supports the NF generating arguments from a config file. For additional reading, see [Examples.md](../../docs/Examples.md) + +See `../example_config.json` for all possible options that can be set. diff --git a/examples/NFD/syn_flood_detection/SYNFloodDetection.cpp b/examples/NFD/syn_flood_detection/SYNFloodDetection.cpp new file mode 100644 index 000000000..15fc1873a --- /dev/null +++ b/examples/NFD/syn_flood_detection/SYNFloodDetection.cpp @@ -0,0 +1,364 @@ +/********************************************************************* + * openNetVM + * https://sdnfv.github.io + * + * BSD LICENSE + * + * Copyright(c) + * 2015-2017 George Washington University + * 2015-2017 University of California Riverside + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * The name of the author may not be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ********************************************************************/ + +/************************************************************************************ +* Filename: SYNFloodDetection.cpp +* Author: Hongyi Huang(hhy17 AT mails.tsinghua.edu.cn), Bangwen Deng, Wenfei Wu +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: This code is the implementation of SYNFloodDetection from NFD project, + a C++ based NF developing framework designed by Wenfei's group + from IIIS, Tsinghua University, China. +*************************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#include "onvm_nflib.h" +#include "onvm_pkt_helper.h" + +#ifdef __cplusplus +} +#endif + +// NFD ADD +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "basic_classes.h" +#include "decode.h" + +using namespace std; + +#define NF_TAG "SYNFloodDetection" + +/* number of package between each print */ +static uint32_t print_delay = 1000000; + +static uint32_t destination; + +/*******************************NFD features********************************/ +Flow::Flow(u_char *packet, int totallength) { + /*Decoding*/ + this->pkt = packet; + struct ether_header *eth_header; + eth_header = (struct ether_header *)packet; + + /* Pointers to start point of various headers */ + u_char *ip_header; + /* Header lengths in bytes */ + int ethernet_header_length = 14; /* Doesn't change */ + int ip_header_length; + ip_header = packet + ethernet_header_length; + ip_header_length = ((*ip_header) & 0x0F); + ip_header_length = ip_header_length * 4; + int src_addr, des_addr; + src_addr = ntohl(*((int *)(ip_header + 12))); + des_addr = ntohl(*((int *)(ip_header + 16))); + this->field_value["sip"] = new IP(src_addr, 32); + this->field_value["dip"] = new IP(des_addr, 32); + this->field_value["tag"] = new int(0); + this->field_value["iplen"] = new int(totallength); + uint8_t protocol = (*(uint8_t *)(ip_header + 9)); + this->field_value["UDP"] = new int((protocol == IPPROTO_UDP) ? 1 : 0); + + /*TCP layer*/ + TCPHdr *tcph = (TCPHdr *)(ip_header + ip_header_length); + this->field_value["sport"] = new int(ntohs(tcph->th_sport)); + this->field_value["dport"] = new int(ntohs(tcph->th_dport)); + /*URG ACK PSH RST SYN FIN*/ + this->field_value["flag_fin"] = new int(tcph->th_flags & 0x1); + this->field_value["flag_syn"] = new int((tcph->th_flags >> 1) & 0x1); + this->field_value["flag_ack"] = new int((tcph->th_flags >> 4) & 0x1); +} +void +Flow::clean() { +} + +int +process(Flow &f); +Flow f_glb; +long int _counter = 0; +long int _drop = 0; +unordered_map F_Type::MAP = unordered_map(); +unordered_map F_Type::MAP2 = unordered_map(); + +struct timeval begin_time; +struct timeval end_time; + +void +_init_() { + (new F_Type())->init(); +} +int _t1 = 100; +int _t2 = 1; +int _t3 = 1; +int _t4 = 1; +int _t5 = 1; +int _t6 = 1; +int _t7 = 1; +int _t8 = 1; +int _t9 = 1; +int _t10 = 1; +int _t11 = 1; +int _t12 = 1; +int _t13 = 1; +int _t14 = 1; +State> blist(*(new unordered_map())); +State threshold(_t1); + +int +process(Flow &f) { + if ((*(int *)f["flag_syn"]) == _t2 && (*(int *)f["tag"]) != _t3) { + blist[f][(*(IP *)f["sip"])] = blist[f][(*(IP *)f["sip"])] + _t4; + (*(int *)f["tag"]) = _t5; + return process(f); + } else if (((*(int *)f["tag"]) == _t6) && (blist[f][(*(IP *)f["sip"])] >= threshold[f])) { + return -1; + } else if (((*(int *)f["tag"]) == _t7) && (blist[f][(*(IP *)f["sip"])] != threshold[f])) { + } else if ((*(int *)f["tag"]) != _t8 && (*(int *)f["flag_syn"]) != _t9 && (*(int *)f["flag_ack"]) == _t10) { + blist[f][(*(IP *)f["sip"])] = blist[f][(*(IP *)f["sip"])] - _t11; + } else if ((*(int *)f["tag"]) != _t12 && (*(int *)f["flag_syn"]) != _t13 && (*(int *)f["flag_ack"]) != _t14) { + } + + f.clean(); + return 0; +} + +int +SYNFD(u_char *pkt, int totallength) { + f_glb = Flow(pkt, totallength); + return process(f_glb); +} + +void +stop() { + gettimeofday(&end_time, NULL); + + double total = end_time.tv_sec - begin_time.tv_sec + (end_time.tv_usec - begin_time.tv_usec) / 1000000.0; + + printf("\n\n**************************************************\n"); + printf("%ld packets are processed\n", _counter); + printf("NF runs for %f seconds\n", total); + printf("**************************************************\n\n"); +} + +/**********************************************************************/ + +/* + * Print a usage message + */ +static void +usage(const char *progname) { + printf("Usage: %s [EAL args] -- [NF_LIB args] -- -d -p \n\n", progname); +} + +/* + * Parse the application arguments. + */ +static int +parse_app_args(int argc, char *argv[], const char *progname) { + int c, dst_flag = 0; + + while ((c = getopt(argc, argv, "d:p:")) != -1) { + switch (c) { + case 'd': + destination = strtoul(optarg, NULL, 10); + dst_flag = 1; + break; + case 'p': + print_delay = strtoul(optarg, NULL, 10); + break; + case '?': + usage(progname); + if (optopt == 'd') + RTE_LOG(INFO, APP, "Option -%c requires an argument.\n", optopt); + else if (optopt == 'p') + RTE_LOG(INFO, APP, "Option -%c requires an argument.\n", optopt); + else if (isprint(optopt)) + RTE_LOG(INFO, APP, "Unknown option `-%c'.\n", optopt); + else + RTE_LOG(INFO, APP, "Unknown option character `\\x%x'.\n", optopt); + return -1; + default: + usage(progname); + return -1; + } + } + + if (!dst_flag) { + RTE_LOG(INFO, APP, "SYN Flood Detection NF requires destination flag -d.\n"); + return -1; + } + + return optind; +} + +/* + * This function displays stats. It uses ANSI terminal codes to clear + * screen when called. It is called from a single non-master + * thread in the server process, when the process is run with more + * than one lcore enabled. + */ +static void +do_stats_display(struct rte_mbuf *pkt) { + const char clr[] = {27, '[', '2', 'J', '\0'}; + const char topLeft[] = {27, '[', '1', ';', '1', 'H', '\0'}; + static uint64_t pkt_process = 0; + struct ipv4_hdr *ip; + + pkt_process += print_delay; + + /* Clear screen and move to top left */ + printf("%s%s", clr, topLeft); + + printf("PACKETS\n"); + printf("-----\n"); + printf("Port : %d\n", pkt->port); + printf("Size : %d\n", pkt->pkt_len); + printf("N° : %" PRIu64 "\n", pkt_process); + printf("\n\n"); + + ip = onvm_pkt_ipv4_hdr(pkt); + if (ip != NULL) { + onvm_pkt_print(pkt); + } else { + printf("No IP4 header found\n"); + } +} + +static int +packet_handler(struct rte_mbuf *buf, struct onvm_pkt_meta *meta, + __attribute__((unused)) struct onvm_nf_local_ctx *nf_local_ctx) { + static uint32_t counter = 0; + if (++counter == print_delay) { + do_stats_display(buf); + counter = 0; + } + ////////add NFD NF process here!//////////////// + _counter++; + u_char * pkt = rte_pktmbuf_mtod(buf, u_char*); + int length = (int)buf->pkt_len; + int ok; + + ok = SYNFD(pkt, length); + + if (ok == -1) { + _drop++; + meta->action = ONVM_NF_ACTION_DROP; + } else { + meta->action = ONVM_NF_ACTION_TONF; + meta->destination = destination; + } + + //////////////////////////////////////////////// + return 0; +} + +int +main(int argc, char *argv[]) { + struct onvm_nf_local_ctx *nf_local_ctx; + struct onvm_nf_function_table *nf_function_table; + int arg_offset; + + const char *progname = argv[0]; + + nf_local_ctx = onvm_nflib_init_nf_local_ctx(); + onvm_nflib_start_signal_handler(nf_local_ctx, NULL); + + nf_function_table = onvm_nflib_init_nf_function_table(); + nf_function_table->pkt_handler = &packet_handler; + + if ((arg_offset = onvm_nflib_init(argc, argv, NF_TAG, nf_local_ctx, nf_function_table)) < 0) { + onvm_nflib_stop(nf_local_ctx); + if (arg_offset == ONVM_SIGNAL_TERMINATION) { + printf("Exiting due to user termination\n"); + return 0; + } else { + rte_exit(EXIT_FAILURE, "Failed ONVM init\n"); + } + } + + argc -= arg_offset; + argv += arg_offset; + + // NFD begin + _init_(); + // NFD end + + if (parse_app_args(argc, argv, progname) < 0) { + onvm_nflib_stop(nf_local_ctx); + rte_exit(EXIT_FAILURE, "Invalid command-line arguments\n"); + } + + // NFD begin + gettimeofday(&begin_time, NULL); + // NFD end + + onvm_nflib_run(nf_local_ctx); + + onvm_nflib_stop(nf_local_ctx); + stop(); + printf("If we reach here, program is ending\n"); + + return 0; +} diff --git a/examples/NFD/syn_flood_detection/SYNFloodDetectionModel.txt b/examples/NFD/syn_flood_detection/SYNFloodDetectionModel.txt new file mode 100755 index 000000000..2089ab249 --- /dev/null +++ b/examples/NFD/syn_flood_detection/SYNFloodDetectionModel.txt @@ -0,0 +1,45 @@ +/********************************************************************************** + NFD project + A C++ based NF developing framework designed by Wenfei's group + from IIIS, Tsinghua University, China. +******************************************************************************/ + +/************************************************************************************ +* Filename: SYNFloodDetectionModel.txt +* Author: Hongyi Huang(hhy17 AT mails.tsinghua.edu.cn), Bangwen Deng, Wenfei Wu +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: This file is a model file for NFD project, used to be compiled into NFD-NF +*************************************************************************************/ + + +program SYNFD{ + map blist; + int threshold=100; + + entry{ + match_flow{f[flag_syn] ==1 && f[tag]!=1} + action_state{blist[f[sip]]=blist[f[sip]]+1; f[tag]=1; resubmit;} + + } + entry{ + match_flow{f[tag]==1} + match_state{blist[f[sip]]== threshold} + action_flow{f[dip]=DROP;} + } + entry{ + match_flow{f[tag]==1} + match_state{blist[f[sip]]!= threshold} + action_flow{pass;} + action_state{pass;} + } + entry{ + match_flow{f[tag]!=1 && f[flag_syn] !=1 && f[flag_ack] ==1} + action_state{blist[f[sip]]=blist[f[sip]]-1;} + } + entry{ + match_flow{ f[tag]!=1 && f[flag_syn] !=1 && f[flag_ack] !=1} + action_flow{pass;} + action_state{pass;} + } +} diff --git a/examples/NFD/syn_flood_detection/go.sh b/examples/NFD/syn_flood_detection/go.sh new file mode 120000 index 000000000..40babd806 --- /dev/null +++ b/examples/NFD/syn_flood_detection/go.sh @@ -0,0 +1 @@ +../go.sh \ No newline at end of file diff --git a/examples/NFD/udp_flood_mitigation/Makefile b/examples/NFD/udp_flood_mitigation/Makefile new file mode 100644 index 000000000..d8aa68bce --- /dev/null +++ b/examples/NFD/udp_flood_mitigation/Makefile @@ -0,0 +1,75 @@ +# openNetVM +# https://github.com/sdnfv/openNetVM +# +# BSD LICENSE +# +# Copyright(c) +# 2015-2017 George Washington University +# 2015-2017 University of California Riverside +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# The name of the author may not be used to endorse or promote +# products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +RTE_TARGET ?= x86_64-native-linuxapp-gcc + +# Default target, can be overriden by command line or environment +include $(RTE_SDK)/mk/rte.vars.mk + +# binary name +APP = udp_flood_mitigation + +# all source are stored in SRCS-y +SRCS-y := UDPFloodMitagation.c + +# OpenNetVM, DPDK and NFD path +ONVM= $(SRCDIR)/../../../onvm +DPDK= $(SRCDIR)/../../../dpdk +NFD= $(SRCDIR)/.. + +CC=g++ + +CPPFLAGS = -O3 $(USER_FLAGS) -lpcap -std=c++11 -march=native +CPPFLAGS += -I$(ONVM)/onvm_nflib +CPPFLAGS += -I$(ONVM)/lib +CPPFLAGS += -I$(DPDK)/build/include +CPPFLAGS += -I$(NFD)/include/ + +LDFLAGS += $(ONVM)/onvm_nflib/$(RTE_TARGET)/libonvm.a +LDFLAGS += $(ONVM)/lib/$(RTE_TARGET)/lib/libonvmhelper.a +LDFLAGS += $(NFD)/lib/libNFD.a -lm -lstdc++ -lpcap + +# workaround for a gcc bug with noreturn attribute +# http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603 +ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +CFLAGS_main.o += -Wno-return-type +endif + +include $(RTE_SDK)/mk/rte.extapp.mk \ No newline at end of file diff --git a/examples/NFD/udp_flood_mitigation/README.md b/examples/NFD/udp_flood_mitigation/README.md new file mode 100644 index 000000000..f523ff7dc --- /dev/null +++ b/examples/NFD/udp_flood_mitigation/README.md @@ -0,0 +1,71 @@ +UDPFloodMitigation +== +UDP Flood Mitigation developed by the NFD framework + + +NFD is a NF developing framework, consisting two parts, NFD language and NFD compiler. NFD compiler translate
+the NF model file wiiten in NFD language, a table-form language, into standard runtime environment(such as C++).

+ +UDP Flood Mitigation is translated from the `UDPFloodMitigationModel.txt` to C++ environment. +
+ +UDP Flood Mitigation identifies source IPs that send an anomalously higher number of UDP packets and uses the statistics (compared to threshold set by user. In this program, we set threshold 100.) to categorize each packet as either attack or benign. +
+ + + +Testing +-- + +The UDP Flood Mitigation NF will record the count of UDP flows from same IPs. If the count reaches to threshold and the sequent UDP packet will be dropped. To trigger dropping process, you just need to send the UDP packets with the same IP over and over and over again. Run these 2 NFs: + +Run the UDP Flood Mitigation NF with: + +``` +./go.sh 1 -d 2 + +``` + +Run Speed Tester NF(to replay pcap file) with: + +``` +./go.sh 2 -d 1 -o pcap/UDPPackets.pcap + +``` + + +Compilation and Execution +-- + +To run this NF, you should use either clang++ or g++ to compile this NF developed by C++. + +``` +cd udp_flood_mitigation +make + +``` + +Then, you can run this code. + +``` +./go.sh CORELIST SERVICE_ID DST [PRINT_DELAY] + +OR + +./go.sh -F CONFIG_FILE -- -- -d DST [-p PRINT_DELAY] + +OR + +sudo ./build/udp_flood_mitigation -l CORELIST -n 3 --proc-type=secondary -- -r SERVICE_ID -- -d DST [-p PRINT_DELAY] +``` + +App Specific Arguments +-- + - `-d `: destination service ID to foward to + - `-p `: number of packets between each print, e.g. `-p 1` prints every packets. + +Config File Support +-- +This NF supports the NF generating arguments from a config file. For additional reading, see [Examples.md](../../docs/Examples.md) + +See `../example_config.json` for all possible options that can be set. diff --git a/examples/NFD/udp_flood_mitigation/UDPFloodMitagation.cpp b/examples/NFD/udp_flood_mitigation/UDPFloodMitagation.cpp new file mode 100644 index 000000000..a08188fc5 --- /dev/null +++ b/examples/NFD/udp_flood_mitigation/UDPFloodMitagation.cpp @@ -0,0 +1,358 @@ +/********************************************************************* + * openNetVM + * https://sdnfv.github.io + * + * BSD LICENSE + * + * Copyright(c) + * 2015-2017 George Washington University + * 2015-2017 University of California Riverside + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * The name of the author may not be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ********************************************************************/ + +/************************************************************************************ +* Filename: UDPFloodMitigation.cpp +* Author: Hongyi Huang(hhy17 AT mails.tsinghua.edu.cn), Bangwen Deng, Wenfei Wu +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: This code is the implementation of UDPFloodMitigation from NFD project, + a C++ based NF developing framework designed by Wenfei's group + from IIIS, Tsinghua University, China. +*************************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#include "onvm_nflib.h" +#include "onvm_pkt_helper.h" + +#ifdef __cplusplus +} +#endif + +// NFD ADD +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "basic_classes.h" +#include "decode.h" + +using namespace std; + +#define NF_TAG "UDPFloodMitigation" + +/* number of package between each print */ +static uint32_t print_delay = 1000000; + +static uint32_t destination; + +/*******************************NFD features********************************/ +Flow::Flow(u_char *packet, int totallength) { + /*Decoding*/ + this->pkt = packet; + struct ether_header *eth_header; + eth_header = (struct ether_header *)packet; + + /* Pointers to start point of various headers */ + u_char *ip_header; + /* Header lengths in bytes */ + int ethernet_header_length = 14; /* Doesn't change */ + int ip_header_length; + ip_header = packet + ethernet_header_length; + ip_header_length = ((*ip_header) & 0x0F); + ip_header_length = ip_header_length * 4; + int src_addr, des_addr; + src_addr = ntohl(*((int *)(ip_header + 12))); + des_addr = ntohl(*((int *)(ip_header + 16))); + this->field_value["sip"] = new IP(src_addr, 32); + this->field_value["dip"] = new IP(des_addr, 32); + this->field_value["tag"] = new int(0); + this->field_value["iplen"] = new int(totallength); + uint8_t protocol = (*(uint8_t *)(ip_header + 9)); + this->field_value["UDP"] = new int((protocol == IPPROTO_UDP) ? 1 : 0); + + /*TCP layer*/ + TCPHdr *tcph = (TCPHdr *)(ip_header + ip_header_length); + this->field_value["sport"] = new int(ntohs(tcph->th_sport)); + this->field_value["dport"] = new int(ntohs(tcph->th_dport)); + /*URG ACK PSH RST SYN FIN*/ + this->field_value["flag_fin"] = new int(tcph->th_flags & 0x1); + this->field_value["flag_syn"] = new int((tcph->th_flags >> 1) & 0x1); + this->field_value["flag_ack"] = new int((tcph->th_flags >> 4) & 0x1); +} +void +Flow::clean() { +} + +int +process(Flow &f); +Flow f_glb; +long int _counter = 0; +long int _drop = 0; +unordered_map F_Type::MAP = unordered_map(); +unordered_map F_Type::MAP2 = unordered_map(); + +struct timeval begin_time; +struct timeval end_time; + +void +_init_() { + (new F_Type())->init(); +} +int _t1 = 100; +int _t2 = 1; +int _t3 = 1; +int _t4 = 1; +int _t5 = 1; +int _t6 = 1; +int _t7 = 1; +int _t8 = 1; +State> udpcounter(*(new unordered_map())); +State> udpflood(*(new unordered_map())); +State threshold(_t1); + +int +process(Flow &f) { + if (((*(int *)f["UDP"]) == 1) && + udpflood[f][(*(IP *)f["sip"])] == 1){ + return -1; + } else if (((*(int *)f["UDP"]) == _t2) && + (udpflood[f][(*(IP *)f["sip"])] != _t3 && udpcounter[f][(*(IP *)f["sip"])] != threshold[f])) { + udpcounter[f][(*(IP *)f["sip"])] = udpcounter[f][(*(IP *)f["sip"])] + _t4; + } else if (((*(int *)f["UDP"]) == _t5) && + (udpflood[f][(*(IP *)f["sip"])] != _t6 && udpcounter[f][(*(IP *)f["sip"])] == threshold[f])) { + udpflood[f][(*(IP *)f["sip"])] = _t7; + return -1; + } else if ((*(int *)f["UDP"]) != _t8) { + } + + f.clean(); + return 0; +} +int +UDPFM(u_char *pkt, int totallength) { + f_glb = Flow(pkt, totallength); + return process(f_glb); +} + +void +stop() { + gettimeofday(&end_time, NULL); + + double total = end_time.tv_sec - begin_time.tv_sec + (end_time.tv_usec - begin_time.tv_usec) / 1000000.0; + + printf("\n\n**************************************************\n"); + printf("%ld packets are processed\n", _counter); + printf("NF runs for %f seconds\n", total); + printf("**************************************************\n\n"); +} + +/**********************************************************************/ + +/* + * Print a usage message + */ +static void +usage(const char *progname) { + printf("Usage: %s [EAL args] -- [NF_LIB args] -- -d -p \n\n", progname); +} + +/* + * Parse the application arguments. + */ +static int +parse_app_args(int argc, char *argv[], const char *progname) { + int c, dst_flag = 0; + + while ((c = getopt(argc, argv, "d:p:")) != -1) { + switch (c) { + case 'd': + destination = strtoul(optarg, NULL, 10); + dst_flag = 1; + break; + case 'p': + print_delay = strtoul(optarg, NULL, 10); + break; + case '?': + usage(progname); + if (optopt == 'd') + RTE_LOG(INFO, APP, "Option -%c requires an argument.\n", optopt); + else if (optopt == 'p') + RTE_LOG(INFO, APP, "Option -%c requires an argument.\n", optopt); + else if (isprint(optopt)) + RTE_LOG(INFO, APP, "Unknown option `-%c'.\n", optopt); + else + RTE_LOG(INFO, APP, "Unknown option character `\\x%x'.\n", optopt); + return -1; + default: + usage(progname); + return -1; + } + } + + if (!dst_flag) { + RTE_LOG(INFO, APP, "UDP Flood Mitigation NF requires destination flag -d.\n"); + return -1; + } + + return optind; +} + +/* + * This function displays stats. It uses ANSI terminal codes to clear + * screen when called. It is called from a single non-master + * thread in the server process, when the process is run with more + * than one lcore enabled. + */ +static void +do_stats_display(struct rte_mbuf *pkt) { + const char clr[] = {27, '[', '2', 'J', '\0'}; + const char topLeft[] = {27, '[', '1', ';', '1', 'H', '\0'}; + static uint64_t pkt_process = 0; + struct ipv4_hdr *ip; + + pkt_process += print_delay; + + /* Clear screen and move to top left */ + printf("%s%s", clr, topLeft); + + printf("PACKETS\n"); + printf("-----\n"); + printf("Port : %d\n", pkt->port); + printf("Size : %d\n", pkt->pkt_len); + printf("N° : %" PRIu64 "\n", pkt_process); + printf("\n\n"); + + ip = onvm_pkt_ipv4_hdr(pkt); + if (ip != NULL) { + onvm_pkt_print(pkt); + } else { + printf("No IP4 header found\n"); + } +} + +static int +packet_handler(struct rte_mbuf *buf, struct onvm_pkt_meta *meta, + __attribute__((unused)) struct onvm_nf_local_ctx *nf_local_ctx) { + static uint32_t counter = 0; + if (++counter == print_delay) { + do_stats_display(buf); + counter = 0; + } + ////////add NFD NF process here!//////////////// + _counter++; + u_char * pkt = rte_pktmbuf_mtod(buf, u_char*); + int length = (int)buf->pkt_len; + int ok; + + ok = UDPFM(pkt, length); + + if (ok == -1) { + _drop++; + meta->action = ONVM_NF_ACTION_DROP; + } else { + meta->action = ONVM_NF_ACTION_TONF; + meta->destination = destination; + } + //////////////////////////////////////////////// + return 0; +} + +int +main(int argc, char *argv[]) { + struct onvm_nf_local_ctx *nf_local_ctx; + struct onvm_nf_function_table *nf_function_table; + int arg_offset; + + const char *progname = argv[0]; + + nf_local_ctx = onvm_nflib_init_nf_local_ctx(); + onvm_nflib_start_signal_handler(nf_local_ctx, NULL); + + nf_function_table = onvm_nflib_init_nf_function_table(); + nf_function_table->pkt_handler = &packet_handler; + + if ((arg_offset = onvm_nflib_init(argc, argv, NF_TAG, nf_local_ctx, nf_function_table)) < 0) { + onvm_nflib_stop(nf_local_ctx); + if (arg_offset == ONVM_SIGNAL_TERMINATION) { + printf("Exiting due to user termination\n"); + return 0; + } else { + rte_exit(EXIT_FAILURE, "Failed ONVM init\n"); + } + } + + argc -= arg_offset; + argv += arg_offset; + + // NFD begin + _init_(); + // NFD end + + if (parse_app_args(argc, argv, progname) < 0) { + onvm_nflib_stop(nf_local_ctx); + rte_exit(EXIT_FAILURE, "Invalid command-line arguments\n"); + } + + // NFD begin + gettimeofday(&begin_time, NULL); + // NFD end + + onvm_nflib_run(nf_local_ctx); + + onvm_nflib_stop(nf_local_ctx); + stop(); + printf("If we reach here, program is ending\n"); + + return 0; +} diff --git a/examples/NFD/udp_flood_mitigation/UDPFloodMitagationModel.txt b/examples/NFD/udp_flood_mitigation/UDPFloodMitagationModel.txt new file mode 100755 index 000000000..378cf1b28 --- /dev/null +++ b/examples/NFD/udp_flood_mitigation/UDPFloodMitagationModel.txt @@ -0,0 +1,37 @@ +/********************************************************************************** + NFD project + A C++ based NF developing framework designed by Wenfei's group + from IIIS, Tsinghua University, China. +******************************************************************************/ + +/************************************************************************************ +* Filename: UDPFloodMitagation.txt +* Author: Hongyi Huang(hhy17 AT mails.tsinghua.edu.cn), Bangwen Deng, Wenfei Wu +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: This file is a model file for NFD project, used to be compiled into NFD-NF +*************************************************************************************/ + + +program UDPFM{ + map udpcounter; + map udpflood; + int threshold=100; + + entry{ + match_flow{f[UDP]==1} + match_state{udpflood[f[sip]]!=1 && udpcounter[f[sip]]!=threshold} + action_state{udpcounter[f[sip]]=udpcounter[f[sip]]+1;} + } + entry{ + match_flow{f[UDP]==1} + match_state{udpflood[f[sip]]!=1 && udpcounter[f[sip]]==threshold} + action_state{udpflood[f[sip]]=1;} + action_flow{f[dip]=DROP;} + } + entry{ + match_flow{f[UDP]!=1} + action_flow{pass;} + action_state{pass;} + } +} diff --git a/examples/NFD/udp_flood_mitigation/go.sh b/examples/NFD/udp_flood_mitigation/go.sh new file mode 120000 index 000000000..40babd806 --- /dev/null +++ b/examples/NFD/udp_flood_mitigation/go.sh @@ -0,0 +1 @@ +../go.sh \ No newline at end of file diff --git a/examples/start_nf.sh b/examples/start_nf.sh index 6e24d4607..0eb263f7d 100755 --- a/examples/start_nf.sh +++ b/examples/start_nf.sh @@ -24,6 +24,8 @@ SCRIPT=$(readlink -f "$0") SCRIPTPATH=$(dirname "$SCRIPT") NF_NAME=$1 NF_PATH=$SCRIPTPATH/$NF_NAME +# For NFD NF +NF_NAME=${NF_PATH##*/} BINARY=$NF_PATH/build/app/$NF_NAME DPDK_BASE_ARGS="-n 3 --proc-type=secondary" # For simple mode, only used for initial dpdk startup diff --git a/onvm/onvm_nflib/onvm_flow_table.h b/onvm/onvm_nflib/onvm_flow_table.h index b6d5a25ca..45c3b00e0 100644 --- a/onvm/onvm_nflib/onvm_flow_table.h +++ b/onvm/onvm_nflib/onvm_flow_table.h @@ -191,11 +191,13 @@ onvm_ft_fill_key_symmetric(struct onvm_ft_ipv4_5tuple *key, struct rte_mbuf *pkt /* Hash a flow key to get an int. From L3 fwd example */ static inline uint32_t onvm_ft_ipv4_hash_crc(const void *data, __rte_unused uint32_t data_len, uint32_t init_val) { - const union ipv4_5tuple_host *k; + union ipv4_5tuple_host *k; uint32_t t; const uint32_t *p; - k = data; + k = (union ipv4_5tuple_host*) malloc(sizeof(union ipv4_5tuple_host)); + memcpy(k, data, sizeof(union ipv4_5tuple_host)); + t = k->proto; p = (const uint32_t *)&k->port_src; From 3b4fbf71aa2e0f7872e32936a026f6e3e035fd69 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Wed, 31 Jul 2019 19:38:45 -0400 Subject: [PATCH 08/12] [Bug Fix] Update Style Guide Links (#152) Some of the links in the style guide where broken, this pr fixes them. Commit log: * Fix google style guide links * Fixed extra tilda character --- style/styleguide.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/style/styleguide.md b/style/styleguide.md index f9631474b..b9c96ece5 100644 --- a/style/styleguide.md +++ b/style/styleguide.md @@ -14,7 +14,7 @@ Your project is likely to include multiple files, so you need to think about how **File Names** should be short but specific to the module residing within them. Reading a list of file names should give someone a quick idea of the main components in a project. -**A Header (.h) File** should generally be included for each .c file. The header should be self-contained and include a `#ifdef` guard to prevent compiler errors. The `ifdef` should use the form `_FILENAME_H`. [More Details.](http://google-styleguide.googlecode.com/svn/trunk/cppguide.html#Header_Files) +**A Header (.h) File** should generally be included for each .c file. The header should be self-contained and include a `#ifdef` guard to prevent compiler errors. The `ifdef` should use the form `_FILENAME_H`. [More Details.](https://google.github.io/styleguide/cppguide.html#Header_Files) **Folders** should be used to separate out major components in large projects. For smaller projects this is not necessary. @@ -71,7 +71,7 @@ Your project is likely to include multiple files, so you need to think about how ```` ## Variable and Function Naming -How you name your variables, data structures, and functions can be the dominant factor on whether your code is readable. In general, we follow the [Composite](https://github.com/gparmer/Composite/blob/master/doc/style_guide/composite_coding_style.pdf?raw=true) and [Google](http://google-styleguide.googlecode.com/svn/trunk/cppguide.html#Naming) naming conventions, although Google's recommendations are for C++ and thus use more capital letters than you are likely to use in a C program. +How you name your variables, data structures, and functions can be the dominant factor on whether your code is readable. In general, we follow the [Composite](https://github.com/gparmer/Composite/blob/master/doc/style_guide/composite_coding_style.pdf?raw=true) and [Google](https://google.github.io/styleguide/cppguide.html#Naming) naming conventions, although Google's recommendations are for C++ and thus use more capital letters than you are likely to use in a C program. **Understandable, Lower Case with Underscores.** Variable names should be meaningful and help users understand what they are for. Do not use CamelCase unless you are writing Java programs. In C, we prefer all lowercase function and variable names, with underscores between words where necessary. Avoid abbreviations except for very obvious cases. @@ -145,7 +145,7 @@ We follow the [Composite Style Guide's](https://github.com/gparmer/Composite/blo ## Style Guide Conformance You can use `gwclint.py` to check some of these style guide rules. Note that this is based on Google's style guide, so it may mark some pieces of code invalid which are fine, and it may not find other problems. -Usage: ``python gwclint.py FILENAME` +Usage: `python gwclint.py FILENAME` In particular, the checker currently *does not*: * Check for 8 space indentation (only checks for a multiple of 4) From 8da03ad759a652125829652afebb7eeba0aab9cd Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Fri, 2 Aug 2019 13:34:38 -0400 Subject: [PATCH 09/12] [Bug Fix] Update Env Setup Script Error Message (#154) Previously the environment setup script would print a wrong message if it wasn't called directly from its own directory. Commit log: * Fixed environment setup message * Removed cd and changed variable name --- scripts/setup_environment.sh | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/scripts/setup_environment.sh b/scripts/setup_environment.sh index b20899c38..fadd158b3 100755 --- a/scripts/setup_environment.sh +++ b/scripts/setup_environment.sh @@ -61,15 +61,11 @@ if [ -z "$RTE_SDK" ]; then exit 1 fi -# Ensure we're working relative to the onvm root directory -if [ $(basename $(pwd)) == "scripts" ]; then - cd .. -fi - -start_dir=$(pwd) +# Grab scripts parent directory location +onvm_home_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../ >/dev/null 2>&1 && pwd )" if [ -z "$ONVM_HOME" ]; then - echo "Please export \$ONVM_HOME and set it to $start_dir" + echo "Please export \$ONVM_HOME and set it to $onvm_home_dir" exit 1 fi From e217d71d8f10b197bc3d0826e7d8041a526d7eec Mon Sep 17 00:00:00 2001 From: Mykola Yurchenko Date: Sat, 3 Aug 2019 00:35:40 -0400 Subject: [PATCH 10/12] Add NF Core Reassignment on NF Termination (#87) This PR adds a feature that allows the onvm_mgr to remap a NF to a different core, if such opens out(when another NF shuts down). This feature is disabled by default, to enable it set the `ONVM_NF_SHUTDOWN_CORE_REASSIGNMENT` macro to true. Commit log: * Add core reassignment on NF shutdown * Fix core allocation order, cleanup * Better naming, fix typos * Fix minor style changes * Fix typo * Fix extra whitespace * Final fixes * Disable core reallocation by default --- onvm/onvm_mgr/onvm_nf.c | 44 ++++++++++++++++++++++++++++++- onvm/onvm_nflib/onvm_common.h | 4 ++- onvm/onvm_nflib/onvm_msg_common.h | 1 + onvm/onvm_nflib/onvm_nflib.c | 9 ++++++- onvm/onvm_nflib/onvm_threading.c | 37 ++++++++++++++++++++++++++ onvm/onvm_nflib/onvm_threading.h | 20 ++++++++++++++ 6 files changed, 112 insertions(+), 3 deletions(-) diff --git a/onvm/onvm_mgr/onvm_nf.c b/onvm/onvm_mgr/onvm_nf.c index d0a797a73..69e689070 100644 --- a/onvm/onvm_mgr/onvm_nf.c +++ b/onvm/onvm_mgr/onvm_nf.c @@ -87,6 +87,17 @@ onvm_nf_ready(struct onvm_nf *nf); inline static int onvm_nf_stop(struct onvm_nf *nf); +/* + * Function to move a NF to another core. + * + * Input : instance id of the NF that needs to be moved + * new_core value of where the NF should be moved + * Output : an error code + * + */ +inline int +onvm_nf_relocate_nf(uint16_t nf, uint16_t new_core); + /* * Function that initializes an LPM object * @@ -290,10 +301,11 @@ onvm_nf_stop(struct onvm_nf *nf) { uint16_t nf_status; uint16_t service_id; uint16_t nb_pkts, i; - int mapIndex; struct onvm_nf_msg *msg; struct rte_mempool *nf_info_mp; struct rte_mbuf *pkts[PACKET_READ_SIZE]; + uint16_t candidate_nf_id, candidate_core; + int mapIndex; if (nf == NULL) return 1; @@ -301,6 +313,7 @@ onvm_nf_stop(struct onvm_nf *nf) { nf_id = nf->instance_id; service_id = nf->service_id; nf_status = nf->status; + candidate_core = nf->thread_info.core; /* Cleanup the allocated tag */ if (nf->tag) { @@ -377,6 +390,15 @@ onvm_nf_stop(struct onvm_nf *nf) { } } + /* As this NF stopped we can reevaluate core mappings */ + if (ONVM_NF_SHUTDOWN_CORE_REASSIGNMENT) { + /* As this NF stopped we can reevaluate core mappings */ + candidate_nf_id = onvm_threading_find_nf_to_reassign_core(candidate_core, cores); + if (candidate_nf_id > 0) { + onvm_nf_relocate_nf(candidate_nf_id, candidate_core); + } + } + return 0; } @@ -395,3 +417,23 @@ onvm_nf_init_lpm_region(struct lpm_request *req_lpm) { req_lpm->status = -1; } } + +inline int +onvm_nf_relocate_nf(uint16_t dest, uint16_t new_core) { + uint16_t *msg_data; + + msg_data = rte_malloc("Change core msg data", sizeof(uint16_t), 0); + *msg_data = new_core; + + cores[nfs[dest].thread_info.core].nf_count--; + + onvm_nf_send_msg(dest, MSG_CHANGE_CORE, msg_data); + + /* We probably need logic that handles if everything is successful */ + + /* TODO Add core number */ + onvm_stats_gen_event_nf_info("NF Ready", &nfs[dest]); + + cores[new_core].nf_count++; + return 0; +} diff --git a/onvm/onvm_nflib/onvm_common.h b/onvm/onvm_nflib/onvm_common.h index 4368790d4..3b7ae24bf 100755 --- a/onvm/onvm_nflib/onvm_common.h +++ b/onvm/onvm_nflib/onvm_common.h @@ -57,6 +57,9 @@ #include "onvm_config_common.h" #include "onvm_msg_common.h" +#define ONVM_NF_HANDLE_TX 1 // should be true if NFs primarily pass packets to each other +#define ONVM_NF_SHUTDOWN_CORE_REASSIGNMENT 0 // should be true if on NF shutdown onvm_mgr tries to reallocate cores + #define ONVM_MAX_CHAIN_LENGTH 4 // the maximum chain length #define MAX_NFS 128 // total number of concurrent NFs allowed (-1 because ID 0 is reserved) #define MAX_SERVICES 32 // total number of unique services allowed @@ -398,7 +401,6 @@ struct lpm_request { #define NF_WAITING_FOR_LPM 13 // NF is waiting for a LPM request to be fulfilled #define NF_NO_ID -1 -#define ONVM_NF_HANDLE_TX 1 // should be true if NFs primarily pass packets to each other /* * Given the rx queue name template above, get the queue name diff --git a/onvm/onvm_nflib/onvm_msg_common.h b/onvm/onvm_nflib/onvm_msg_common.h index 3895aa723..1bb9af3db 100644 --- a/onvm/onvm_nflib/onvm_msg_common.h +++ b/onvm/onvm_nflib/onvm_msg_common.h @@ -53,6 +53,7 @@ #define MSG_SCALE 5 #define MSG_FROM_NF 6 #define MSG_REQUEST_LPM_REGION 7 +#define MSG_CHANGE_CORE 8 struct onvm_nf_msg { uint8_t msg_type; /* Constant saying what type of message is */ diff --git a/onvm/onvm_nflib/onvm_nflib.c b/onvm/onvm_nflib/onvm_nflib.c index 2e3ecee9e..1f6b94585 100755 --- a/onvm/onvm_nflib/onvm_nflib.c +++ b/onvm/onvm_nflib/onvm_nflib.c @@ -656,11 +656,18 @@ onvm_nflib_handle_msg(struct onvm_nf_msg *msg, struct onvm_nf_local_ctx *nf_loca onvm_nflib_scale((struct onvm_nf_scale_info*)msg->msg_data); break; case MSG_FROM_NF: - RTE_LOG(INFO, APP, "Recieved MSG from other NF"); + RTE_LOG(INFO, APP, "Received MSG from other NF\n"); if (nf_local_ctx->nf->function_table->msg_handler != NULL) { nf_local_ctx->nf->function_table->msg_handler(msg->msg_data, nf_local_ctx); } break; + case MSG_CHANGE_CORE: + RTE_LOG(INFO, APP, "Received relocation message...\n"); + RTE_LOG(INFO, APP, "Moving NF to core %d\n", *(uint16_t *)msg->msg_data); + nf_local_ctx->nf->thread_info.core = *(uint16_t *)msg->msg_data; + onvm_threading_core_affinitize(nf_local_ctx->nf->thread_info.core); + rte_free(msg->msg_data); + break; case MSG_NOOP: default: break; diff --git a/onvm/onvm_nflib/onvm_threading.c b/onvm/onvm_nflib/onvm_threading.c index f2aebd890..bcc65322a 100644 --- a/onvm/onvm_nflib/onvm_threading.c +++ b/onvm/onvm_nflib/onvm_threading.c @@ -137,3 +137,40 @@ onvm_threading_core_affinitize(int cpu) { return rte_thread_set_affinity(&cpus); } + +int +onvm_threading_find_nf_to_reassign_core(uint16_t candidate_core, struct core_status *cores) { + uint16_t candidate_nf_id, most_used_core, max_nfs_per_core; + int i; + + candidate_nf_id = most_used_core = max_nfs_per_core = 0; + + for (i = 0; i < onvm_threading_get_num_cores(); i++) { + if (cores[i].nf_count > max_nfs_per_core) { + max_nfs_per_core = cores[i].nf_count; + most_used_core = i; + } + } + + /* Core reassignment isn't required if the maximum nf_count doesn't exceed 1 NF + * or if the stopped NF was running on one of the most used cores */ + if (max_nfs_per_core == 1 || cores[candidate_core].nf_count >= max_nfs_per_core - 1) + return 0; + + /* Chooses one of the NFs running on candidate core + * TODO (@Deep) if we do this based of NF load this would be impressive, we should + * maintain a variable in onvm_nf struct describing NF load and choose the candidate based on that */ + for (i = 0; i < MAX_NFS; i++) { + if (onvm_nf_is_valid(&nfs[i]) && nfs[i].thread_info.core == most_used_core) { + candidate_nf_id = nfs[i].instance_id; + break; + } + } + + /* Sanity check, should not happen */ + if (candidate_nf_id == 0) { + rte_exit(EXIT_FAILURE, "Cannot locate NF running on core %u", most_used_core); + } + + return candidate_nf_id; +} diff --git a/onvm/onvm_nflib/onvm_threading.h b/onvm/onvm_nflib/onvm_threading.h index 39b06ac63..c33b2402b 100644 --- a/onvm/onvm_nflib/onvm_threading.h +++ b/onvm/onvm_nflib/onvm_threading.h @@ -43,6 +43,12 @@ #include "onvm_common.h" +/********************************Global variables*****************************/ + +extern struct onvm_nf *nfs; + +/********************************Interfaces***********************************/ + /** * Gets the total number of cores. */ @@ -84,4 +90,18 @@ onvm_threading_get_core(uint16_t *core_value, uint8_t flags, struct core_status int onvm_threading_core_affinitize(int core); +/** + * Based on current core usage decides if any NF should be moved to passed candidate core. + * + * @param candidate_core + * An integer core value to possibly move a NF to + * @param cores + * A pointer to the core_status map containing core information + * + * @return + * An instance ID of a candidate NF that should be moved to candidate_core + */ +int +onvm_threading_find_nf_to_reassign_core(uint16_t candidate_core, struct core_status *cores); + #endif // _ONVM_THREADING_H_ From ca145c65fe3181ce3aca560a661f8aba72f3bcba Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Tue, 6 Aug 2019 16:37:31 -0400 Subject: [PATCH 11/12] Add Pktgen Check for CI (#148) CI will now run Pktgen as an additional test metric. Commit log: * Save performance in ci * Add flask to install * Allowed for public linting of unauthorized users * Starting the queue * Start to fixing the queue problem * Added run mode for future advancements * Major fixes to ci list system * Remove debug lines * Clean up definitions * Remove exit statement * Add error handling in run_ci and fix docs * Create polling thread for requests * Delete new line * Account for run mode from #140 * Errors are much more difficult in threads :( * Smarter than using tuples * Added event-based handling instead of polling * Let request handler clears event * Initialize ci for pktgen * Huge update to pktgen configuration * Updated gitignore for log files * Added speed tester and pktgen in conjunction * Fix redundancies and bugs * Disabled flow lookup and created better benchmarks * Fixed things so nn30 works with pktgen * Fixes from github comments * Make results more stable by restarting manager * Added fixes from review and redundancies * Fixes from github review * Additions to readme about CI developments * Add a section about Pktgen helper nodes * Fix missed line * Add compatability for different interfaces * Fixes for user input and debugging * Fixes for Pktgen and binding interface * Renaming a ton of files * Fixed symbolic link * Added benchmark info to README and renamed file * Fix Pktgen install requested changes * Address Github change requests * Adjustments to interface configuration * Fixed manager debug --- ci/.gitignore | 6 +- ci/README.md | 37 +++++- ci/ci_busy.sh | 2 +- ...nctions.sh => helper-manager-functions.sh} | 45 +------ ci/install_pktgen/helper-worker-functions.sh | 1 + ci/install_pktgen/install-pktgen.sh | 41 +++++++ ci/install_pktgen/pktgen-timed-config.lua | 110 ++++++++++++++++++ ci/manager.sh | 43 ++++--- ci/pktgen-analysis.py | 44 +++++++ ci/post-msg.py | 44 +++++-- ci/prepare-worker.py | 8 +- ci/run-workload.py | 6 +- ci/run_ci.sh | 2 +- ci/speed-tester-analysis.py | 8 +- ci/worker.sh | 69 ----------- ci/worker_files/helper-worker-functions.sh | 95 +++++++++++++++ ci/worker_files/pktgen-worker.sh | 56 +++++++++ ci/worker_files/run-pktgen.py | 33 ++++++ ci/worker_files/speed-tester-worker.sh | 38 ++++++ ci/worker_files/worker-config | 7 ++ ci/worker_files/worker.sh | 66 +++++++++++ 21 files changed, 610 insertions(+), 151 deletions(-) rename ci/{helper-functions.sh => helper-manager-functions.sh} (81%) create mode 120000 ci/install_pktgen/helper-worker-functions.sh create mode 100755 ci/install_pktgen/install-pktgen.sh create mode 100644 ci/install_pktgen/pktgen-timed-config.lua create mode 100644 ci/pktgen-analysis.py delete mode 100755 ci/worker.sh create mode 100644 ci/worker_files/helper-worker-functions.sh create mode 100755 ci/worker_files/pktgen-worker.sh create mode 100644 ci/worker_files/run-pktgen.py create mode 100755 ci/worker_files/speed-tester-worker.sh create mode 100755 ci/worker_files/worker-config create mode 100755 ci/worker_files/worker.sh diff --git a/ci/.gitignore b/ci/.gitignore index 100ae87b0..1bc848e3c 100644 --- a/ci/.gitignore +++ b/ci/.gitignore @@ -1,4 +1,3 @@ -access_log config webhook-config.json encrypted_secret.bin @@ -10,4 +9,7 @@ repository linter-output.txt *key *.pub -*.stats +*stats +*out* +*log* +nimbnode* diff --git a/ci/README.md b/ci/README.md index a84513c52..2caf627a9 100644 --- a/ci/README.md +++ b/ci/README.md @@ -69,7 +69,7 @@ The CI process can be broken into multiple steps: 5. Run linter on the checked out code - Runs the `run_linter` function in `helper-functions.sh` + Runs the `run_linter` function in `helper-manager-functions.sh` 6. Clean up and restart all worker nodes @@ -83,11 +83,15 @@ The CI process can be broken into multiple steps: Use paramiko to ssh and run `run-workload.py` -9. Acquire results from the worker nodes +9. Run modes are supplied to tell the worker which applications to test + + Handle installation with `worker_files/worker.sh` for builds, and setting up manager for performance tests + +10. Acquire results from the worker nodes Use scp to copy the result stat file from worker -10. Submit results as a comment on github +11. Submit results as a comment on github Uses the `post-msg.py` script @@ -106,6 +110,33 @@ ProxyPassReverse /onvm-ci/ http://nimbnode44:8080/ ``` (Also need to setup github webhook to post to **http://nimbus.seas.gwu.edu/onvm-ci/github-webhook**) +### Public and Private CI Runs + +CI is now able to accept requests from unauthenticated users. There is a list of Github users in the public project allowed to create a full run. Anyone who is able to view the private `-dev` repository is able to run CI there as well. In `openNetVM`, if a user is not in our list, the linter and branch checks will be executed, ignoring statistics calculations from the worker nodes. + +### Setting Up a Connected Worker + +Connecting two nodes is useful for measuring statistics with tools like Pktgen and the MTCP stack. There is a bit of setup required to get working connection working. Firstly, an SFP+ 10Gb Intel cable will be required to connect the Network Interface Cards in the two machines. Once this is done, attempt to bring up the correct interfaces for a stable connection. Some debugging might be required: +- If you don't know which `ifconfig -a` interface is correct, use `ethtool -p 120` + - This will blink a light on the interface (you have to be next to the machine for this to help) +- Do this on both machines, to find the name of the interfaces that are linked +- Run `sudo ifconfig 11.0.0.1/24 up` on the first machine and `sudo ifconfig 11.0.0.2/24 up` + - This will ensure `ping` understands what IP address it is supposed to talk to +- If `ping -I 11.0.0.2` on the first machine works, great, if not, try changing the IP addresses or viewing `dmesg` + +Now that the interfaces are connected, choose which machine will be the CI worker, and which is a helper (Pktgen for example). Install Pktgen on this node by sending the `ci/install_pktgen` files to that machines' home folder. *Remember public keys must be created for all new machines*. Store these public keys in a folder with the server name, see the next section on statistics for more information. Run `chmod +x install-pktgen.sh` if it's not already an executable and run `./install-pktgen.sh` to install everything. If there are dependency errors, the machine might be a different version, so try to install the necessary packages. Once everything is installed, test ONVM->Pktgen between the machines, and if a connection is established, CI should work just fine with no more setup! + +### Advanced Statistics + +As CI continued to improve, with more programs to test with, benchmarks were made to track the average performance of a worker. In the future, CI will be able to handle multiple workers running many different tests. Since server configurations are not all the same, some with different hardware (Intel x710 vs. x520 NIC for example), performance of the nodes will not be the same. All that matters with CI is that the result of a run is the same or better, not globally across all nodes, but based on the specific server it ran on. For each worker, create a folder in the ci directory with the name of the worker IP. For example if `nimbnode17` is the current worker, a folder with path `/ci/nimbnode17/` should exist. In this folder, 3 files should be there at least. Firstly, a `benchmarks` file (used by the manager) should look similar to this: + +``` +AVG_SPEED_TESTER_SPEED=40000000 +AVG_PKTGEN_SPEED=10000000 +AVG_MTCP_SPEED=.230 +``` +This is a configuration file, sourced by the manager to keep track of `nimbnode17`'s average performance for each test (currently Speed Tester, Pktgen, and mTCP). The other two files in the folder should be the two public keys, one for the worker, and the second for the worker's client server. Check the previous section on setting up a connection for more information. + ### Checking if Online If you are worried if the CI build is offline or want to make sure it is listening for events, you can check the following url: curl http://nimbus.seas.gwu.edu/onvm-ci/status. If that URL returns 404, CI is offline. Otherwise it will display a message saying it is online. diff --git a/ci/ci_busy.sh b/ci/ci_busy.sh index 59d510f39..66038c6de 100755 --- a/ci/ci_busy.sh +++ b/ci/ci_busy.sh @@ -2,7 +2,7 @@ set -e -. helper-functions.sh +. helper-manager-functions.sh SCRIPT_LOC=$(pwd) print_header "Validating Config File and Sourcing Variables" diff --git a/ci/helper-functions.sh b/ci/helper-manager-functions.sh similarity index 81% rename from ci/helper-functions.sh rename to ci/helper-manager-functions.sh index 0e28417f5..c60d8fb8f 100644 --- a/ci/helper-functions.sh +++ b/ci/helper-manager-functions.sh @@ -38,45 +38,6 @@ print_header() { echo "" } - # sets up dpdk, sets env variables, and runs the install script -install_env() { - git submodule sync - git submodule update --init - - echo export ONVM_HOME=$(pwd) >> ~/.bashrc - export ONVM_HOME=$(pwd) - - cd dpdk - - echo export RTE_SDK=$(pwd) >> ~/.bashrc - export RTE_SDK=$(pwd) - - echo export RTE_TARGET=x86_64-native-linuxapp-gcc >> ~/.bashrc - export RTE_TARGET=x86_64-native-linuxapp-gcc - - echo export ONVM_NUM_HUGEPAGES=1024 >> ~/.bashrc - export ONVM_NUM_HUGEPAGES=1024 - - echo $RTE_SDK - - sudo sh -c "echo 0 > /proc/sys/kernel/randomize_va_space" - - cd ../ - pwd - . ./scripts/install.sh -} - -# makes all onvm code -build_onvm() { - cd onvm - make clean && make - cd ../ - - cd examples - make clean && make - cd ../ -} - # obtains core config in cores.out file obtain_core_config() { cd scripts @@ -140,3 +101,9 @@ run_linter() { fi done } + +# inputs are key_file, worker ip address, stats file - in that order 1,2,3 +fetch_files() { + scp -i $1 -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $2:$3 ./$2.$3 + check_exit_code "ERROR: Failed to fetch results from $2" +} diff --git a/ci/install_pktgen/helper-worker-functions.sh b/ci/install_pktgen/helper-worker-functions.sh new file mode 120000 index 000000000..ab2dbde82 --- /dev/null +++ b/ci/install_pktgen/helper-worker-functions.sh @@ -0,0 +1 @@ +../worker_files/helper-worker-functions.sh \ No newline at end of file diff --git a/ci/install_pktgen/install-pktgen.sh b/ci/install_pktgen/install-pktgen.sh new file mode 100755 index 000000000..7811ef947 --- /dev/null +++ b/ci/install_pktgen/install-pktgen.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +. helper-install-functions.sh + +set -e + +sudo rm -rf repository + +git clone https://github.com/sdnfv/openNetVM.git repository +check_exit_code "ERROR: Failed cloning" + +print_header "Installing Dependencies" +sudo apt-get update +sudo apt-get upgrade -y +sudo apt-get install -y build-essential linux-headers-$(uname -r) git +sudo apt-get install -y libnuma1 +sudo apt-get install -y libnuma-dev +sudo apt-get install libpcap-dev +sudo apt-get install libreadline-dev + +print_header "Installing Lua" +curl -R -O http://www.lua.org/ftp/lua-5.3.5.tar.gz +tar zxf lua-5.3.5.tar.gz +cd lua-5.3.5 +sudo make linux test +sudo make install + +cd repository + +print_header "Installing Environment" +install_env $RUN_PKT +check_exit_code "ERROR: Installing environment failed" + +print_header "Make pktgen-dpdk" +cd ~/repository/tools/Pktgen/pktgen-dpdk/ +make + +print_header "Updating lua script" +cp ~/pktgen-timed-config.lua ~/repository/tools/Pktgen/openNetVM-Scripts/pktgen-config.lua + +print_header "Pktgen installed" diff --git a/ci/install_pktgen/pktgen-timed-config.lua b/ci/install_pktgen/pktgen-timed-config.lua new file mode 100644 index 000000000..d5b3cac9d --- /dev/null +++ b/ci/install_pktgen/pktgen-timed-config.lua @@ -0,0 +1,110 @@ +-- openNetVM +-- https://github.com/sdnfv/openNetVM +-- +-- BSD LICENSE +-- +-- Copyright(c) +-- 2015-2016 George Washington University +-- 2015-2016 University of California Riverside +-- All rights reserved. + +-- Redistribution and use in source and binary forms, with or without +-- modification, are permitted provided that the following conditions +-- are met: + +-- Redistributions of source code must retain the above copyright +-- notice, this list of conditions and the following disclaimer. +-- Redistributions in binary form must reproduce the above copyright +-- notice, this list of conditions and the following disclaimer in +-- the documentation and/or other materials provided with the +-- distribution. +-- The name of the author may not be used to endorse or promote +-- products derived from this software without specific prior +-- written permission. + +-- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +-- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +-- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +-- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +-- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +-- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +-- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +-- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +-- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +-- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +-- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-- Change any of the settings below to configure Pktgen-DPDK + +-- A list of the test script for Pktgen and Lua. +-- Each command somewhat mirrors the pktgen command line versions. +-- A couple of the arguments have be changed to be more like the others. + +package.path = package.path ..";?.lua;test/?.lua;app/?.lua;" + +require "Pktgen" + +local function doWait(port, waitTime) + local idx; + + pktgen.delay(1000); + + pkt_rate_file = io.open("port_stats", "w"); + + if ( waitTime == 0 ) then + return; + end + waitTime = waitTime - 1; + + -- Try to wait for the total number of packets to be sent. + local idx = 0; + while( idx < waitTime ) do + -- Write port stats to output file separated by line + pkt_rate_file:write(pktgen.portStats("all", "rate")[0]["pkts_rx"] .. "\n"); + idx = idx + 1; + + local sending = pktgen.isSending(port); + if ( sending[tonumber(port)] == "n" ) then + break; + end + pktgen.delay(1000); + end + + pkt_rate_file:close() +end + +printf("Lua Version : %s\n", pktgen.info.Lua_Version); +printf("Pktgen Version : %s\n", pktgen.info.Pktgen_Version); +printf("Pktgen Copyright : %s\n", pktgen.info.Pktgen_Copyright); + +prints("pktgen.info", pktgen.info); + +printf("Port Count %d\n", pktgen.portCount()); +printf("Total port Count %d\n", pktgen.totalPorts()); + + +-- set up a mac address to set flow to +-- +-- TO DO LIST: +-- +-- Please update this part with the destination mac address, source and destination ip address you would like to sent packets to + +pktgen.set_mac("0", "90:e2:ba:5e:73:21"); +pktgen.set_ipaddr("0", "dst", "10.11.1.17"); +pktgen.set_ipaddr("0", "src", "10.11.1.16"); + +pktgen.set_proto("all", "udp"); +pktgen.set_type("all", "ipv4"); + +pktgen.set("all", "size", 64) +pktgen.set("all", "burst", 32); +pktgen.set("all", "sport", 1234); +pktgen.set("all", "dport", 1234); +pktgen.set("all", "count", 1000000000); +pktgen.set("all", "rate",100); + +pktgen.vlan_id("all", "start", 1); + +pktgen.start("all"); +doWait("all", 30); +pktgen.quit(); diff --git a/ci/manager.sh b/ci/manager.sh index dce469143..2b7005f17 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -4,7 +4,7 @@ set -e # source helper functions file -. helper-functions.sh +. helper-manager-functions.sh SCRIPT_LOC=$(pwd) print_header "Validating Config File and Sourcing Variables" @@ -78,7 +78,7 @@ fi print_header "Cleaning up Old Results" sudo rm -f *.txt -sudo rm -rf stats +sudo rm -rf *stats sudo rm -rf repository print_header "Checking Worker and GitHub Creds Exist" @@ -127,7 +127,6 @@ then fi print_header "Preparing Workers" - for worker_tuple in "${WORKER_LIST[@]}" do tuple_arr=($worker_tuple) @@ -157,10 +156,16 @@ do tuple_arr=($worker_tuple) worker_ip="${tuple_arr[0]}" worker_key_file="${tuple_arr[1]}" - scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -r ./repository $worker_ip: - check_exit_code "ERROR: Failed to copy ONVM files to $worker_ip" - scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null helper-functions.sh worker.sh $worker_ip: + # make sure the config file is updated with the correct run mode + sed -i "/WORKER_MODE*/c\\WORKER_MODE=\"${RUN_MODE}\"" worker_files/worker-config + # create directory for scp + mkdir temp + # put all files in one temporary folder for one scp + cp -r ./$worker_ip/* ./repository ./worker_files/* temp + scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -r ./temp/* $worker_ip: check_exit_code "ERROR: Failed to copy ONVM files to $worker_ip" + # get rid of the temp folder now for next worker + sudo rm -rf temp done print_header "Running Workloads on Workers" @@ -174,18 +179,30 @@ do done print_header "Obtaining Performance Results from all workers" - -rm -f results_summary.stats - for worker_tuple in "${WORKER_LIST[@]}" do tuple_arr=($worker_tuple) worker_ip="${tuple_arr[0]}" worker_key_file="${tuple_arr[1]}" - scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $worker_ip:stats ./$worker_ip.stats - check_exit_code "ERROR: Failed to fetch results from $worker_ip" - # TODO: this will overwrite results if we have more than 1 worker, investigate this case - python3 speed-tester-analysis.py ./$worker_ip.stats $worker_ip results_summary.stats + # get the benchmarks for each node (some servers are faster) + . ./$worker_ip/benchmarks + # TODO: this will overwrite results if we have more than 1 worker, investigate this case + if [[ "$RUN_MODE" -eq "0" ]] + then + # fetch pktgen stats + fetch_files $worker_key_file $worker_ip pktgen_stats + python3 pktgen-analysis.py ./$worker_ip.pktgen_stats $worker_ip pktgen_summary.stats $AVG_PKTGEN_SPEED + check_exit_code "Failed to parse Pktgen stats" + # fetch speed_tester stats + fetch_files $worker_key_file $worker_ip speed_stats + python3 speed-tester-analysis.py ./$worker_ip.speed_stats $worker_ip speed_summary.stats $AVG_SPEED_TESTER_SPEED + check_exit_code "Failed to parse Speed Tester stats" + else + # only fetch speed tester stats if mode is not 0 + fetch_files $worker_key_file $worker_ip speed_stats + python3 speed-tester-analysis.py ./$worker_ip.speed_stats $worker_ip speed_summary.stats $AVG_SPEED_TESTER_SPEED + check_exit_code "Failed to parse Speed Tester stats" + fi check_exit_code "ERROR: Failed to analyze results from $worker_ip" done diff --git a/ci/pktgen-analysis.py b/ci/pktgen-analysis.py new file mode 100644 index 000000000..9dfb6ffa0 --- /dev/null +++ b/ci/pktgen-analysis.py @@ -0,0 +1,44 @@ +import sys +import os +import json + +""" +get median value of tx pps +""" +def median(array): + array = sorted(array) + half, odd = divmod(len(array), 2) + if odd: + return array[half] + else: + return (array[half - 1] + array[half]) / 2.0 + +if(len(sys.argv) != 5): + print("ERROR: Invalid arguments.") + sys.exit(1) + +STATS_FILE = sys.argv[1] +STATS_NODE_NAME = sys.argv[2] +OUT_FILE = sys.argv[3] +AVG_SPEED = int(sys.argv[4]) + +contents = tuple(open(STATS_FILE, "r")) +data = [int(x.rstrip()) for x in contents if x.rstrip()] + +median_speed = median(data) + +results = {} +results['node'] = STATS_NODE_NAME +results['speed'] = median_speed + +performance_rating = (median_speed / AVG_SPEED) * 100 +if (performance_rating < 97): + results['pass_performance_check'] = False +else: + results['pass_performance_check'] = True +results['performance_rating'] = performance_rating +results['results_from'] = "[Results from %s]" % (STATS_NODE_NAME) +results['summary'] = "\n - Median TX pps for Pktgen: %d\n Performance rating - %.2f%% (compared to %d average)" % (median_speed, performance_rating, AVG_SPEED) + +with open(OUT_FILE, 'w') as outfile: + json.dump(results, outfile) diff --git a/ci/post-msg.py b/ci/post-msg.py index b9b1124c9..f105ec514 100644 --- a/ci/post-msg.py +++ b/ci/post-msg.py @@ -4,6 +4,7 @@ import json import sys +from os import path REPO_OWNER = None REPO_NAME = None @@ -14,6 +15,30 @@ print("ERROR! Incorrect number of arguments") sys.exit(1) +# add results of statistics file to Github comments +def process_results_from_worker(file, name): + global comment_body + global ACTION + if path.exists(file): + with open(file) as f: + results = json.load(f) + if (results['pass_performance_check']): + comment_body += " :heavy_check_mark: {} performance check passed\n".format(name) + else: + comment_body += " :x: PR drops {} performance below minimum requirement\n".format(name) + ACTION = 'REQUEST_CHANGES' + +# workers have many stats files, make sure its name only displays once +def add_results_from_worker(file): + global comment_body + global previous_results_from + with open(file) as f: + results = json.load(f) + if previous_results_from != results['results_from']: + comment_body += "\n" + results['results_from'] + previous_results_from = results['results_from'] + comment_body += "\n" + results['summary'] + with open(sys.argv[1], "r") as credsfile: creds = [x.strip() for x in credsfile.readlines()] @@ -45,6 +70,7 @@ print("ERROR: Could not get PR information from GitHub for PR %d" % int(sys.argv[2])) sys.exit(1) +previous_results_from = "" comment_body="" for line in REQUEST.split('\n'): comment_body += "> " + line + "\n" @@ -62,13 +88,10 @@ # PR must not affect performance if POST_RESULTS: - with open('./results_summary.stats') as f: - results = json.load(f) - if (results['pass_performance_check']): - comment_body += " :heavy_check_mark: Speed tester performance check passed\n" - else: - comment_body += " :x: PR drops speed tester perforamce below minimum requirement\n" - ACTION = 'REQUEST_CHANGES' + file = './pktgen_summary.stats' + process_results_from_worker(file, "Pktgen") + file = './speed_summary.stats' + process_results_from_worker(file, "Speed Test") # PR must pass linter check linter_output = None @@ -85,9 +108,10 @@ comment_body += " :heavy_check_mark: Linter passed\n" if POST_RESULTS: - with open('./results_summary.stats') as f: - results = json.load(f) - comment_body += "\n " + results['summary'] + file = './pktgen_summary.stats' + add_results_from_worker(file) + file = './speed_summary.stats' + add_results_from_worker(file) if POST_LINTER_OUTPUT: linter_output = None diff --git a/ci/prepare-worker.py b/ci/prepare-worker.py index 8195c45d5..a8ba22b0b 100644 --- a/ci/prepare-worker.py +++ b/ci/prepare-worker.py @@ -17,11 +17,9 @@ client.connect(worker_ip, timeout = 30, pkey = key) -(stdin, stdout, stderr) = client.exec_command("sudo rm -rf /mnt/huge/*") -(stdin, stdout, stderr) = client.exec_command("sudo rm -rf repository") -(stdin, stdout, stderr) = client.exec_command("sudo rm -rf stats") -(stdin, stdout, stderr) = client.exec_command("sudo rm -f worker.sh") -(stdin, stdout, stderr) = client.exec_command("sudo rm -f helper-functions.sh") +# put all removals in one line to save execution time +wipe_files = "sudo rm -rf /mnt/huge/* repository *stats* *config* *key* *.py *.sh" +(stdin, stdout, stderr) = client.exec_command(wipe_files) (stdin, stdout, stderr) = client.exec_command("sudo reboot") print("Successfully sent {} to reboot".format(worker_ip)) diff --git a/ci/run-workload.py b/ci/run-workload.py index 9f7dc202f..4b81d8cf4 100644 --- a/ci/run-workload.py +++ b/ci/run-workload.py @@ -4,6 +4,7 @@ from paramiko import SSHClient from paramiko import AutoAddPolicy import sys +import datetime """ get buffered stream of input from the client channel while the worker.sh script executes @@ -30,9 +31,6 @@ def line_buffered(f): client.connect(ip, timeout = 30, pkey = key) -(stdin, stdout, stderr) = client.exec_command("sudo ./worker.sh") - -for l in line_buffered(stdout): - print(l.strip("\n")) +(stdin, stdout, stderr) = client.exec_command("sudo ./worker.sh worker-config", get_pty=True) print(str(stderr.read())) diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 4b05c4e3d..b19f32ba1 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -4,7 +4,7 @@ set -e # source helper functions file -. helper-functions.sh +. helper-manager-functions.sh SCRIPT_LOC=$(pwd) print_header "Validating Input Variables" diff --git a/ci/speed-tester-analysis.py b/ci/speed-tester-analysis.py index 8d34194f7..a18c82bd4 100644 --- a/ci/speed-tester-analysis.py +++ b/ci/speed-tester-analysis.py @@ -3,8 +3,6 @@ import json import re -AVG_SPEED = 35000000 - """ get median value of tx pps """ @@ -16,13 +14,14 @@ def median(array): else: return (array[half - 1] + array[half]) / 2.0 -if(len(sys.argv) != 4): +if(len(sys.argv) != 5): print("ERROR: Invalid arguments.") sys.exit(1) STATS_FILE = sys.argv[1] STATS_NODE_NAME = sys.argv[2] OUT_FILE = sys.argv[3] +AVG_SPEED = int(sys.argv[4]) with open(STATS_FILE, "r") as f: contents = f.read() @@ -41,7 +40,8 @@ def median(array): else: results['pass_performance_check'] = True results['performance_rating'] = performance_rating -results['summary'] = "[Results from %s]\n - Median TX pps for Speed Tester: %d\n - Performance rating - %.2f%% (compared to %d average)" % (STATS_NODE_NAME, median_speed, performance_rating, AVG_SPEED) +results['results_from'] = "[Results from %s]" % (STATS_NODE_NAME) +results['summary'] = "\n - Median TX pps for Speed Tester: %d\n Performance rating - %.2f%% (compared to %d average)" % (median_speed, performance_rating, AVG_SPEED) with open(OUT_FILE, 'w') as outfile: json.dump(results, outfile) diff --git a/ci/worker.sh b/ci/worker.sh deleted file mode 100755 index c49baae9d..000000000 --- a/ci/worker.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - - # source helper functions file -. helper-functions.sh - -sudo apt-get update -sudo apt-get upgrade -y - -sudo apt-get install -y build-essential linux-headers-$(uname -r) git -sudo apt-get install -y libnuma1 -sudo apt-get install -y libnuma-dev - -cd repository - -print_header "Beginning Execution of Workload" - -print_header "Installing Environment" -install_env -check_exit_code "ERROR: Installing environment failed" - -print_header "Building ONVM" -build_onvm -check_exit_code "ERROR: Building ONVM failed" - -print_header "Running ONVM Manager" -cd onvm -./go.sh 0,1,2,3 0 0xF0 -s web -a 0x7f000000000 & -mgr_pid=$? -if [ $mgr_pid -ne 0 ] -then - echo "ERROR: Starting manager failed" - exit 1 -fi - -# wait for the manager to come online -sleep 15 - -# Some weirdness with PIDS, look into this -# Works, but we don't know why - -print_header "Running Speed Tester NF" -cd ../examples/speed_tester -./go.sh 1 -d 1 &>~/stats & -spd_tstr_pid=$? -if [ $spd_tstr_pid -ne 0 ] -then - echo "ERROR: Starting speed tester failed" - sudo kill $mgr_pid - exit 1 -fi - - # wait for speed tester to come online -print_header "Collecting Statistics" -sleep 15 - -print_header "Exiting ONVM" - -echo "Speed tester pid: ${spd_tstr_pid}" -echo $spd_tstr_pid -echo "Manager pid: ${mgr_pid}" -echo $mgr_pid - -sudo kill $spd_tstr_pid -check_exit_code "Error: Killing speed tester failed!" - -sudo kill $mgr_pid -check_exit_code "ERROR: Killing manager failed" - -print_header "Performance Tests Completed Successfully" diff --git a/ci/worker_files/helper-worker-functions.sh b/ci/worker_files/helper-worker-functions.sh new file mode 100644 index 000000000..e6f093426 --- /dev/null +++ b/ci/worker_files/helper-worker-functions.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +LOGFILE=worker.log +# remove previous log if it's over 300 lines +RETAIN_NUM_LINES=300 +DPDK_DEVBIND=$RTE_SDK/usertools/dpdk-devbind.py + +# print to stdout and the logfile +logsetup() { + TMP=$(tail -n $RETAIN_NUM_LINES $LOGFILE 2>/dev/null) && echo "${TMP}" > $LOGFILE + exec > >(tee -a $LOGFILE) + exec 2>&1 +} + +log() { + # check that the argument exists + if [ -z "$1" ] + then + echo "No argument supplied to log!" + return 1 + fi + + echo "[$(date --rfc-3339=seconds)]: $*" +} + +# set up our log file +logsetup + +# given a 10G NIC interface, bring down and bind to dpdk +bind_nic_from_iface() { + sudo ifconfig $1 down + id=$($DPDK_DEVBIND --status | grep -e "if=$1 drv=ixgbe" | cut -f 1 -d " ") + sudo $DPDK_DEVBIND -b igb_uio $id +} + + # sets up dpdk, sets env variables, and runs the install script +install_env() { + git submodule sync + git submodule update --init + + echo export ONVM_HOME=$(pwd) >> ~/.bashrc + export ONVM_HOME=$(pwd) + + cd dpdk + + echo export RTE_SDK=$(pwd) >> ~/.bashrc + export RTE_SDK=$(pwd) + + echo export RTE_TARGET=x86_64-native-linuxapp-gcc >> ~/.bashrc + export RTE_TARGET=x86_64-native-linuxapp-gcc + + echo export ONVM_NUM_HUGEPAGES=1024 >> ~/.bashrc + export ONVM_NUM_HUGEPAGES=1024 + + sudo sh -c "echo 0 > /proc/sys/kernel/randomize_va_space" + + # helper for binding interfaces + export DPDK_DEVBIND=$RTE_SDK/usertools/dpdk-devbind.py + + for iface in $($DPDK_DEVBIND --status | grep -oP "if=\K(\w+)\sdrv=ixgbe" | cut -f 1 -d " ") + do + # bring all ixgbe interfaces up so install script can't bind + sudo ifconfig $iface 11.0.0.1 up + done + + cd ../ + . ./scripts/install.sh + + # bring client facing interface down for dpdk + bind_nic_from_iface $CLIENT_IFACE + # disable flow table lookup for faster results + sed -i "/ENABLE_FLOW_LOOKUP\=1/c\\ENABLE_FLOW_LOOKUP=0" ~/repository/onvm/onvm_mgr/Makefile +} + +# makes all onvm code +build_onvm() { + cd onvm + make clean && make + cd ../ + + cd examples + make clean && make + cd ../ +} + +# checks if a command has failed (exited with code != 0) +# if it does, print the error message, exit the build, and post to github +check_exit_code() { + if [ $? -ne 0 ] + then + echo $1 + cd $SCRIPT_LOC + exit 1 + fi +} diff --git a/ci/worker_files/pktgen-worker.sh b/ci/worker_files/pktgen-worker.sh new file mode 100755 index 000000000..2f561a4a3 --- /dev/null +++ b/ci/worker_files/pktgen-worker.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +run_and_fetch_pktgen_stats() { + python3 ~/run-pktgen.py $WORKER_IP $WORKER_KEY_FILE $WORKER_USER + # get Pktgen stats from server + scp -i $WORKER_KEY_FILE -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $WORKER_USER@$WORKER_IP:~/repository/tools/Pktgen/pktgen-dpdk/port_stats ~/pktgen_stats +} + +log "Running ONVM Manager" +cd ~/repository/onvm +./go.sh 0,1,2,3 3 0xF0 -a 0x7f000000000 -s stdout &>~/onvm_pktgen_stats & +mgr_pid=0 +if [ $mgr_pid -ne 0 ] +then + echo "ERROR: Starting manager failed" + return 1 +fi + +# wait for the manager to come online +sleep 15 +log "Manager is live" + +log "Running Basic Monitor NF" +cd ~/repository/examples/basic_monitor +./go.sh 1 &>~/bsc_stats & +bsc_mntr_pid=0 +if [ $bsc_mntr_pid -ne 0 ] +then + echo "ERROR: Starting basic monitor failed" + return 1 +fi + +# make sure basic monitor initializes +sleep 10 + +# run pktgen +log "Collecting Pktgen Statistics" +python3 ~/run-pktgen.py $WORKER_IP $WORKER_KEY_FILE $WORKER_USER +# get Pktgen stats from server +scp -i $WORKER_KEY_FILE -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $WORKER_USER@$WORKER_IP:~/repository/tools/Pktgen/pktgen-dpdk/port_stats ~/pktgen_stats + +# check if pktgen returned results (non-zero) +if [ -z "$(grep -v "^0$" ~/pktgen_stats | cat)" ] +then + log "Running Pktgen again" + python3 ~/run-pktgen.py $WORKER_IP $WORKER_KEY_FILE $WORKER_USER + # get Pktgen stats from server + scp -i $WORKER_KEY_FILE -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $WORKER_USER@$WORKER_IP:~/repository/tools/Pktgen/pktgen-dpdk/port_stats ~/pktgen_stats +fi + +log "Killing Basic Monitor" +sudo pkill -f /basic_monitor + +log "Exiting ONVM" +sudo pkill -f onvm_mgr +check_exit_code "ERROR: Killing manager failed" diff --git a/ci/worker_files/run-pktgen.py b/ci/worker_files/run-pktgen.py new file mode 100644 index 000000000..5aa2e2fd9 --- /dev/null +++ b/ci/worker_files/run-pktgen.py @@ -0,0 +1,33 @@ +from paramiko import RSAKey +from paramiko import SSHClient +from paramiko import AutoAddPolicy +import sys +import datetime + +if len(sys.argv) != 4: + print("ERROR! Incorrect number of arguments") + sys.exit(1) + +worker_ip = sys.argv[1] +key_file = sys.argv[2] +worker_user = sys.argv[3] + +key = RSAKey.from_private_key_file(key_file) + +client = SSHClient() +client.set_missing_host_key_policy(AutoAddPolicy()) +client.connect(worker_ip, timeout = 30, username=worker_user, pkey = key) + +# Pktgen requires a pseudoterminal, created by get_pty +(stdin, stdout, stderr) = client.exec_command("sudo ~/repository/tools/Pktgen/openNetVM-Scripts/run-pktgen.sh 1", get_pty=True) +# block until script finishes +exit_status = stdout.channel.recv_exit_status() + +# write to file for debugging +with open('/home/' + worker_user + 'paramiko_pktgen_out.log', 'a+') as paramiko_out: + paramiko_out.write(datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y") + "\n") + paramiko_out.write(stdout.read().decode('ascii') + "\n") + +print("Successfully ran pktgen".format(worker_ip)) + +client.close() diff --git a/ci/worker_files/speed-tester-worker.sh b/ci/worker_files/speed-tester-worker.sh new file mode 100755 index 000000000..a849ac5da --- /dev/null +++ b/ci/worker_files/speed-tester-worker.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +log "Running ONVM Manager" +cd ~/repository/onvm +./go.sh 0,1,2,3 3 0xF0 -a 0x7f000000000 -s stdout &>~/onvm_speed_stats & +mgr_pid=$? +if [ $mgr_pid -ne 0 ] +then + echo "ERROR: Starting manager failed" + exit 1 +fi + +# wait for the manager to come online +sleep 15 +log "Manager is live" + +log "Running Speed Tester NF" +cd ~/repository/examples/speed_tester +./go.sh 1 -d 1 &>~/speed_stats & +spd_tstr_pid=$? +if [ $spd_tstr_pid -ne 0 ] +then + echo "ERROR: Starting speed tester failed" + return 1 +fi + +# wait for speed tester to come online +log "Collecting Speed Tester Statistics" +sleep 15 + +log "Killing Speed Tester" +sudo pkill -f /speed_tester + +log "Exiting Manager" + +echo "Manager pid: ${mgr_pid}" +sudo pkill -f onvm_mgr +check_exit_code "ERROR: Killing manager failed" diff --git a/ci/worker_files/worker-config b/ci/worker_files/worker-config new file mode 100755 index 000000000..297204a29 --- /dev/null +++ b/ci/worker_files/worker-config @@ -0,0 +1,7 @@ +WORKER_MODE="0" +CLIENT_IFACE="p2p1" +SERVER_IP_ADDR="11.0.0.17" +MTCP_RUN_TIME="15" +WORKER_IP="nimbnode18" +WORKER_KEY_FILE=~/ci18key +WORKER_USER="kdeems" diff --git a/ci/worker_files/worker.sh b/ci/worker_files/worker.sh new file mode 100755 index 000000000..878632041 --- /dev/null +++ b/ci/worker_files/worker.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +if [[ -z $1 ]] +then + echo "ERROR: Missing first argument, path to config file!" + exit 1 +fi + +if [[ ! -f $1 ]] +then + echo "ERROR: Could not find config file at given path!" + exit 1 +fi + +. $1 # source config file + +if [[ -z $WORKER_MODE ]] +then + echo "ERROR: Missing mode argument of config!" + exit 1 +fi + +# source helper functions file +. helper-worker-functions.sh + +sudo apt-get install -y build-essential linux-headers-$(uname -r) git +sudo apt-get install -y libnuma1 +sudo apt-get install -y libnuma-dev +sudo apt-get install -y python3 + +cd repository +log "Beginning Execution of Workload" + +log "Installing Environment" +install_env +check_exit_code "ERROR: Installing environment failed" + +log "Building ONVM" +build_onvm +check_exit_code "ERROR: Building ONVM failed" + +for mode in $WORKER_MODE +do + # run functionality for each mode + case "$mode" in + "0") + . ~/speed-tester-worker.sh + . ~/pktgen-worker.sh + . ~/mtcp-worker.sh + ;; + "1") + . ~/speed-tester-worker.sh + ;; + "2") + . ~/pktgen-worker.sh + ;; + "3") + . ~/mtcp-worker.sh + ;; + *) + echo "Worker mode $mode has not been implemented" + ;; + esac +done + +log "Performance Tests Completed Successfully" From e13c9e53ba4be952e0f045a4f3ef13a59f4d1862 Mon Sep 17 00:00:00 2001 From: Mykola Yurchenko Date: Wed, 7 Aug 2019 00:02:16 -0400 Subject: [PATCH 12/12] Update Releases Notes for ONVM v19.07 (#155) Commit log: * Update Releases.md * Update Releases.md * Update Releases.md * Apply suggestions from code review Co-Authored-By: Tim Wood * Update Releases.md * Update Releases.md --- docs/Releases.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/Releases.md b/docs/Releases.md index 221fb24f0..0963d9b3c 100644 --- a/docs/Releases.md +++ b/docs/Releases.md @@ -13,6 +13,42 @@ use a date based versioning system. Now, a release version can look like `17.11` where the "major" number is the year and the "minor" number is the month. +## v19.07 (7/19): NFD library and example NFs, Continuous Integration updates, minor improvements and bug fixes. +A CloudLab template is available with the latest release here: https://www.cloudlab.us/p/GWCloudLab/onvm + +**Performance**: This release includes a new macro `ENABLE_FLOW_LOOKUP` which controls whether a flow lookup is performed for every incoming packet. If disabled, all packets are forwarded to the default service ID which improves performance. The flow lookup is still enabled by default for backward compatibility with other applications that use ONVM. + +### NFD library with example NFS +Add example NFs based on NFD, a C++-based NF developing compiler designed by Wenfei Wu's group (http://wenfei-wu.github.io/) from IIIS, Tsinghua University, China. NFD compiles the NF logic into a common C++ program by using table-form language to model NFs' behavior. + +**The NFD compiler itself isn't included, only the NFs that were created with it.** + +A list of provided NFs using NFD library: + - DNS Amplification Mitigation + - Super Spread Detection + - Heavy Hitter Detection + - SYN Flood Detection + - UDP Flood Detection + - Stateless Firewall + - Stateful Firewall + - NAPT + +### Continuous Integration updates: +CI got a few major updates this release: + - CI will do basic lint checks and branch checks(all PRs should be submitted against the *develop* branch) for unauthorized users + - If CI is working on a request and receives another request it will append it to the queue instead of dropping it + - CI will now run Pktgen as an additional test metric. + +### Minor improvements +**Shared core functionality for messages** - adds functionality for NFs using shared core mode to work with NF messages. This means the NF will now sleep when no messages and no packets are enqueued onto a NF's message ring and wakeup if either one is received. + +**NF core rebalancing** - adds functionality for onvm_mgr to remap a NF to a different core, if such occurs(when another NF shuts down). This is disabled by default and can be enabled using the `ONVM_NF_SHUTDOWN_CORE_REASSIGNMENT` macro. + +### Bug fixes: + - Fix Style guide links + - Fix Typo in Stats Header bug fix + - Fix Stats Header in Release Notes (twice) + ## v19.05 (5/19): Shared Core Mode, Major Architectural Changes, Advanced Rings Changes, Stats Updates, CI PR Review, LPM Firewall NF, Payload Search NF, TTL Flags, minor improvements and bug fixes. A CloudLab template is available with the latest release here: https://www.cloudlab.us/p/GWCloudLab/onvm