wallet: clear memory leaks from radio doge functions

-total redesign of dogecoin_unregister_watch_address_with_node to alleviate calls to concat
-populate wallet->filename on dogecoin_wallet_new otherwise it's populated by name and freed
-added replace_last_after_delim which searchs for a match and replaces with replacement
-added remove_substr which returns a string with substr removed
-changed wallet cmd to sanity
-sanity cmd test now iterates through multiple addresses
-free waddr and buf and call dogecoin_wallet_next_addr on edge case where we delete our last address
-refactor/fix spvtool.py, fetch.py, .gitignore dummy
-clear spvnode wallet sanity cmd of memleaks
-fix wallet sanity check on windows
-added file_copy function to utils
-check if file exists before performance of file operations
-free remaining waddr during sanity check
-run wallet sanity check in spvtool.py test and verify wallet balance
-run spvtool.py only on returncode match from previous subprocess completion
-add Dockerfile for easier implementation
-add to x86_64-linux-dbg in ci.yml
-adapt to use -l for zero prompts during automated testing

.github/workflows/ci.yml

@@ -245,6 +245,8 @@ jobs:
               "x86_64-linux-dbg")
               make check -j"$(getconf _NPROCESSORS_ONLN)" V=1
               python3 tooltests.py
+              sudo ./rpctest/fetch.py --host x86_64-linux-gnu
+              sudo rm /usr/local/bin/dogecoind
               ;;
               "x86_64-linux-openenclave")
               make check -j"$(getconf _NPROCESSORS_ONLN)" V=1

.gitignore

@@ -118,6 +118,7 @@ dist/
 contrib/gitian/
 contrib/gitian/*
 example
+dummy
 
 # CMake files
 build/*

include/dogecoin/utils.h

@@ -81,19 +81,20 @@ LIBDOGECOIN_API void prepend(char* s, const char* t);
 LIBDOGECOIN_API void append(char* s, char* t);
 LIBDOGECOIN_API char* concat(char* prefix, char* suffix);
 LIBDOGECOIN_API void slice(const char *str, char *result, size_t start, size_t end);
+LIBDOGECOIN_API void replace_last_after_delim(const char *str, char* delim, char* replacement);
 LIBDOGECOIN_API void text_to_hex(char* in, char* out);
 LIBDOGECOIN_API const char* get_build();
 LIBDOGECOIN_API char* getpass(const char *prompt);
 LIBDOGECOIN_API void dogecoin_str_reverse(char s[]);
 LIBDOGECOIN_API void dogecoin_uitoa(int n, char s[]);
 LIBDOGECOIN_API bool dogecoin_network_enabled();
-
+LIBDOGECOIN_API int integer_length(int x);
+LIBDOGECOIN_API int file_copy (char src [], char dest []);
 unsigned int base64_int(unsigned int ch);
 unsigned int base64_encoded_size(unsigned int in_size);
 unsigned int base64_decoded_size(unsigned int in_size);
 unsigned int base64_encode(const unsigned char* in, unsigned int in_len, unsigned char* out);
 unsigned int base64_decode(const unsigned char* in, unsigned int in_len, unsigned char* out);
-int integer_length(int x);
 
 #define _SEARCH_PRIVATE
 #ifdef _SEARCH_PRIVATE

include/dogecoin/wallet.h

@@ -71,7 +71,7 @@ DISABLE_WARNING_POP
 
 /** single key/value record */
 typedef struct dogecoin_wallet_ {
-    const char* filename;
+    const char filename[311]; // max path length
     FILE *dbfile;
     dogecoin_hdnode* masterkey;
     uint32_t next_childindex; //cached next child index

rpctest/Dockerfile

@@ -0,0 +1,17 @@
+FROM ubuntu:jammy AS build
+
+# configure the shell before the first RUN
+SHELL ["/bin/bash", "-ex", "-o", "pipefail", "-c"]
+
+WORKDIR /home/root
+
+COPY ./rpctest ./rpctest
+COPY spvnode .
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+    python3 \
+    python3-requests \
+    libevent-dev
+
+RUN ./rpctest/fetch.py --host x86_64-linux-gnu

rpctest/fetch.py

@@ -98,13 +98,13 @@
 for f in deps_path:
     src = "dogecoin-1.14.6/bin/" + f
     src_path = os.path.join(os.getcwd(), src)
-    dst_path = os.path.join(os.getcwd(), "dogecoind")
-    shutil.move(src_path, dst_path)
+    if os.path.isdir('/usr/local/bin'):
+        dst_path = os.path.join('/usr/local/bin', f)
+        shutil.move(src_path, dst_path)
 
-subprocess.run([os.path.join(os.getcwd(), "rpctest/path.sh")])
 subprocess.run([os.path.join(os.getcwd(), "rpctest/spvtool.py")])
 
-rmlist = ['./dogecoin-*', 'dummy', 'dogecoind', '*.tar.gz', '*.zip', '*.asc']
+rmlist = ['./dogecoin-*', '*.dmg', '*.tar.gz', '*.zip', '*.asc']
 for path in rmlist:
     for name in glob.glob(path):
         if os.path.isdir(name):

rpctest/spvtool.py

@@ -23,9 +23,34 @@ def setup_network(self, split=False):
         cur_time = int(time.time())- 100*600
         for i in range(100):
             self.nodes[0].setmocktime(cur_time + 600)
+            min_relay_tx_fee = self.nodes[0].getnetworkinfo()['relayfee']
+            # This test is not meant to test fee estimation and we'd like
+            # to be sure all txs are sent at a consistent desired feerate
+            for node in self.nodes:
+                node.settxfee(min_relay_tx_fee)
+
+            # if the fee's positive delta is higher than this value tests will fail,
+            # neg. delta always fail the tests.
+            # The size of the signature of every input may be at most 2 bytes larger
+            # than a minimum sized signature.
+
+            #            = 2 bytes * minRelayTxFeePerByte
+            feeTolerance = 2 * min_relay_tx_fee/1000
+
+            self.nodes[0].generate(121)
+            self.sync_all()
+
+            # ensure that setting changePosition in fundraw with an exact match is handled properly
+            rawmatch = self.nodes[0].createrawtransaction([], {self.nodes[0].getnewaddress():500000})
+            rawmatch = self.nodes[0].fundrawtransaction(rawmatch, {"changePosition":1, "subtractFeeFromOutputs":[0]})
+            assert_equal(rawmatch["changepos"], -1)
+
+            self.nodes[0].sendtoaddress("mggFqzCUQmWWnh9vaoyT4BwKen7EbqhBmY", 1.5)
+            self.nodes[0].sendtoaddress("mrvi2kJiHJGb3fSyHVmRa19Pt1xwanxuEF", 1.0)
+            self.nodes[0].sendtoaddress("mmzGnpWs4VnwLvMoyRqbmf2GKbHTZks3bm", 5.0)
+
             self.nodes[0].generate(1)
             cur_time += 600
-
         self.nodes[0].setmocktime(cur_time + 1600)
 
     def execute_and_get_response(self, cmd):
@@ -47,22 +72,24 @@ def run_test (self):
         max_size = 1000
         log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16)
         log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16)
-
+        address = "mggFqzCUQmWWnh9vaoyT4BwKen7EbqhBmY mrvi2kJiHJGb3fSyHVmRa19Pt1xwanxuEF mmzGnpWs4VnwLvMoyRqbmf2GKbHTZks3bm"
         #sync with no headers database (-f 0) and debug (-d) only against localhost
-        cmd = "./spvnode --regtest -f 0 -d -i 127.0.0.1:"+str(p2p_port(0))+" scan"
+        cmd = "./spvnode --regtest -l -f 0 -d -i 127.0.0.1:"+str(p2p_port(0))+" -a '" + address + "' scan"
         data = self.execute_and_get_response(cmd)
-        assert("Sync completed, at height 100" in data)
-
+        assert("Sync completed, at height 11725" in data)
+        cmd = "./spvnode --regtest -a " + address + " sanity"
+        data = self.execute_and_get_response(cmd)
+        assert("total:          6.00000000" in data)
         # do the same with a headers db
-        try:
-            os.remove("headers.db")
-        except OSError:
-            pass
-        cmd = "./spvnode --regtest -d -i 127.0.0.1:"+str(p2p_port(0))+" scan"
+        cmd = "./spvnode --regtest -l -d -i 127.0.0.1:"+str(p2p_port(0))+" -a '" + address + "' scan"
+        data = self.execute_and_get_response(cmd)
+        assert("Sync completed, at height 11725" in data)
+        cmd = "./spvnode --regtest -a " + address + " sanity"
         data = self.execute_and_get_response(cmd)
-        assert("Sync completed, at height 100" in data)
+        assert("total:          6.00000000" in data)
         try:
-            os.remove("headers.db")
+            os.remove("regtest_headers.db")
+            os.remove("regtest_wallet.db")
         except OSError:
             pass
 

src/cli/spvnode.c

@@ -488,29 +488,93 @@ int main(int argc, char* argv[]) {
 #endif
             }
         dogecoin_ecc_stop();
-    } else if (strcmp(data, "wallet") == 0) {
+    } else if (strcmp(data, "sanity") == 0) {
 #if WITH_WALLET
     dogecoin_ecc_start();
     if (address != NULL) {
-        int res = dogecoin_register_watch_address_with_node(address);
-        printf("registered:     %d %s\n", res, address);
-        uint64_t amount = dogecoin_get_balance(address);
-        if (amount > 0) {
-            printf("amount:         %s\n", dogecoin_get_balance_str(address));
-            unsigned int utxo_count = dogecoin_get_utxos_length(address);
-            if (utxo_count) {
-                printf("utxo count:     %d\n", utxo_count);
-                unsigned int i = 1;
-                for (; i <= utxo_count; i++) {
-                    printf("txid:           %s\n", dogecoin_get_utxo_txid_str(address, i));
-                    printf("vout:           %d\n", dogecoin_get_utxo_vout(address, i));
-                    printf("amount:         %s\n", dogecoin_get_utxo_amount(address, i));
+        char delim[] = " ";
+        // copy address into a new string, strtok modifies the string
+        char* address_copy = strdup(address);
+
+        // backup existing default wallet file prior to radio doge functions test
+        const dogecoin_chainparams *params = chain_from_b58_prefix(address_copy);
+        dogecoin_wallet *tmp = dogecoin_wallet_new(params);
+        int result;
+        FILE *file;
+        if ((file = fopen(tmp->filename, "r")))
+        {
+            fclose(file);
+#ifdef WIN32
+            #include <winbase.h>
+            result = CopyFile((char*)tmp->filename, "tmp.bin", true);
+            if (result == 1) result = 0;
+#else
+            result = file_copy((char *)tmp->filename, "tmp.bin");
+#endif
+            if (result != 0) {
+                printf( "could not copy '%s' %d\n", tmp->filename, result );
+            } else {
+                printf( "File '%s' copied to 'tmp.bin'\n", tmp->filename);
+            }
+        }
+
+        char *ptr;
+        char* temp_address_copy = address_copy;
+
+        while((ptr = strtok_r(temp_address_copy, delim, &temp_address_copy))) {
+            int res = dogecoin_register_watch_address_with_node(ptr);
+            printf("registered:     %d %s\n", res, ptr);
+            uint64_t amount = dogecoin_get_balance(ptr);
+            if (amount > 0) {
+                char* amount_str = dogecoin_get_balance_str(ptr);
+                printf("total:          %s\n", amount_str);
+                unsigned int utxo_count = dogecoin_get_utxos_length(ptr);
+                if (utxo_count) {
+                    printf("utxo count:     %d\n", utxo_count);
+                    unsigned int i = 1;
+                    for (; i <= utxo_count; i++) {
+                        printf("txid:           %s\n", dogecoin_get_utxo_txid_str(ptr, i));
+                        printf("vout:           %d\n", dogecoin_get_utxo_vout(ptr, i));
+                        char* utxo_amount_str = dogecoin_get_utxo_amount(ptr, i);
+                        printf("amount:         %s\n", utxo_amount_str);
+                        dogecoin_free(utxo_amount_str);
+                    }
                 }
+                dogecoin_free(amount_str);
+            }
+            res = dogecoin_unregister_watch_address_with_node(ptr);
+            printf("unregistered:   %s\n", res ? "true" : "false");
+        }
+
+        if ((file = fopen("tmp.bin", "r"))) {
+            fclose(file);
+#ifdef WIN32
+            #include <winbase.h>
+            char *tmp_filename = _strdup((char *)tmp->filename);
+            char *filename = _strdup((char *)tmp->filename);
+            replace_last_after_delim(filename, "\\", "tmp.bin");
+            LPVOID message;
+            result = DeleteFile(tmp->filename);
+            if (!result) {
+                FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR)&message, 0, NULL);
+                printf("ERROR: %s\n", (char *)message);
+            }
+            result = rename(filename, tmp->filename);
+            dogecoin_free(filename);
+            dogecoin_free(tmp_filename);
+#else
+            result = rename("tmp.bin", tmp->filename);
+#endif
+            if( result != 0 ) {
+                printf( "could not copy 'tmp.bin' %d\n", result );
+            } else {
+                printf( "File 'tmp.bin' copied to '%s'\n", tmp->filename);
             }
         }
-        res = dogecoin_unregister_watch_address_with_node(address);
-        printf("unregistered:   %s\n", res ? "true" : "false");
+        dogecoin_wallet_free(tmp);
+        dogecoin_free(address_copy);
     }
+
     dogecoin_ecc_stop();
 #endif
     } else {

src/utils.c

@@ -620,6 +620,31 @@ void slice(const char *str, char *result, size_t start, size_t end)
     strncpy(result, str + start, end - start);
 }
 
+void remove_substr(char *string, char *sub) {
+    char *match;
+    int len = strlen(sub);
+    while ((match = strstr(string, sub))) {
+        *match = '\0';
+        strcat(string, match+len);
+    }
+}
+
+void replace_last_after_delim(const char *str, char* delim, char* replacement) {
+    char* tmp = strdup((char*)str);
+    char* new = tmp;
+    char *strptr = strtok(new, delim);
+    char* last = NULL;
+    while (strptr != NULL) {
+        last = strptr;
+        strptr = strtok(NULL, delim);
+    }
+    if (last) {
+        remove_substr((char*)str, last);
+        append((char*)str, replacement);
+    }
+    dogecoin_free(tmp);
+}
+
 /**
  * @brief function to convert ascii text to hexadecimal string
  *
@@ -768,6 +793,38 @@ bool dogecoin_network_enabled() {
 #endif
 }
 
+int integer_length(int x) {
+    int count = 0;
+    while (x > 0) {
+        x /= 10;
+        count++;
+    }
+    return count > 0 ? count : 1;
+}
+
+int file_copy(char src [], char dest [])
+{
+    int   c;
+    FILE *stream_read;
+    FILE *stream_write; 
+
+    stream_read = fopen (src, "r");
+    if (stream_read == NULL)
+        return -1;
+    stream_write = fopen (dest, "w");   //create and write to file
+    if (stream_write == NULL)
+     {
+        fclose (stream_read);
+        return -2;
+     }    
+    while ((c = fgetc(stream_read)) != EOF)
+        fputc (c, stream_write);
+    fclose (stream_read);
+    fclose (stream_write);
+
+    return 0;
+}
+
 unsigned char base64_char[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 
 unsigned int base64_int(unsigned int ch) {
@@ -869,12 +926,3 @@ unsigned int base64_decode(const unsigned char* in, unsigned int in_len, unsigne
 
 	return k;
 }
-
-int integer_length(int x) {
-    int count = 0;
-    while (x > 0) {
-        x /= 10;
-        count++;
-    }
-    return count > 0 ? count : 1;
-}