Merge pull request #6037 from guymguym/guy-md5

md5_mb (multibuffer) using isa-l_crypto
noobaa · Jun 4, 2020 · 4c87f61 · 4c87f61
2 parents c32fa14 + 3a6a8b9
commit 4c87f61
Show file tree

Hide file tree

Showing 19 changed files with 412 additions and 96 deletions.
diff --git a/src/agent/agent_cli.js b/src/agent/agent_cli.js
@@ -2,6 +2,7 @@
 'use strict';
 
 require('../util/panic');
+require('../util/fips');
 
 const _ = require('lodash');
 const fs = require('fs');

diff --git a/src/deploy/NVA_build/dev.Dockerfile b/src/deploy/NVA_build/dev.Dockerfile
@@ -0,0 +1,35 @@
+FROM centos:8 
+
+ENV container docker
+
+RUN dnf update -y -q && \
+    dnf install -y -q \
+        bash bash-completion \
+        wget curl nc unzip which less vim \
+        python2 python2-setuptools \
+        python3 python3-setuptools \
+        gdb strace lsof \
+        openssl && \
+    dnf --enablerepo=PowerTools install -y -q yasm && \
+    dnf group install -y -q "Development Tools" && \
+    dnf clean all
+
+RUN mkdir -p /usr/local/lib/python3.6/site-packages
+RUN alternatives --set python /usr/bin/python3
+
+WORKDIR /noobaa
+
+COPY ./.nvmrc ./.nvmrc
+COPY ./src/deploy/NVA_build/install_nodejs.sh ./
+RUN chmod +x ./install_nodejs.sh && \
+    ./install_nodejs.sh $(cat .nvmrc) && \
+    npm config set unsafe-perm true && \
+    echo '{ "allow_root": true }' > /root/.bowerrc
+
+COPY ./package*.json ./
+RUN npm install && \
+    npm cache clean --force
+
+COPY . ./
+USER 0:0
+CMD [ "bash" ]
diff --git a/src/endpoint/endpoint.js b/src/endpoint/endpoint.js
@@ -1,12 +1,12 @@
 /* Copyright (C) 2016 NooBaa */
 'use strict';
 
-require('../util/panic');
-
 // load .env file before any other modules so that it will contain
 // all the arguments even when the modules are loading.
 console.log('loading .env file');
 require('../util/dotenv').load();
+require('../util/panic');
+require('../util/fips');
 
 const http = require('http');
 const https = require('https');

diff --git a/src/hosted_agents/hosted_agents_starter.js b/src/hosted_agents/hosted_agents_starter.js
@@ -2,6 +2,8 @@
 'use strict';
 
 require('../util/dotenv').load();
+require('../util/panic');
+require('../util/fips');
 
 const url = require('url');
 

diff --git a/src/native/chunk/splitter.cpp b/src/native/chunk/splitter.cpp
@@ -32,17 +32,28 @@ Splitter::Splitter(
     , _hash(0)
     , _md5_ctx(0)
     , _sha256_ctx(0)
+    , _md5_mb_ctx(0)
+    , _md5_mb_mgr(0)
 {
     assert(_min_chunk > 0);
     assert(_min_chunk <= _max_chunk);
     assert(_avg_chunk_bits >= 0);
     nb_buf_init_alloc(&_window, NB_RABIN_WINDOW_LEN);
     memset(_window.data, 0, _window.len);
-    if (calc_md5) {
-        _md5_ctx = EVP_MD_CTX_new();
-        EVP_DigestInit_ex(_md5_ctx, EVP_md5(), NULL);
+    if (_calc_md5) {
+        extern bool fips_mode;
+        if (fips_mode) {
+            posix_memalign((void**)&_md5_mb_mgr, 16, sizeof(MD5_HASH_CTX_MGR));
+            posix_memalign((void**)&_md5_mb_ctx, 16, sizeof(MD5_HASH_CTX));
+            md5_ctx_mgr_init(_md5_mb_mgr);
+            hash_ctx_init(_md5_mb_ctx);
+            md5_mb_submit_and_flush(0, 0, HASH_FIRST);
+        } else {
+            _md5_ctx = EVP_MD_CTX_new();
+            EVP_DigestInit_ex(_md5_ctx, EVP_md5(), NULL);
+        }
     }
-    if (calc_sha256) {
+    if (_calc_sha256) {
         _sha256_ctx = EVP_MD_CTX_new();
         EVP_DigestInit_ex(_sha256_ctx, EVP_sha256(), NULL);
     }
@@ -51,15 +62,18 @@ Splitter::Splitter(
 Splitter::~Splitter()
 {
     nb_buf_free(&_window);
-    if (_calc_md5) EVP_MD_CTX_free(_md5_ctx);
-    if (_calc_sha256) EVP_MD_CTX_free(_sha256_ctx);
+    if (_md5_ctx) EVP_MD_CTX_free(_md5_ctx);
+    if (_sha256_ctx) EVP_MD_CTX_free(_sha256_ctx);
+    if (_md5_mb_ctx) free(_md5_mb_ctx);
+    if (_md5_mb_mgr) free(_md5_mb_mgr);
 }
 
 void
 Splitter::push(const uint8_t* data, int len)
 {
-    if (_calc_md5) EVP_DigestUpdate(_md5_ctx, data, len);
-    if (_calc_sha256) EVP_DigestUpdate(_sha256_ctx, data, len);
+    if (_md5_ctx) EVP_DigestUpdate(_md5_ctx, data, len);
+    if (_sha256_ctx) EVP_DigestUpdate(_sha256_ctx, data, len);
+    if (_md5_mb_ctx) md5_mb_submit_and_flush(data, len, HASH_UPDATE);
     while (_next_point(&data, &len)) {
         _split_points.push_back(_chunk_pos);
         _chunk_pos = 0;
@@ -69,8 +83,23 @@ Splitter::push(const uint8_t* data, int len)
 void
 Splitter::finish(uint8_t* md5, uint8_t* sha256)
 {
-    if (md5 && _calc_md5) EVP_DigestFinal_ex(_md5_ctx, md5, 0);
-    if (sha256 && _calc_sha256) EVP_DigestFinal_ex(_sha256_ctx, sha256, 0);
+    if (md5) {
+        if (_md5_mb_ctx) {
+            md5_mb_submit_and_flush(0, 0, HASH_LAST);
+            memcpy(md5, hash_ctx_digest(_md5_mb_ctx), MD5_DIGEST_NWORDS * 4);
+        } else if (_md5_ctx) {
+            EVP_DigestFinal_ex(_md5_ctx, md5, 0);
+        } else {
+            PANIC("no md5 context");
+        }
+    }
+    if (sha256) {
+        if (_sha256_ctx) {
+            EVP_DigestFinal_ex(_sha256_ctx, sha256, 0);
+        } else {
+            PANIC("no sha256 context");
+        }
+    }
 }
 
 bool
@@ -142,4 +171,5 @@ Splitter::_next_point(const uint8_t** const p_data, int* const p_len)
         return false;
     }
 }
-}
+
+} // namespace noobaa
diff --git a/src/native/chunk/splitter.h b/src/native/chunk/splitter.h
@@ -7,6 +7,7 @@
 
 #include "../util/rabin.h"
 #include "../util/struct_buf.h"
+#include "../third_party/isa-l_crypto/include/md5_mb.h"
 
 namespace noobaa
 {
@@ -46,11 +47,23 @@ class Splitter
     Points _split_points;
     Point _chunk_pos;
     Rabin::Hash _hash;
-    EVP_MD_CTX *_md5_ctx;
-    EVP_MD_CTX *_sha256_ctx;
+
+    EVP_MD_CTX* _md5_ctx;
+    EVP_MD_CTX* _sha256_ctx;
+    MD5_HASH_CTX* _md5_mb_ctx;
+    MD5_HASH_CTX_MGR* _md5_mb_mgr;
+
+    void md5_mb_submit_and_flush(const void* data, uint32_t size, HASH_CTX_FLAG flag)
+    {
+        md5_ctx_mgr_submit(_md5_mb_mgr, _md5_mb_ctx, data, size, flag);
+        while (hash_ctx_processing(_md5_mb_ctx)) {
+            md5_ctx_mgr_flush(_md5_mb_mgr);
+        }
+    }
 
     static Rabin _rabin;
 
     bool _next_point(const uint8_t** const p_data, int* const p_len);
 };
-}
+
+} // namespace noobaa
diff --git a/src/native/chunk/splitter_napi.cpp b/src/native/chunk/splitter_napi.cpp
@@ -5,6 +5,7 @@
 
 namespace noobaa
 {
+
 #define SPLITTER_JS_SIGNATURE "function chunk_splitter(state, buffers, callback?)"
 
 static Napi::Value _chunk_splitter(const Napi::CallbackInfo& info);
@@ -60,7 +61,7 @@ class SplitterWorker : public Napi::AsyncWorker
     virtual void OnOK()
     {
         auto result = _splitter_result(Env(), _splitter);
-        Callback().MakeCallback(Env().Global(), {Env().Null(), result});
+        Callback().MakeCallback(Env().Global(), { Env().Null(), result });
     }
 
 private:
@@ -167,4 +168,5 @@ _splitter_result(Napi::Env env, Splitter* splitter)
     }
     return arr;
 }
-}
+
+} // namespace noobaa
diff --git a/src/native/nb_native.cpp b/src/native/nb_native.cpp
@@ -5,7 +5,7 @@ namespace noobaa
 {
 
 void b64_napi(Napi::Env env, Napi::Object exports);
-void ssl_napi(napi_env env, napi_value exports);
+void ssl_napi(Napi::Env env, Napi::Object exports);
 void syslog_napi(Napi::Env env, Napi::Object exports);
 void splitter_napi(Napi::Env env, Napi::Object exports);
 void chunk_coder_napi(napi_env env, napi_value exports);

diff --git a/src/native/nb_native.gyp b/src/native/nb_native.gyp
@@ -36,6 +36,8 @@
             'third_party/cm256.gyp:cm256',
             'third_party/snappy.gyp:snappy',
             'third_party/isa-l.gyp:isa-l-ec',
+            'third_party/isa-l.gyp:isa-l-md5',
+            'third_party/isa-l.gyp:isa-l-sha1',
         ],
         'sources': [
             # module

diff --git a/src/native/third_party/isa-l.gyp b/src/native/third_party/isa-l.gyp
@@ -120,26 +120,43 @@
                 'isa-l_crypto/md5_mb/',
             ],
             'sources': [
+                # include
+                'isa-l_crypto/include/md5_mb.h',
+                'isa-l_crypto/include/types.h',
+                'isa-l_crypto/include/multi_buffer.h',
+                'isa-l_crypto/include/intrinreg.h',
+                'isa-l_crypto/include/reg_sizes.asm',
+                'isa-l_crypto/include/multibinary.asm',
+                'isa-l_crypto/include/datastruct.asm',
+                # asm
+                'isa-l_crypto/md5_mb/md5_multibinary.asm',
+                'isa-l_crypto/md5_mb/md5_job.asm',
+                'isa-l_crypto/md5_mb/md5_mb_mgr_datastruct.asm',
+                # ctx
+                # 'isa-l_crypto/md5_mb/md5_ctx_base.c',
                 'isa-l_crypto/md5_mb/md5_ctx_sse.c',
                 'isa-l_crypto/md5_mb/md5_ctx_avx.c',
                 'isa-l_crypto/md5_mb/md5_ctx_avx2.c',
+                'isa-l_crypto/md5_mb/md5_ctx_avx512.c',
+                # mgr_init
                 'isa-l_crypto/md5_mb/md5_mb_mgr_init_sse.c',
                 'isa-l_crypto/md5_mb/md5_mb_mgr_init_avx2.c',
                 'isa-l_crypto/md5_mb/md5_mb_mgr_init_avx512.c',
-                'isa-l_crypto/md5_mb/md5_mb_mgr_submit_sse.asm',
-                'isa-l_crypto/md5_mb/md5_mb_mgr_submit_avx.asm',
-                'isa-l_crypto/md5_mb/md5_mb_mgr_submit_avx2.asm',
+                # mgr_flush
                 'isa-l_crypto/md5_mb/md5_mb_mgr_flush_sse.asm',
                 'isa-l_crypto/md5_mb/md5_mb_mgr_flush_avx.asm',
                 'isa-l_crypto/md5_mb/md5_mb_mgr_flush_avx2.asm',
+                'isa-l_crypto/md5_mb/md5_mb_mgr_flush_avx512.asm',
+                # mgr_submit
+                'isa-l_crypto/md5_mb/md5_mb_mgr_submit_sse.asm',
+                'isa-l_crypto/md5_mb/md5_mb_mgr_submit_avx.asm',
+                'isa-l_crypto/md5_mb/md5_mb_mgr_submit_avx2.asm',
+                'isa-l_crypto/md5_mb/md5_mb_mgr_submit_avx512.asm',
+                # x4x2
                 'isa-l_crypto/md5_mb/md5_mb_x4x2_sse.asm',
                 'isa-l_crypto/md5_mb/md5_mb_x4x2_avx.asm',
                 'isa-l_crypto/md5_mb/md5_mb_x8x2_avx2.asm',
-                'isa-l_crypto/md5_mb/md5_multibinary.asm',
-                'isa-l_crypto/md5_mb/md5_mb_mgr_submit_avx512.asm',
-                'isa-l_crypto/md5_mb/md5_mb_mgr_flush_avx512.asm',
                 'isa-l_crypto/md5_mb/md5_mb_x16x2_avx512.asm',
-                'isa-l_crypto/md5_mb/md5_ctx_avx512.c',
             ],
         },
 
@@ -308,6 +325,24 @@
             'sources': ['isa-l/erasure_code/erasure_code_test.c']
         },
 
+        {
+            'target_name': 'md5_mb_test',
+            'type': 'executable',
+            'dependencies': ['isa-l-md5'],
+            'include_dirs': ['isa-l_crypto/include/'],
+            'sources': ['isa-l_crypto/md5_mb/md5_mb_test.c']
+        },
+
+        {
+            # TODO: dynamic linking to openssl usually needs more CFLAGS(-I) and LDFLAGS(-L)
+            'target_name': 'md5_mb_vs_ossl_perf',
+            'type': 'executable',
+            'dependencies': ['isa-l-md5'],
+            'include_dirs': ['isa-l_crypto/include/'],
+            'ldflags': ['-lssl','-lcrypto'],
+            'sources': ['isa-l_crypto/md5_mb/md5_mb_vs_ossl_perf.c']
+        },
+
         {
             'target_name': 'sha256_mb_test',
             'type': 'executable',