Merge pull request #495 from novoselrok/multithreaded-build

Multithreaded build

README.rst

@@ -78,7 +78,7 @@ Full Python API
 
 * ``AnnoyIndex(f, metric)`` returns a new index that's read-write and stores vector of ``f`` dimensions. Metric can be ``"angular"``, ``"euclidean"``, ``"manhattan"``, ``"hamming"``, or ``"dot"``.
 * ``a.add_item(i, v)`` adds item ``i`` (any nonnegative integer) with vector ``v``. Note that it will allocate memory for ``max(i)+1`` items.
-* ``a.build(n_trees)`` builds a forest of ``n_trees`` trees. More trees gives higher precision when querying. After calling ``build``, no more items can be added.
+* ``a.build(n_trees, n_jobs=-1)`` builds a forest of ``n_trees`` trees. More trees gives higher precision when querying. After calling ``build``, no more items can be added. ``n_jobs`` specifies the number of threads used to build the trees. ``n_jobs=-1`` uses all available CPU cores.
 * ``a.save(fn, prefault=False)`` saves the index to disk and loads it (see next function). After saving, no more items can be added.
 * ``a.load(fn, prefault=False)`` loads (mmaps) an index from disk. If `prefault` is set to `True`, it will pre-read the entire file into memory (using mmap with `MAP_POPULATE`). Default is `False`.
 * ``a.unload()`` unloads.

README_GO.rst

@@ -58,7 +58,7 @@ Right now it only accepts integers as identifiers for items. Note that it will a
 Full Go API
 ---------------
 
-See annoygomodule.h. Generally the same as Python API except some arguments are not optional. 
+See annoygomodule.h. Generally the same as Python API except some arguments are not optional. Go binding does not support multithreaded build.
 
 Tests
 -------

README_Lua.md

@@ -64,7 +64,7 @@ end
 Full Lua API
 ------------
 
-Lua API closely resembles Python API, see main README.
+Lua API closely resembles Python API, see main README. Lua binding does not support multithreaded build.
 
 
 Tests

examples/precision_test.cpp

@@ -25,7 +25,7 @@ int precision(int f=40, int n=1000000){
 
  //******************************************************
  //Building the tree
- AnnoyIndex<int, double, Angular, Kiss32Random> t = AnnoyIndex<int, double, Angular, Kiss32Random>(f);
+ AnnoyIndex<int, double, Angular, Kiss32Random, AnnoyIndexMultiThreadedBuildPolicy> t = AnnoyIndex<int, double, Angular, Kiss32Random, AnnoyIndexMultiThreadedBuildPolicy>(f);
 
  std::cout << "Building index ... be patient !!" << std::endl;
  std::cout << "\"Trees that are slow to grow bear the best fruit\" (Moliere)" << std::endl;

examples/s_compile_cpp.sh

@@ -2,6 +2,6 @@
 
 
 echo "compiling precision example..."
-cmd="g++ precision_test.cpp -o precision_test -std=c++11"
+cmd="g++ precision_test.cpp -DANNOYLIB_MULTITHREADED_BUILD -o precision_test -std=c++14 -pthread"
 eval $cmd
 echo "Done"

setup.py

@@ -19,6 +19,7 @@
 import codecs
 import os
 import platform
+import sys
 
 readme_note = """\
 .. note::
@@ -49,10 +50,19 @@
 if os.name != 'nt':
  extra_compile_args += ['-O3', '-ffast-math', '-fno-associative-math']
 
+# Add multithreaded build flag for all platforms using Python 3 and
+# for non-Windows Python 2 platforms
+python_major_version = sys.version_info[0]
+if python_major_version == 3 or (python_major_version == 2 and os.name != 'nt'):
+ extra_compile_args += ['-DANNOYLIB_MULTITHREADED_BUILD']
+
+ if os.name != 'nt':
+ extra_compile_args += ['-std=c++14']
+
 # #349: something with OS X Mojave causes libstd not to be found
 if platform.system() == 'Darwin':
- extra_compile_args += ['-std=c++11', '-mmacosx-version-min=10.9']
- extra_link_args += ['-stdlib=libc++', '-mmacosx-version-min=10.9']
+ extra_compile_args += ['-mmacosx-version-min=10.12']
+ extra_link_args += ['-stdlib=libc++', '-mmacosx-version-min=10.12']
 
 # Manual configuration, you're on your own here.
 manual_compiler_args = os.environ.get('ANNOY_COMPILER_ARGS', None)

src/annoygomodule.h

@@ -17,7 +17,7 @@ class AnnoyIndex {
  ptr->add_item(item, w);
  };
  void build(int q) {
- ptr->build(q);
+ ptr->build(q, 1);
  };
  bool save(const char* filename, bool prefault) {
  return ptr->save(filename, prefault);
@@ -69,23 +69,23 @@ class AnnoyIndexAngular : public AnnoyIndex
 {
  public:
  AnnoyIndexAngular(int f) {
- ptr = new ::AnnoyIndex<int32_t, float, ::Angular, ::Kiss64Random>(f);
+ ptr = new ::AnnoyIndex<int32_t, float, ::Angular, ::Kiss64Random, AnnoyIndexSingleThreadedBuildPolicy>(f);
  this->f = f;
  }
 };
 
 class AnnoyIndexEuclidean : public AnnoyIndex {
  public:
  AnnoyIndexEuclidean(int f) {
- ptr = new ::AnnoyIndex<int32_t, float, ::Euclidean, ::Kiss64Random>(f);
+ ptr = new ::AnnoyIndex<int32_t, float, ::Euclidean, ::Kiss64Random, AnnoyIndexSingleThreadedBuildPolicy>(f);
  this->f = f;
  }
 };
 
 class AnnoyIndexManhattan : public AnnoyIndex {
  public:
  AnnoyIndexManhattan(int f) {
- ptr = new ::AnnoyIndex<int32_t, float, ::Manhattan, ::Kiss64Random>(f);
+ ptr = new ::AnnoyIndex<int32_t, float, ::Manhattan, ::Kiss64Random, AnnoyIndexSingleThreadedBuildPolicy>(f);
  this->f = f;
  }
 };