forked from uqfoundation/multiprocess
-
Notifications
You must be signed in to change notification settings - Fork 0
/
setup.py
425 lines (356 loc) · 14 KB
/
setup.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
#!/usr/bin/env python
import re
import os
import sys
import glob
stable_version = '0.70.6'
pymajor,pyminor = sys.version_info[:2]
pkgdir = 'py%s.%s' % (pymajor,pyminor)
if sys.version_info < (2, 5):
raise ValueError('Versions of Python before 2.5 are not supported')
elif sys.version_info >= (2, 6):
pkgname = 'multiprocess'
else: # (2, 5)
pkgname = 'processing' #XXX: oddity, due to lazyness at the moment
# if sys.version is higher than explicitly supported, try the latest version
ver = float(pkgdir[2:])
HERE = os.path.dirname(os.path.abspath(__file__))
while not os.path.exists(os.path.join(HERE,'py%s' % ver)):
ver -= 0.1
if 'py%s' % ver != pkgdir:
msg = 'Warning: Python %s is not currently supported, reverting to %s'
print(msg % (pkgdir[2:],ver))
pkgdir = 'py%s' % ver
srcdir = '%s/Modules/_%s' % (pkgdir, pkgname)
libdir = '%s/%s' % (pkgdir, pkgname)
try:
from setuptools import setup, Extension, find_packages
has_setuptools = True
except ImportError:
from distutils.core import setup, Extension # noqa
find_packages = lambda **kwds: [pkgname, pkgname+'.dummy']
has_setuptools = False
from distutils import sysconfig
from distutils.errors import CCompilerError, DistutilsExecError, \
DistutilsPlatformError
ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError)
if sys.platform == 'win32' and sys.version_info >= (2, 6):
# distutils.msvc9compiler can raise IOError if the compiler is missing
ext_errors += (IOError, )
is_jython = sys.platform.startswith('java')
is_pypy = hasattr(sys, 'pypy_version_info')
is_py3k = sys.version_info[0] == 3
BUILD_WARNING = """
-----------------------------------------------------------------------
WARNING: The C extensions could not be compiled
-----------------------------------------------------------------------
Maybe you do not have a C compiler installed on this system?
The reason was:
%s
This is just a warning as most of the functionality will work even
without the updated C extension. It will simply fallback to the
built-in _multiprocessing module. Most notably you will not be able to use
FORCE_EXECV on POSIX systems. If this is a problem for you then please
install a C compiler or fix the error(s) above.
-----------------------------------------------------------------------
"""
# -*- extra config (setuptools) -*-
if has_setuptools:
extras = dict(install_requires=['dill>=0.2.7.1'])
else:
extras = dict()
# -*- Distribution Meta -*-
here = os.path.abspath(os.path.dirname(__file__))
meta_fh = open(os.path.join(here, '%s/__init__.py' % libdir))
try:
meta = {}
for line in meta_fh:
if line.startswith('__version__'):
version = line.split()[-1].strip("'").strip('"')
break
meta['version'] = version
finally:
meta_fh.close()
#
# Macros and libraries
#
# The `macros` dict determines the macros that will be defined when
# the C extension is compiled. Each value should be either 0 or 1.
# (An undefined macro is assumed to have value 0.) `macros` is only
# used on Unix platforms.
#
# The `libraries` dict determines the libraries to which the C
# extension will be linked. This should probably be either `['rt']`
# if you need `librt` or else `[]`.
#
# Meaning of macros
#
# HAVE_SEM_OPEN
# Set this to 1 if you have `sem_open()`. This enables the use of
# posix named semaphores which are necessary for the
# implementation of the synchronization primitives on Unix. If
# set to 0 then the only way to create synchronization primitives
# will be via a manager (e.g. "m = Manager(); lock = m.Lock()").
#
# HAVE_SEM_TIMEDWAIT
# Set this to 1 if you have `sem_timedwait()`. Otherwise polling
# will be necessary when waiting on a semaphore using a timeout.
#
# HAVE_FD_TRANSFER
# Set this to 1 to compile functions for transferring file
# descriptors between processes over an AF_UNIX socket using a
# control message with type SCM_RIGHTS. On Unix the pickling of
# of socket and connection objects depends on this feature.
#
# If you get errors about missing CMSG_* macros then you should
# set this to 0.
#
# HAVE_BROKEN_SEM_GETVALUE
# Set to 1 if `sem_getvalue()` does not work or is unavailable.
# On Mac OSX it seems to return -1 with message "[Errno 78]
# Function not implemented".
#
# HAVE_BROKEN_SEM_UNLINK
# Set to 1 if `sem_unlink()` is unnecessary. For some reason this
# seems to be the case on Cygwin where `sem_unlink()` is missing
# from semaphore.h.
#
if sys.platform == 'win32': # Windows
macros = dict()
libraries = ['ws2_32']
elif sys.platform.startswith('darwin'): # Mac OSX
macros = dict(
HAVE_SEM_OPEN=1,
HAVE_SEM_TIMEDWAIT=0,
HAVE_FD_TRANSFER=1,
HAVE_BROKEN_SEM_GETVALUE=1
)
libraries = []
elif sys.platform.startswith('cygwin'): # Cygwin
macros = dict(
HAVE_SEM_OPEN=1,
HAVE_SEM_TIMEDWAIT=1,
HAVE_FD_TRANSFER=0,
HAVE_BROKEN_SEM_UNLINK=1
)
libraries = []
elif sys.platform in ('freebsd4', 'freebsd5', 'freebsd6'):
# FreeBSD's P1003.1b semaphore support is very experimental
# and has many known problems. (as of June 2008)
macros = dict( # FreeBSD 4-6
HAVE_SEM_OPEN=0,
HAVE_SEM_TIMEDWAIT=0,
HAVE_FD_TRANSFER=1,
)
libraries = []
elif re.match('^(gnukfreebsd(8|9|10|11)|freebsd(7|8|9|10))', sys.platform):
macros = dict( # FreeBSD 7+ and GNU/kFreeBSD 8+
HAVE_SEM_OPEN=bool(
sysconfig.get_config_var('HAVE_SEM_OPEN') and not
bool(sysconfig.get_config_var('POSIX_SEMAPHORES_NOT_ENABLED'))
),
HAVE_SEM_TIMEDWAIT=1,
HAVE_FD_TRANSFER=1,
)
libraries = []
elif sys.platform.startswith('openbsd'):
macros = dict( # OpenBSD
HAVE_SEM_OPEN=0, # Not implemented
HAVE_SEM_TIMEDWAIT=0,
HAVE_FD_TRANSFER=1,
)
libraries = []
else: # Linux and other unices
macros = dict(
HAVE_SEM_OPEN=1,
HAVE_SEM_TIMEDWAIT=1,
HAVE_FD_TRANSFER=1,
)
libraries = ['rt']
if sys.platform == 'win32':
multiprocessing_srcs = [
'%s/%s.c' % (srcdir, pkgname),
'%s/semaphore.c' % srcdir,
]
if sys.version_info < (3, 3):
multiprocessing_srcs += [
'%s/pipe_connection.c' % srcdir,
'%s/socket_connection.c' % srcdir,
'%s/win32_functions.c' % srcdir,
]
else:
multiprocessing_srcs = [ '%s/%s.c' % (srcdir, pkgname) ]
if sys.version_info < (3, 3):
multiprocessing_srcs.append('%s/socket_connection.c' % srcdir)
if macros.get('HAVE_SEM_OPEN', False):
multiprocessing_srcs.append('%s/semaphore.c' % srcdir)
long_description = \
'''-----------------------------------------------------------------
multiprocess: better multiprocessing and multithreading in python
-----------------------------------------------------------------
About Multiprocess
====================
multiprocess is a fork of multiprocessing, and is developed as part of pathos:
https://github.com/uqfoundation/pathos
`Multiprocessing` is a package for the Python language which supports the
spawning of processes using the API of the standard library's
`threading` module. `multiprocessing` has been distributed in the standard
library since python 2.6.
Features:
- Objects can be transferred between processes using pipes or multi-producer/multi-consumer queues.
- Objects can be shared between processes using a server process or (for simple data) shared memory.
- Equivalents of all the synchronization primitives in `threading` are available.
- A `Pool` class makes it easy to submit tasks to a pool of worker processes.
Pathos is a python framework for heterogeneous computing.
Pathos is in active development, so any user feedback, bug reports, comments,
or suggestions are highly appreciated. A list of known issues is maintained at http://trac.mystic.cacr.caltech.edu/project/pathos/query.html, with a public ticket list at https://github.com/uqfoundation/pathos/issues.
NOTE: A C compiler is required to build the included extension module. For python 3.3 and above, a C compiler is suggested, but not required.
Major Changes
==============
- enhanced serialization, using dill
Current Release
===============
This version is multiprocess-%(relver)s (a fork of multiprocessing-0.70a1).
The latest released pathos fork of multiprocessing is available from::
https://pypi.org/project/multiprocess
Multiprocessing is distributed under a BSD license.
Development Version
===================
You can get the latest development version with all the shiny new features at::
https://github.com/uqfoundation
If you have a new contribution, please submit a pull request.
Installation
============
Multiprocess is packaged to install from source, so you must
download the tarball, unzip, and run the installer::
[download]
$ tar -xvzf multiprocess-%(relver)s.tgz
$ cd multiprocess-%(relver)s
$ python setup.py build
$ python setup.py install
You will be warned of any missing dependencies and/or settings
after you run the "build" step above.
Alternately, multiprocess can be installed with pip or easy_install::
$ pip install multiprocess
NOTE: A C compiler is required to build the included extension module. For python 3.3 and above, a C compiler is suggested, but not required.
Requirements
============
Multiprocess requires::
- python2, version >= 2.5 *or* python3, version >= 3.1
Optional requirements::
- setuptools, version >= 0.6
- dill, version >= 0.2.7.1
More Information
================
Probably the best way to get started is to look at the examples that are
provided within multiprocess. See the examples directory for a set of
example scripts. Please feel free to submit a ticket on github, or ask
a question on stackoverflow (@Mike McKerns).
Pathos is an active research tool. There are a growing number of publications
and presentations that discuss real-world examples and new features of pathos
in greater detail than presented in the user's guide. If you would like to
share how you use pathos in your work, please post a link or send an email
(to mmckerns at uqfoundation dot org).
Citation
========
If you use pathos to do research that leads to publication, we ask that you
acknowledge use of pathos by citing the following in your publication::
M.M. McKerns, L. Strand, T. Sullivan, A. Fang, M.A.G. Aivazis,
"Building a framework for predictive science", Proceedings of
the 10th Python in Science Conference, 2011;
http://arxiv.org/pdf/1202.1056
Michael McKerns and Michael Aivazis,
"pathos: a framework for heterogeneous computing", 2010- ;
http://trac.mystic.cacr.caltech.edu/project/pathos
Please see http://trac.mystic.cacr.caltech.edu/project/pathos or
http://arxiv.org/pdf/1202.1056 for further information.
''' % {'relver': stable_version}
#long_description = open(os.path.join(HERE, 'README.md')).read()
#long_description += """
#
#===========
#Changes
#===========
#
#"""
#long_description += open(os.path.join(HERE, 'CHANGES.txt')).read()
#if not is_py3k:
# long_description = long_description.encode('ascii', 'replace')
# -*- Installation Requires -*-
py_version = sys.version_info
is_jython = sys.platform.startswith('java')
is_pypy = hasattr(sys, 'pypy_version_info')
#def strip_comments(l):
# return l.split('#', 1)[0].strip()
#
#def reqs(f):
# return list(filter(None, [strip_comments(l) for l in open(
# os.path.join(os.getcwd(), 'requirements', f)).readlines()]))
#
#if py_version[0] == 3:
# tests_require = reqs('test3.txt')
#else:
# tests_require = reqs('test.txt')
def _is_build_command(argv=sys.argv, cmds=('install', 'build', 'bdist')):
for arg in argv:
if arg.startswith(cmds):
return arg
def run_setup(with_extensions=True):
extensions = []
if with_extensions:
extensions = [
Extension(
'_%s' % pkgname,
sources=multiprocessing_srcs,
define_macros=list(macros.items()),
libraries=libraries,
include_dirs=[srcdir],
depends=glob.glob('%s/*.h' % srcdir) + ['setup.py'],
),
]
packages = find_packages(
where=pkgdir,
exclude=['ez_setup', 'examples', 'doc', 'tests*', ],
)
config = dict(
name='multiprocess',
version=meta['version'],
description=('better multiprocessing and multithreading in python'),
long_description=long_description,
packages=packages,
ext_modules=extensions,
author='Mike McKerns',
maintainer='Mike McKerns',
url='https://pypi.org/project/multiprocess',
download_url='https://github.com/uqfoundation/multiprocess/releases/download/multiprocess-%s/multiprocess-%s.tar.gz' % (stable_version, stable_version),
zip_safe=False,
license='BSD',
package_dir={'' : pkgdir}, #XXX: {pkgname+'.tests' : 'tests'} ?
# tests_require=tests_require,
# test_suite='nose.collector',
classifiers=[
'Development Status :: 5 - Production/Stable',
'Intended Audience :: Developers',
'Intended Audience :: Science/Research',
'License :: OSI Approved :: BSD License',
'Programming Language :: Python :: 2',
'Programming Language :: Python :: 3',
'Topic :: Scientific/Engineering',
'Topic :: Software Development',
],
**extras
)
setup(**config)
try:
run_setup(not (is_jython or is_pypy))# or is_py3k))
except BaseException:
if _is_build_command(sys.argv):
import traceback
msg = BUILD_WARNING % '\n'.join(traceback.format_stack())
if not is_py3k:
exec('print >> sys.stderr, msg')
else:
exec('print(msg, file=sys.stderr)')
run_setup(False)
else:
raise