Merge pull request #151 from RobertLuptonTheGood/tickets/DM-42570

DM-42570: Allow os.makedirs to work if the directory already exists

.github/workflows/ci.yaml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
+        python-version: [3.7, 3.8, 3.9, "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v3

.gitignore

@@ -30,6 +30,6 @@ bin/setups.zsh
 TAGS
 autom4te.cache
 eups_impl.py
-eups
+bin/eups
 bin/eups_setup
 bin/eups_setup_impl.py

Makefile.in

@@ -127,7 +127,7 @@ configure : configure.ac
 tests : test
 test :
 	@ echo "Running tests"
-	"$(EUPS_PYTHON)" tests/testAll.py
+	"$(EUPS_PYTHON)" -Wd tests/testAll.py
 	@ echo "Running server tests (requires network)"
 	"$(EUPS_PYTHON)" tests/testServerAll.py
 clean :

bin/eups.in

@@ -2,7 +2,6 @@
 #
 # The main eups programme
 #
-from __future__ import print_function
 import sys
 import os
 

bin/eups_setup.in

@@ -2,7 +2,6 @@
 #
 # The EUPS setup programme
 #
-from __future__ import print_function
 import sys, os, re
 
 sys.argv[0] = "eups"

bin/mksetup

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 #
-from __future__ import print_function
 import os, re, sys
 
 sys.argv.pop(0)                         # remove script name
@@ -27,7 +26,6 @@ setup = os.path.join('$EUPS_DIR',"bin","eups_setup");
 
 # Keep only one copy of each directory in sys.argv[1], but make sure that sys.argv[2] is present
 unique_path="""
-from __future__ import print_function
 import sys
 pp = []
 for d in sys.argv[1].split(":"):

callbacks/distribInstallPostHook.py

@@ -5,10 +5,7 @@
 be critical (e.g. to patch up os/x's SIP protection)
 """
 
-from __future__ import absolute_import, print_function
-
 from distutils.spawn import find_executable
-import io
 import locale
 import mmap
 import re
@@ -98,7 +95,7 @@ def fix_shebang(path, build_python, verbose=0):
         return
 
     prefenc = locale.getpreferredencoding()
-    with io.open(path, encoding=prefenc, mode='r+') as fi:
+    with open(path, encoding=prefenc, mode='r+') as fi:
         try:
             data = fi.read(100)
             fi.seek(0)
@@ -139,7 +136,7 @@ def fix_shebang(path, build_python, verbose=0):
 
     # save original file mode
     mode = os.stat(path).st_mode
-    with io.open(path, 'w', encoding=encoding) as fo:
+    with open(path, 'w', encoding=encoding) as fo:
         fo.write(new_data.decode(encoding))
     # restore file mode
     os.chmod(path, mode)

python/eups/Eups.py

@@ -1,7 +1,6 @@
 """
 The Eups class
 """
-from __future__ import absolute_import, print_function
 import glob
 import re
 import os
@@ -23,7 +22,7 @@
 from .utils      import cmp_or_key, xrange, cmp
 from . import hooks
 
-class Eups(object):
+class Eups:
     """
     An application interface to EUPS functionality.
 
@@ -45,7 +44,7 @@ class Eups(object):
     #  managed software stack
     ups_db = "ups_db"
 
-    # staticmethod;  would use a decorator if we knew we had a new enough python
+    @staticmethod
     def setEupsPath(path=None, dbz=None):
         if not path:
             path = os.environ.get("EUPS_PATH", [])
@@ -71,9 +70,7 @@ def setEupsPath(path=None, dbz=None):
         os.environ["EUPS_PATH"] = ":".join(eups_path)
         return eups_path
 
-    setEupsPath = staticmethod(setEupsPath)
-
-    # staticmethod;  would use a decorator if we knew we had a new enough python
+    @staticmethod
     def _processDefaultTags(opts):
         """Handle any default tags defined as hooks"""
         if opts.tag in (["None"], [""]):
@@ -104,8 +101,6 @@ def _processDefaultTags(opts):
                 if not (hasattr(opts, "unsetup") and opts.unsetup):
                     print(msg, "; ".join(tagMsg), file=utils.stdinfo)
 
-    _processDefaultTags = staticmethod(_processDefaultTags)
-
     def __init__(self, flavor=None, path=None, dbz=None, root=None, readCache=True,
                  shell=None, verbose=0, quiet=0,
                  noaction=False, force=False, ignore_versions=False,
@@ -513,14 +508,11 @@ def _userStackCache(self, eupsPathDir):
     def _makeUserCacheDir(self, eupsPathDir):
         cachedir = self._userStackCache(eupsPathDir)
         if cachedir and not os.path.exists(cachedir):
-            os.makedirs(cachedir)
+            os.makedirs(cachedir, exist_ok=True)
             try:
-                readme = open(os.path.join(cachedir,"README"), "w")
-                try:
+                with open(os.path.join(cachedir,"README"), "w") as readme:
                     print("User cache directory for", eupsPathDir, file=readme)
-                finally:
-                    readme.close()
-            except:
+            except Exception:
                 pass
         return cachedir
 
@@ -1220,7 +1212,7 @@ def _findTaggedProductFromFile(self, name, fileName, eupsPathDirs=None, flavor=N
 
         try:
             fd = open(fileName, "r")
-        except IOError:
+        except OSError:
             raise TagNotRecognized(str(fileName))
 
         version = None
@@ -1459,7 +1451,7 @@ def getUpsDB(self, eupsPathDir, create=False, noRaise=False):
 
         if not os.path.isdir(upsDB):
             if create:
-                os.makedirs(upsDB)
+                os.makedirs(upsDB, exist_ok=True)
 
         if not os.path.isdir(upsDB):
             if noRaise:
@@ -2414,7 +2406,7 @@ def declare(self, productName, versionName, productDir=None, eupsPathDir=None, t
         if self.isUserTag(tag):
             ups_db = self.getUpsDB(self.userDataDir)
             if not os.path.isdir(ups_db):
-                os.makedirs(ups_db)
+                os.makedirs(ups_db, exist_ok=True)
             eupsPathDir = self.userDataDir
 
         if not eupsPathDir:
@@ -2469,7 +2461,7 @@ def cleanup(*args):
                 umask = os.umask(0o02); os.umask(umask)
                 try:
                     perms = os.fstat(tfd.fileno()).st_mode
-                except:
+                except Exception:
                     perms = 0
 
                 perms |= (0o666 & ~umask) # set read/write for everyone (modulu umask)
@@ -2580,8 +2572,10 @@ def cleanup(*args):
                     if not os.path.exists(pathOut):
                         differences += ["Adding %s" % pathOut]
                     else:
-                        crcOld = zlib.crc32(open(fileNameIn, "rb").read())
-                        crcNew = zlib.crc32(open(pathOut, "rb").read())
+                        with open(fileNameIn, "rb") as fd:
+                            crcOld = zlib.crc32(fd.read())
+                        with open(pathOut, "rb") as fd:
+                            crcNew = zlib.crc32(fd.read())
 
                         if crcOld != crcNew:
                             differences += ["%s's CRC32 changed" % pathOut]
@@ -2719,7 +2713,7 @@ def cleanup(*args):
                 else:
                     if self.verbose > 1:
                         print("mkdir -p %s" % (dirName), file=utils.stdinfo)
-                    os.makedirs(dirName)
+                    os.makedirs(dirName, exist_ok=True)
 
             if not (os.path.exists(pathOut) and os.path.samefile(fileNameIn, pathOut)):
                 if self.noaction:
@@ -3272,7 +3266,7 @@ def remove(self, productName, versionName, recursive=False, checkRecursive=False
             if interactive:
                 yn = default_yn
                 while yn != "!":
-                    yn = raw_input("Remove %s %s: (ynq!) [%s] " % (product.name, product.version, default_yn))
+                    yn = input("Remove %s %s: (ynq!) [%s] " % (product.name, product.version, default_yn))
 
                     if yn == "":
                         yn = default_yn
@@ -3724,7 +3718,7 @@ def selectVRO(self, tag=None, productDir=None, versionName=None, dbz=None, inexa
 
         self._vro = self.preferredTags  # we should get rid of preferredTags; it's left over from Ray Plante
 
-    # staticmethod;  would use a decorator if we knew we had a new enough python
+    @staticmethod
     def __mergeWarnings(vro):
         """Replace consecutive sequences of warn:X by warn:min (otherwise we may get repeated warnings);
 such sequences can be generated while rewriting the VRO"""
@@ -3756,8 +3750,6 @@ def __mergeWarnings(vro):
 
         return cleanedVro
 
-    __mergeWarnings = staticmethod(__mergeWarnings)
-
     def makeVroExact(self):
         """Modify the VRO to support setup --exact even if the table files don't have an
            if(type == exact) { block }"""
@@ -3822,7 +3814,7 @@ def getVRO(self):
 _ClassEups = Eups                       # so we can say, "isinstance(Eups, _ClassEups)"
 
 
-class _TagSet(object):
+class _TagSet:
     def __init__(self, eups, tags):
         self.eups = eups
         self.lu = {}

python/eups/Product.py

@@ -1,11 +1,8 @@
 # from Table import *
-from __future__ import absolute_import, print_function
 import os
 import re
-try:
-    from configparser import ConfigParser
-except ImportError:
-    from ConfigParser import ConfigParser
+from configparser import ConfigParser
+
 from . import table as mod_table
 from . import utils
 from .exceptions import ProductNotFound, TableFileNotFound
@@ -16,7 +13,7 @@
             "UPS_DIR":   re.compile(r"^\$UPS_DIR\b"),
             "UPS_DB":    re.compile(r"^\$UPS_DB\b")     }
 
-class Product(object):
+class Product:
     """
     a description of a Product as stored in the stack database.
 
@@ -464,7 +461,7 @@ def getConfig(self, section="DEFAULT", option=None, getType=None):
         return config
 
     # this replaces from initFromDirectory()
-    # @staticmethod   # requires python 2.4
+    @staticmethod
     def createLocal(productName, productDir, flavor=None, checkForTable=True, tablefile=None):
         localDir = Product._decode_dir(productDir)
         if localDir:
@@ -482,8 +479,6 @@ def createLocal(productName, productDir, flavor=None, checkForTable=True, tablef
             if not os.path.exists(out.tablefile):
                 out.tablefile = "none"
         return out
-    createLocal = staticmethod(createLocal)  # should work as of python 2.2
-
 
     def envarDirName(self):
         return utils.dirEnvNameFor(self.name)

python/eups/Uses.py

@@ -8,13 +8,13 @@
 #
 # Cache for the Uses tree
 #
-class Props(object):
+class Props:
     def __init__(self, version, optional, depth):
         self.version = version
         self.optional = optional
         self.depth = depth
 
-class Uses(object):
+class Uses:
     """
     a class for tracking product dependencies.  Typically an instance of
     this class is created via a call to Eups.uses().  This class is used

python/eups/VersionCompare.py

@@ -2,7 +2,7 @@
 
 from .utils import cmp
 
-class VersionCompare(object):
+class VersionCompare:
     """
     A comparison function class that compares two product versions.
     """

python/eups/VersionParser.py

@@ -3,7 +3,7 @@
 import os
 import re
 
-class VersionParser(object):
+class VersionParser:
     """Evaluate a logical expression, returning a Bool.  The grammar is:
 
         expr : term

python/eups/__init__.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 from .exceptions import *
 from .tags       import Tags, Tag, TagNotRecognized
 from .Product    import Product

python/eups/app.py

@@ -2,14 +2,10 @@
 common high-level EUPS functions appropriate for calling from an application.
 """
 
-from __future__ import absolute_import, print_function
 import fnmatch
 import re
 import os
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
+import pickle
 from .distrib import builder
 from .Eups           import Eups
 from .exceptions     import ProductNotFound
@@ -19,7 +15,7 @@
 from .stack          import ProductStack, persistVersionName as cacheVersion
 from . import utils, table, hooks
 from .exceptions import EupsException
-from .utils import cmp_or_key, cmp
+from .utils import cmp_or_key
 
 def printProducts(ostrm, productName=None, versionName=None, eupsenv=None,
                   tags=None, setup=False, tablefile=False, directory=False,