diff --git a/src/backend_model.cc b/src/backend_model.cc
index 455098bde..f6eb27e6a 100644
--- a/src/backend_model.cc
+++ b/src/backend_model.cc
@@ -75,10 +75,19 @@ TritonModel::Create(
   }
 
   // Localize the content of the model repository corresponding to
-  // 'model_path'. This model holds a handle to the localized content
-  // so that it persists as long as the model is loaded.
+  // 'model_path'. This model holds a handle to
+  // the localized content so that it persists as long as the model is loaded.
   std::shared_ptr<LocalizedPath> localized_model_dir;
-  RETURN_IF_ERROR(LocalizePath(model_path, &localized_model_dir));
+  RETURN_IF_ERROR(
+      LocalizePath(model_path, false /*recursive*/, &localized_model_dir));
+
+  const auto version_path = JoinPath({model_path, std::to_string(version)});
+  std::shared_ptr<LocalizedPath> localized_version_dir;
+  RETURN_IF_ERROR(LocalizePath(
+      version_path, true /*recursive*/, localized_model_dir->Path(),
+      &localized_version_dir));
+
+  localized_model_dir->other_localized_path.push_back(localized_version_dir);
 
   // Localize paths in backend model config
   // [FIXME] Remove once a more permanent solution is implemented (DLIS-4211)
@@ -110,12 +119,11 @@ TritonModel::Create(
   // Get the path to the backend shared library. Search path is
   // version directory, model directory, global backend directory.
   const auto localized_model_path = localized_model_dir->Path();
-  const auto version_path =
-      JoinPath({localized_model_path, std::to_string(version)});
+  const auto localized_version_path = localized_version_dir->Path();
   const std::string global_path =
       JoinPath({backend_dir, specialized_backend_name});
   const std::vector<std::string> search_paths = {
-      version_path, localized_model_path, global_path};
+      localized_version_path, localized_model_path, global_path};
 
   std::string backend_libdir;
   std::string backend_libpath;
diff --git a/src/filesystem/api.cc b/src/filesystem/api.cc
index 4f3883ccc..477347619 100644
--- a/src/filesystem/api.cc
+++ b/src/filesystem/api.cc
@@ -558,11 +558,23 @@ ReadTextProto(const std::string& path, google::protobuf::Message* msg)
 }
 
 Status
-LocalizePath(const std::string& path, std::shared_ptr<LocalizedPath>* localized)
+LocalizePath(
+    const std::string& path, const bool recursive,
+    std::shared_ptr<LocalizedPath>* localized)
 {
   std::shared_ptr<FileSystem> fs;
   RETURN_IF_ERROR(fsm_.GetFileSystem(path, fs));
-  return fs->LocalizePath(path, localized);
+  return fs->LocalizePath(path, recursive, "", localized);
+}
+
+Status
+LocalizePath(
+    const std::string& path, const bool recursive, const std::string& mount_dir,
+    std::shared_ptr<LocalizedPath>* localized)
+{
+  std::shared_ptr<FileSystem> fs;
+  RETURN_IF_ERROR(fsm_.GetFileSystem(path, fs));
+  return fs->LocalizePath(path, recursive, mount_dir, localized);
 }
 
 Status
@@ -607,11 +619,12 @@ ReadBinaryProto(const std::string& path, google::protobuf::MessageLite* msg)
 }
 
 Status
-MakeDirectory(const std::string& dir, const bool recursive)
+MakeDirectory(
+    const std::string& dir, const bool recursive, const bool allow_dir_exist)
 {
   std::shared_ptr<FileSystem> fs;
   RETURN_IF_ERROR(fsm_.GetFileSystem(dir, fs));
-  return fs->MakeDirectory(dir, recursive);
+  return fs->MakeDirectory(dir, recursive, allow_dir_exist);
 }
 
 Status
diff --git a/src/filesystem/api.h b/src/filesystem/api.h
index e7711ce8e..040889efa 100644
--- a/src/filesystem/api.h
+++ b/src/filesystem/api.h
@@ -145,11 +145,27 @@ Status ReadTextFile(const std::string& path, std::string* contents);
 
 /// Create an object representing a local copy of a path.
 /// \param path The path of the directory or file.
+/// \param recursive If true, will fetch all sub-directories in
+/// the provided path.
 /// \param localized Returns the LocalizedPath object
 /// representing the local copy of the path.
 /// \return Error status
 Status LocalizePath(
-    const std::string& path, std::shared_ptr<LocalizedPath>* localized);
+    const std::string& path, const bool recursive,
+    std::shared_ptr<LocalizedPath>* localized);
+
+/// Create an object representing a local copy of a path.
+/// \param path The path of the directory or file.
+/// \param recursive If true, will fetch all sub-directories in
+/// the provided path.
+/// \param mount_dir If not empty, will use provided local directory
+/// for localization.
+/// \param localized Returns the LocalizedPath object
+/// representing the local copy of the path.
+/// \return Error status
+Status LocalizePath(
+    const std::string& path, const bool recursive, const std::string& mount_dir,
+    std::shared_ptr<LocalizedPath>* localized);
 
 /// Write a string to a file.
 /// \param path The path of the file.
@@ -189,8 +205,14 @@ Status ReadBinaryProto(
 /// \param dir The path to the directory.
 /// \param recursive Whether the parent directories will be created
 /// if not exist.
+/// \param allow_dir_exist Controls the behavior on condition,
+/// when `dir` already exists. If true and `dir` exists, returns success.
+/// If false and `dir` exists, fails with `Status::Code::INTERNAL`
+/// and reports errno EEXIST. Default value: false.
 /// \return Error status if the directory can't be created
-Status MakeDirectory(const std::string& dir, const bool recursive);
+Status MakeDirectory(
+    const std::string& dir, const bool recursive,
+    const bool allow_dir_exist = false);
 
 /// Create a temporary directory of the specified filesystem type.
 /// \param type The type of the filesystem.
diff --git a/src/filesystem/implementations/as.h b/src/filesystem/implementations/as.h
index 13eb80d99..ac055c14d 100644
--- a/src/filesystem/implementations/as.h
+++ b/src/filesystem/implementations/as.h
@@ -86,14 +86,17 @@ class ASFileSystem : public FileSystem {
       const std::string& path, std::set<std::string>* files) override;
   Status ReadTextFile(const std::string& path, std::string* contents) override;
   Status LocalizePath(
-      const std::string& path,
+      const std::string& path, const bool recursive,
+      const std::string& mount_dir,
       std::shared_ptr<LocalizedPath>* localized) override;
   Status WriteTextFile(
       const std::string& path, const std::string& contents) override;
   Status WriteBinaryFile(
       const std::string& path, const char* contents,
       const size_t content_len) override;
-  Status MakeDirectory(const std::string& dir, const bool recursive) override;
+  Status MakeDirectory(
+      const std::string& dir, const bool recursive,
+      const bool allow_dir_exist) override;
   Status MakeTemporaryDirectory(std::string* temp_dir) override;
   Status DeletePath(const std::string& path) override;
 
@@ -113,7 +116,8 @@ class ASFileSystem : public FileSystem {
 
   Status DownloadFolder(
       const std::string& container, const std::string& path,
-      const std::string& dest);
+      const std::string& dest, const bool recursive,
+      const bool allow_dir_exist);
 
   std::shared_ptr<asb::BlobServiceClient> client_;
   re2::RE2 as_regex_;
@@ -389,7 +393,7 @@ ASFileSystem::FileExists(const std::string& path, bool* exists)
 Status
 ASFileSystem::DownloadFolder(
     const std::string& container, const std::string& path,
-    const std::string& dest)
+    const std::string& dest, const bool recursive, const bool allow_dir_exist)
 {
   auto container_client = client_->GetBlobContainerClient(container);
   auto func = [&](const std::vector<asb::Models::BlobItem>& blobs,
@@ -405,17 +409,14 @@ ASFileSystem::DownloadFolder(
             "Failed to download file at " + blob_item.Name + ":" + ex.what());
       }
     }
-    for (const auto& directory_item : blob_prefixes) {
-      const auto& local_path = JoinPath({dest, BaseName(directory_item)});
-      int status = mkdir(
-          const_cast<char*>(local_path.c_str()), S_IRUSR | S_IWUSR | S_IXUSR);
-      if (status == -1) {
-        return Status(
-            Status::Code::INTERNAL,
-            "Failed to create local folder: " + local_path +
-                ", errno:" + strerror(errno));
+    if (recursive) {
+      for (const auto& directory_item : blob_prefixes) {
+        const auto& local_path = JoinPath({dest, BaseName(directory_item)});
+        RETURN_IF_ERROR(triton::core::MakeDirectory(
+            local_path, recursive, allow_dir_exist));
+        RETURN_IF_ERROR(DownloadFolder(
+            container, directory_item, local_path, recursive, allow_dir_exist));
       }
-      RETURN_IF_ERROR(DownloadFolder(container, directory_item, local_path));
     }
     return Status::Success;
   };
@@ -424,7 +425,8 @@ ASFileSystem::DownloadFolder(
 
 Status
 ASFileSystem::LocalizePath(
-    const std::string& path, std::shared_ptr<LocalizedPath>* localized)
+    const std::string& path, const bool recursive, const std::string& mount_dir,
+    std::shared_ptr<LocalizedPath>* localized)
 {
   bool exists;
   RETURN_IF_ERROR(FileExists(path, &exists));
@@ -441,21 +443,31 @@ ASFileSystem::LocalizePath(
         "AS file localization not yet implemented " + path);
   }
 
-  std::string folder_template = "/tmp/folderXXXXXX";
-  char* tmp_folder = mkdtemp(const_cast<char*>(folder_template.c_str()));
-  if (tmp_folder == nullptr) {
-    return Status(
-        Status::Code::INTERNAL,
-        "Failed to create local temp folder: " + folder_template +
-            ", errno:" + strerror(errno));
+  // Create a local directory for azure model store.
+  // If `mount_dir` or ENV variable are not set,
+  // creates a temporary directory under `/tmp` with the format: "folderXXXXXX".
+  // Otherwise, will create a folder under specified directory with the name
+  // indicated in path (i.e. everything after the last encounter of `/`).
+  const char* env_mount_dir = std::getenv("TRITON_AZURE_MOUNT_DIRECTORY");
+  std::string tmp_folder;
+  if (mount_dir.empty() && env_mount_dir == nullptr) {
+    RETURN_IF_ERROR(triton::core::MakeTemporaryDirectory(
+        FileSystemType::LOCAL, &tmp_folder));
+  } else {
+    tmp_folder = mount_dir.empty() ? std::string(env_mount_dir) : mount_dir;
+    tmp_folder =
+        JoinPath({tmp_folder, path.substr(path.find_last_of('/') + 1)});
+    RETURN_IF_ERROR(triton::core::MakeDirectory(
+        tmp_folder, true /*recursive*/, true /*allow_dir_exist*/));
   }
-  localized->reset(new LocalizedPath(path, tmp_folder));
 
-  std::string dest(folder_template);
+  localized->reset(new LocalizedPath(path, tmp_folder));
 
+  std::string dest(tmp_folder);
   std::string container, blob;
   RETURN_IF_ERROR(ParsePath(path, &container, &blob));
-  return DownloadFolder(container, blob, dest);
+  return DownloadFolder(
+      container, blob, dest, recursive, true /*allow_dir_exist*/);
 }
 
 Status
@@ -487,7 +499,8 @@ ASFileSystem::WriteBinaryFile(
 }
 
 Status
-ASFileSystem::MakeDirectory(const std::string& dir, const bool recursive)
+ASFileSystem::MakeDirectory(
+    const std::string& dir, const bool recursive, const bool allow_dir_exist)
 {
   return Status(
       Status::Code::UNSUPPORTED,
diff --git a/src/filesystem/implementations/common.h b/src/filesystem/implementations/common.h
index 5f45444dd..8deceba10 100644
--- a/src/filesystem/implementations/common.h
+++ b/src/filesystem/implementations/common.h
@@ -83,14 +83,17 @@ class FileSystem {
   virtual Status ReadTextFile(
       const std::string& path, std::string* contents) = 0;
   virtual Status LocalizePath(
-      const std::string& path, std::shared_ptr<LocalizedPath>* localized) = 0;
+      const std::string& path, const bool recursive,
+      const std::string& mount_dir,
+      std::shared_ptr<LocalizedPath>* localized) = 0;
   virtual Status WriteTextFile(
       const std::string& path, const std::string& contents) = 0;
   virtual Status WriteBinaryFile(
       const std::string& path, const char* contents,
       const size_t content_len) = 0;
   virtual Status MakeDirectory(
-      const std::string& dir, const bool recursive) = 0;
+      const std::string& dir, const bool recursive,
+      const bool allow_dir_exist) = 0;
   virtual Status MakeTemporaryDirectory(std::string* temp_dir) = 0;
   virtual Status DeletePath(const std::string& path) = 0;
 };
diff --git a/src/filesystem/implementations/gcs.h b/src/filesystem/implementations/gcs.h
index 56c3d8d34..c46157b00 100644
--- a/src/filesystem/implementations/gcs.h
+++ b/src/filesystem/implementations/gcs.h
@@ -75,14 +75,17 @@ class GCSFileSystem : public FileSystem {
       const std::string& path, std::set<std::string>* files) override;
   Status ReadTextFile(const std::string& path, std::string* contents) override;
   Status LocalizePath(
-      const std::string& path,
+      const std::string& path, const bool recursive,
+      const std::string& mount_dir,
       std::shared_ptr<LocalizedPath>* localized) override;
   Status WriteTextFile(
       const std::string& path, const std::string& contents) override;
   Status WriteBinaryFile(
       const std::string& path, const char* contents,
       const size_t content_len) override;
-  Status MakeDirectory(const std::string& dir, const bool recursive) override;
+  Status MakeDirectory(
+      const std::string& dir, const bool recursive,
+      const bool allow_dir_exist) override;
   Status MakeTemporaryDirectory(std::string* temp_dir) override;
   Status DeletePath(const std::string& path) override;
 
@@ -363,7 +366,8 @@ GCSFileSystem::ReadTextFile(const std::string& path, std::string* contents)
 
 Status
 GCSFileSystem::LocalizePath(
-    const std::string& path, std::shared_ptr<LocalizedPath>* localized)
+    const std::string& path, const bool recursive, const std::string& mount_dir,
+    std::shared_ptr<LocalizedPath>* localized)
 {
   bool exists;
   RETURN_IF_ERROR(FileExists(path, &exists));
@@ -380,9 +384,23 @@ GCSFileSystem::LocalizePath(
         "GCS file localization not yet implemented " + path);
   }
 
+  // Create a local directory for s3 model store.
+  // If `mount_dir` or ENV variable are not set,
+  // creates a temporary directory under `/tmp` with the format: "folderXXXXXX".
+  // Otherwise, will create a folder under specified directory with the name
+  // indicated in path (i.e. everything after the last encounter of `/`).
+  const char* env_mount_dir = std::getenv("TRITON_GCS_MOUNT_DIRECTORY");
   std::string tmp_folder;
-  RETURN_IF_ERROR(
-      triton::core::MakeTemporaryDirectory(FileSystemType::LOCAL, &tmp_folder));
+  if (mount_dir.empty() && env_mount_dir == nullptr) {
+    RETURN_IF_ERROR(triton::core::MakeTemporaryDirectory(
+        FileSystemType::LOCAL, &tmp_folder));
+  } else {
+    tmp_folder = mount_dir.empty() ? std::string(env_mount_dir) : mount_dir;
+    tmp_folder =
+        JoinPath({tmp_folder, path.substr(path.find_last_of('/') + 1)});
+    RETURN_IF_ERROR(triton::core::MakeDirectory(
+        tmp_folder, true /*recursive*/, true /*allow_dir_exist*/));
+  }
 
   localized->reset(new LocalizedPath(path, tmp_folder));
 
@@ -402,7 +420,7 @@ GCSFileSystem::LocalizePath(
       std::string local_fpath =
           JoinPath({(*localized)->Path(), gcs_removed_path});
       RETURN_IF_ERROR(IsDirectory(gcs_fpath, &is_subdir));
-      if (is_subdir) {
+      if (recursive && is_subdir) {
         // Create local mirror of sub-directories
 #ifdef _WIN32
         int status = mkdir(const_cast<char*>(local_fpath.c_str()));
@@ -425,7 +443,7 @@ GCSFileSystem::LocalizePath(
              ++itr) {
           contents.insert(JoinPath({gcs_fpath, *itr}));
         }
-      } else {
+      } else if (!is_subdir) {
         // Create local copy of file
         std::string file_bucket, file_object;
         RETURN_IF_ERROR(ParsePath(gcs_fpath, &file_bucket, &file_object));
@@ -472,7 +490,8 @@ GCSFileSystem::WriteBinaryFile(
 }
 
 Status
-GCSFileSystem::MakeDirectory(const std::string& dir, const bool recursive)
+GCSFileSystem::MakeDirectory(
+    const std::string& dir, const bool recursive, const bool allow_dir_exist)
 {
   return Status(
       Status::Code::UNSUPPORTED,
diff --git a/src/filesystem/implementations/local.h b/src/filesystem/implementations/local.h
index f6deeb5e6..d89d56ecb 100644
--- a/src/filesystem/implementations/local.h
+++ b/src/filesystem/implementations/local.h
@@ -49,14 +49,17 @@ class LocalFileSystem : public FileSystem {
       const std::string& path, std::set<std::string>* files) override;
   Status ReadTextFile(const std::string& path, std::string* contents) override;
   Status LocalizePath(
-      const std::string& path,
+      const std::string& path, const bool recursive,
+      const std::string& mount_dir,
       std::shared_ptr<LocalizedPath>* localized) override;
   Status WriteTextFile(
       const std::string& path, const std::string& contents) override;
   Status WriteBinaryFile(
       const std::string& path, const char* contents,
       const size_t content_len) override;
-  Status MakeDirectory(const std::string& dir, const bool recursive) override;
+  Status MakeDirectory(
+      const std::string& dir, const bool recursive,
+      const bool allow_dir_exist) override;
   Status MakeTemporaryDirectory(std::string* temp_dir) override;
   Status DeletePath(const std::string& path) override;
 };
@@ -204,7 +207,8 @@ LocalFileSystem::ReadTextFile(const std::string& path, std::string* contents)
 
 Status
 LocalFileSystem::LocalizePath(
-    const std::string& path, std::shared_ptr<LocalizedPath>* localized)
+    const std::string& path, const bool recursive, const std::string& mount_dir,
+    std::shared_ptr<LocalizedPath>* localized)
 {
   // For local file system we don't actually need to download the
   // directory or file. We use it in place.
@@ -246,7 +250,8 @@ LocalFileSystem::WriteBinaryFile(
 }
 
 Status
-LocalFileSystem::MakeDirectory(const std::string& dir, const bool recursive)
+LocalFileSystem::MakeDirectory(
+    const std::string& dir, const bool recursive, const bool allow_dir_exist)
 {
 #ifdef _WIN32
   if (mkdir(dir.c_str()) == -1)
@@ -254,10 +259,14 @@ LocalFileSystem::MakeDirectory(const std::string& dir, const bool recursive)
   if (mkdir(dir.c_str(), S_IRWXU) == -1)
 #endif
   {
-    // Only allow the error due to parent directory does not exist
-    // if 'recursive' is requested
+    // Return success if directory already exists and it is permitted
+    if (allow_dir_exist && errno == EEXIST) {
+      return Status::Success;
+    }
+    // In all other cases only allow the error due to parent directory
+    // does not exist, if 'recursive' is requested
     if ((errno == ENOENT) && (!dir.empty()) && recursive) {
-      RETURN_IF_ERROR(MakeDirectory(DirName(dir), recursive));
+      RETURN_IF_ERROR(MakeDirectory(DirName(dir), recursive, allow_dir_exist));
       // Retry the creation
 #ifdef _WIN32
       if (mkdir(dir.c_str()) == -1)
diff --git a/src/filesystem/implementations/s3.h b/src/filesystem/implementations/s3.h
index fe6da18c3..67862aa0c 100644
--- a/src/filesystem/implementations/s3.h
+++ b/src/filesystem/implementations/s3.h
@@ -151,14 +151,17 @@ class S3FileSystem : public FileSystem {
       const std::string& path, std::set<std::string>* files) override;
   Status ReadTextFile(const std::string& path, std::string* contents) override;
   Status LocalizePath(
-      const std::string& path,
+      const std::string& path, const bool recursive,
+      const std::string& mount_dir,
       std::shared_ptr<LocalizedPath>* localized) override;
   Status WriteTextFile(
       const std::string& path, const std::string& contents) override;
   Status WriteBinaryFile(
       const std::string& path, const char* contents,
       const size_t content_len) override;
-  Status MakeDirectory(const std::string& dir, const bool recursive) override;
+  Status MakeDirectory(
+      const std::string& dir, const bool recursive,
+      const bool allow_dir_exist) override;
   Status MakeTemporaryDirectory(std::string* temp_dir) override;
   Status DeletePath(const std::string& path) override;
 
@@ -628,7 +631,8 @@ S3FileSystem::ReadTextFile(const std::string& path, std::string* contents)
 
 Status
 S3FileSystem::LocalizePath(
-    const std::string& path, std::shared_ptr<LocalizedPath>* localized)
+    const std::string& path, const bool recursive, const std::string& mount_dir,
+    std::shared_ptr<LocalizedPath>* localized)
 {
   // Check if the directory or file exists
   bool exists;
@@ -652,10 +656,23 @@ S3FileSystem::LocalizePath(
     effective_path = path;
   }
 
-  // Create temporary directory
+  // Create a local directory for s3 model store.
+  // If `mount_dir` or ENV variable are not set,
+  // creates a temporary directory under `/tmp` with the format: "folderXXXXXX".
+  // Otherwise, will create a folder under specified directory with the name
+  // indicated in path (i.e. everything after the last encounter of `/`).
+  const char* env_mount_dir = std::getenv("TRITON_AWS_MOUNT_DIRECTORY");
   std::string tmp_folder;
-  RETURN_IF_ERROR(
-      triton::core::MakeTemporaryDirectory(FileSystemType::LOCAL, &tmp_folder));
+  if (mount_dir.empty() && env_mount_dir == nullptr) {
+    RETURN_IF_ERROR(triton::core::MakeTemporaryDirectory(
+        FileSystemType::LOCAL, &tmp_folder));
+  } else {
+    tmp_folder = mount_dir.empty() ? std::string(env_mount_dir) : mount_dir;
+    tmp_folder =
+        JoinPath({tmp_folder, path.substr(path.find_last_of('/') + 1)});
+    RETURN_IF_ERROR(triton::core::MakeDirectory(
+        tmp_folder, true /*recursive*/, true /*allow_dir_exist*/));
+  }
 
   // Specify contents to be downloaded
   std::set<std::string> contents;
@@ -693,7 +710,7 @@ S3FileSystem::LocalizePath(
               : JoinPath({(*localized)->Path(), s3_removed_path});
       bool is_subdir;
       RETURN_IF_ERROR(IsDirectory(s3_fpath, &is_subdir));
-      if (is_subdir) {
+      if (recursive && is_subdir) {
         // Create local mirror of sub-directories
 #ifdef _WIN32
         int status = mkdir(const_cast<char*>(local_fpath.c_str()));
@@ -716,7 +733,7 @@ S3FileSystem::LocalizePath(
              ++itr) {
           contents.insert(JoinPath({s3_fpath, *itr}));
         }
-      } else {
+      } else if (!is_subdir) {
         // Create local copy of file
         std::string file_bucket, file_object;
         RETURN_IF_ERROR(ParsePath(s3_fpath, &file_bucket, &file_object));
@@ -766,7 +783,8 @@ S3FileSystem::WriteBinaryFile(
 }
 
 Status
-S3FileSystem::MakeDirectory(const std::string& dir, const bool recursive)
+S3FileSystem::MakeDirectory(
+    const std::string& dir, const bool recursive, const bool allow_dir_exist)
 {
   return Status(
       Status::Code::UNSUPPORTED,
diff --git a/src/model_config_utils.cc b/src/model_config_utils.cc
index 850c322c8..729c243af 100644
--- a/src/model_config_utils.cc
+++ b/src/model_config_utils.cc
@@ -918,8 +918,8 @@ LocalizePythonBackendExecutionEnvironmentPath(
           model_path_slash) {
         // Localize the file
         std::shared_ptr<LocalizedPath> localized_exec_env_path;
-        RETURN_IF_ERROR(
-            LocalizePath(abs_exec_env_path, &localized_exec_env_path));
+        RETURN_IF_ERROR(LocalizePath(
+            abs_exec_env_path, true /*recursive*/, &localized_exec_env_path));
         // Persist the localized temporary path
         (*localized_model_dir)
             ->other_localized_path.push_back(localized_exec_env_path);