From 569bb318d1b5a4d192943daf9cbe493e028cc34c Mon Sep 17 00:00:00 2001 From: alandefreitas Date: Thu, 26 Oct 2023 16:35:25 -0300 Subject: [PATCH] Archive variants Add extra archive variants such as docs-only and source-only. These variants can reduce expenses with JFrog download bandwidth, provide users with archives that are simpler to use, and provide docs-only archives for the website. The MakeBoostDistro.py script includes parameters to determine what types of files should be included in the distribution. All other functions are adapted to handle these requirements accordingly. fix #50 --- MakeBoostDistro.py | 577 +++++++++++++++++++++++++++++++++----------- ci_boost_release.py | 270 +++++++++++++-------- 2 files changed, 601 insertions(+), 246 deletions(-) diff --git a/MakeBoostDistro.py b/MakeBoostDistro.py index c839f8c..a2c2a2a 100755 --- a/MakeBoostDistro.py +++ b/MakeBoostDistro.py @@ -12,13 +12,16 @@ from __future__ import print_function -import os, sys +import os +import sys import shutil import stat import six import datetime +import fnmatch +import re -IgnoreFiles = shutil.ignore_patterns( +ignored_files_pattern = shutil.ignore_patterns( "[.]*", "[.]gitattributes", "[.]gitignore", @@ -29,17 +32,102 @@ ) -def IgnoreFile(src, name): - return len(IgnoreFiles(src, [name])) > 0 +def should_ignore_file(src, name): + return len(ignored_files_pattern(src, [name])) > 0 + + +source_file_patterns = [ + "*.jam", + "Jamfile.v2", + "Jamfile", + "Jamfile.jam", + "jamfile.jam", + "jamfile.v2", + "jamfile", + "build.jam", + "Jamroot", + "*.cpp", + "*.hpp", + "*.ipp", + "*.cxx", + "*.hxx", + "*.c", + "*.h", + "*.asm", + "*.S", + "CMakeLists.txt", + "*.cmake", + "Makefile", + "*.py", + "*.sh", + "*.bash", + "*.zsh", + "*.ksh", + "*.csh", + "*.bat", + "*.pl", + "INSTALL", + "LICENSE*", + "COPYING*", + "COPYRIGHT*", + "LICENCE*", + "UNLICENSE*", +] + + +def is_image_file(file_path): + return any( + file_path.endswith(ext) for ext in [".png", ".jpg", ".jpeg", ".gif", ".svg"] + ) + + +def is_html_content_file(file_path): + return ( + any( + file_path.endswith(ext) + for ext in [ + ".html", + ".htm", + ".css", + ".js", + ".txt", + "README.md", + ".pdf", + "readme", + ".md", + ] + ) + and os.path.split(file_path)[1] != "CMakeLists.txt" + ) + + +# Determine if file should be included in a source release +def is_source_file(f): + return any( + fnmatch.fnmatch(os.path.split(f)[1], pattern) + for pattern in source_file_patterns + ) + +DocFiles = ["INSTALL", "LICENSE*", "COPYING*", "COPYRIGHT*", "LICENCE*", "UNLICENSE*"] -## from -def MergeTree(src, dst, symlinks=False): + +# Determine if file should be included in a documentation release +def is_doc_file(f): + return ( + is_html_content_file(f) + or is_image_file(f) + or any(fnmatch.fnmatch(os.path.split(f)[1], pattern) for pattern in DocFiles) + ) + + +# http://stackoverflow.com/questions/1868714/how-do-i-copy-an-entire-directory-of-files-into-an-existing-directory-using-pyth +def merge_dir_tree(src, dst, symlinks=False): if not os.path.exists(dst): os.makedirs(dst) shutil.copystat(src, dst) lst = os.listdir(src) - excl = IgnoreFiles(src, lst) + excl = ignored_files_pattern(src, lst) lst = [x for x in lst if x not in excl] for item in lst: s = os.path.join(src, item) @@ -55,35 +143,78 @@ def MergeTree(src, dst, symlinks=False): except: pass # lchmod not available elif os.path.isdir(s): - MergeTree(s, d, symlinks) + merge_dir_tree(s, d, symlinks) else: if os.path.exists(d): print("## Overwriting file %s with %s" % (d, s)) shutil.copy2(s, d) -def CopyFile(s, d, f): - if os.path.isfile(os.path.join(s, f)) and not IgnoreFile(s, f): - shutil.copy2(os.path.join(s, f), os.path.join(d, f)) +# Merge dd into '{s}/{d}' if it exists +def merge_dir_tree_if_exists(s, d, dd): + if os.path.exists(os.path.join(s, dd)): + merge_dir_tree(os.path.join(s, dd), os.path.join(d, dd), symlinks=False) -def CopyDir(s, d, dd): - if os.path.isdir(os.path.join(s, dd)) and not IgnoreFile(s, dd): - shutil.copytree( - os.path.join(s, dd), os.path.join(d, dd), symlinks=False, ignore=IgnoreFiles +# Copy a file is it belongs to this release variant +def copy_distro_file(s, d, f, include_source, include_docs): + is_full_dist = include_source and include_docs + if ( + os.path.isfile(os.path.join(s, f)) + and not should_ignore_file(s, f) + and ( + is_full_dist or (include_source and is_source_file(f)) or (include_docs and is_doc_file(f)) ) + ): + shutil.copy2(os.path.join(s, f), os.path.join(d, f)) -def MergeIf(s, d, dd): - # if dd == 'detail': - # print "MergeIf %s -> %s" % (os.path.join(s, dd), os.path.join(d, dd)) - if os.path.exists(os.path.join(s, dd)): - MergeTree(os.path.join(s, dd), os.path.join(d, dd), symlinks=False) - - -def CopyInclude(src, dst): +# Copy file s to directory d if it's a source file +def copy_abs_source_file(s, d): + if os.path.isfile(s) and is_source_file(s): + shutil.copy2(s, d) + + +# Copy file s to directory d if it's a documentation file +def copy_abs_doc_file(s, d): + if os.path.isfile(s) and is_doc_file(s): + shutil.copy2(s, d) + + +# Copy file s to directory d if it belongs to the distribution +def copy_distro_dir(s, d, dd, include_source, include_docs): + if os.path.isdir(os.path.join(s, dd)) and not should_ignore_file(s, dd): + full_dist = include_source and include_docs + if full_dist: + if os.path.isdir(os.path.join(s, dd)) and not should_ignore_file(s, dd): + shutil.copytree( + os.path.join(s, dd), + os.path.join(d, dd), + symlinks=False, + ignore=ignored_files_pattern, + ) + elif not include_docs: + shutil.copytree( + os.path.join(s, dd), + os.path.join(d, dd), + symlinks=False, + ignore=ignored_files_pattern, + copy_function=copy_abs_source_file, + ) + elif not include_source: + shutil.copytree( + os.path.join(s, dd), + os.path.join(d, dd), + symlinks=False, + ignore=ignored_files_pattern, + copy_function=copy_abs_doc_file, + ) + + +# Copy an include directory +def copy_distro_include_dir(src, dst, include_source, include_docs): for item in os.listdir(src): - if IgnoreFile(src, item): + if should_ignore_file(src, item): continue if item == "pending": continue @@ -92,133 +223,289 @@ def CopyInclude(src, dst): s = os.path.join(src, item) d = os.path.join(dst, item) if os.path.isdir(s): - MergeTree(s, d, symlinks=False) + merge_dir_tree(s, d, symlinks=False) else: + if not include_docs and not is_doc_file(item): + continue + if not include_source and not is_source_file(item): + continue if os.path.exists(d): print("## Overwriting file %s with %s" % (d, s)) - CopyFile(src, dst, item) - - -def CopySubProject(src, dst, headers, p): - # First, everything except the "include" directory - Source = os.path.join(src, p) - Dest = os.path.join(dst, p) - # print "CopySubProject %p" % p - os.makedirs(Dest) - for item in os.listdir(Source): - if os.path.isfile(os.path.join(Source, item)): - CopyFile(Source, Dest, item) + copy_distro_file(src, dst, item, include_source, include_docs) + + +# Copy a subproject according to the distribution rules +def copy_distro_subproject(src, dst, headers, p, include_source, include_docs): + # First, everything except the "include" directory + source_project_dir = os.path.join(src, p) + dest_project_dir = os.path.join(dst, p) + # print "CopySubProject %p" % p + os.makedirs(dest_project_dir) + for item in os.listdir(source_project_dir): + if item == "antora": + continue + if os.path.isfile(os.path.join(source_project_dir, item)): + copy_distro_file( + source_project_dir, dest_project_dir, item, include_source, include_docs + ) elif item != "include": - CopyDir(Source, Dest, item) - - # shutil.copytree(Source, Dest, symlinks=False, ignore=shutil.ignore_patterns('\.*', "include")) + copy_distro_dir( + source_project_dir, dest_project_dir, item, include_source, include_docs + ) # Now the includes - Source = os.path.join(src, "%s/include/boost" % p) - if os.path.exists(Source): - CopyInclude(Source, headers) - # MergeTree(Source, Dest, symlinks=False, ignore=shutil.ignore_patterns('\.*', 'detail', 'pending')) - MergeIf(Source, headers, "detail") - MergeIf(Source, headers, "pending") - - -def CopyNestedProject(src, dst, headers, p): - # First, everything except the "include" directory - Source = os.path.join(src, p[1]) - Dest = os.path.join(dst, p[1]) - os.makedirs(Dest) - for item in os.listdir(Source): - if os.path.isfile(os.path.join(Source, item)): - CopyFile(Source, Dest, item) + source_project_dir = os.path.join(src, "%s/include/boost" % p) + if os.path.exists(source_project_dir): + copy_distro_include_dir( + source_project_dir, headers, include_source, include_docs + ) + merge_dir_tree_if_exists(source_project_dir, headers, "detail") + merge_dir_tree_if_exists(source_project_dir, headers, "pending") + + +def copy_distro_nested_project(src, dst, headers, p, include_source, include_docs): + # First, everything except the "include" directory + source_nested_dir = os.path.join(src, p[1]) + dest_nested_dir = os.path.join(dst, p[1]) + os.makedirs(dest_nested_dir) + for item in os.listdir(source_nested_dir): + if os.path.isfile(os.path.join(source_nested_dir, item)): + copy_distro_file( + source_nested_dir, dest_nested_dir, item, include_source, include_docs + ) elif item != "include": - CopyDir(Source, Dest, item) - # shutil.copytree(Source, Dest, symlinks=False, ignore=shutil.ignore_patterns('\.*', "include")) - - Source = os.path.join(src, "%s/include/boost" % (p[1])) - # Dest = os.path.join(headers, p) - # print "Installing headers from %s to %s" % (Source, headers) - CopyInclude(Source, headers) - # # MergeTree(Source, Dest, symlinks=False, ignore=shutil.ignore_patterns('\.*', 'detail', 'pending')) - # MergeIf(Source, headers, 'detail') - # MergeIf(Source, headers, 'pending') - - -BoostHeaders = "boost" -BoostLibs = "libs" - -BoostSpecialFolders = ["doc", "more", "status", "tools"] - -SourceRoot = sys.argv[1] -DestRoot = sys.argv[2] - -print("Source = %s" % SourceRoot) -print("Dest = %s" % DestRoot) - -if not os.path.exists(SourceRoot): - print("## Error: %s does not exist" % SourceRoot) - exit(1) + copy_distro_dir( + source_nested_dir, dest_nested_dir, item, include_source, include_docs + ) + source_nested_dir = os.path.join(src, "%s/include/boost" % (p[1])) + copy_distro_include_dir(source_nested_dir, headers, include_source, include_docs) + + +# Remove any empty subdirectories that might have been left after +# recursively skipping files +def remove_empty_subdirs(path): + subdirs = [ + os.path.join(path, d) + for d in os.listdir(path) + if os.path.isdir(os.path.join(path, d)) + ] + for subdir in subdirs: + remove_empty_subdirs(subdir) + try: + os.rmdir(subdir) + except OSError: + pass + + +def main( + src_root, dest_root, include_source=True, include_docs=True, cmake_distro=False +): + if not os.path.isabs(src_root): + src_root = os.path.abspath(src_root) + print("Source = %s" % src_root) + + if not os.path.isabs(dest_root): + dest_root = os.path.abspath(dest_root) + print("Dest = %s" % dest_root) + + if not os.path.exists(src_root): + print("## Error: %s does not exist" % src_root) + exit(1) + + if os.path.exists(dest_root): + print( + "The destination directory already exists. Renaming it, so that a new one can be generated.\n" + ) + timestamp1 = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") + os.rename(dest_root, dest_root + "_bck_" + timestamp1) + + if not os.path.exists(dest_root): + print("Creating destination directory %s" % dest_root) + os.makedirs(dest_root) + + # Step 1: Copy all the files at the root level to the dest folder + for f in os.listdir(src_root): + if f != "CMakeLists.txt" or cmake_distro: + copy_distro_file(src_root, dest_root, f, include_source, include_docs) + + # Step 2: Copy all the "special" root folders to the dest folder + special_root_folders = ["tools"] + if include_docs: + special_root_folders += ["doc", "more", "status"] + if cmake_distro: + special_root_folders += ["libs"] + for d in special_root_folders: + copy_distro_dir(src_root, dest_root, d, include_source, include_docs) + + # Step 3: copy all the files from $SOURCE/libs/* to $DEST/libs/* + libs_dir_name = "libs" + dest_libs_dir = os.path.join(dest_root, libs_dir_name) + if not os.path.exists(dest_libs_dir): + os.makedirs(dest_libs_dir) + source_libs_dir = os.path.join(src_root, libs_dir_name) + for f in os.listdir(source_libs_dir): + copy_distro_file( + source_libs_dir, dest_libs_dir, f, include_source, include_docs + ) -if os.path.exists(DestRoot): - print( - "The destination directory already exists. Renaming it, so that a new one can be generated.\n" - ) - timestamp1 = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") - os.rename(DestRoot, DestRoot + "_bck_" + timestamp1) - -if not os.path.exists(DestRoot): - print("Creating destination directory %s" % DestRoot) - os.makedirs(DestRoot) - -DestHeaders = os.path.join(DestRoot, BoostHeaders) -DestLibs = os.path.join(DestRoot, BoostLibs) -os.makedirs(DestHeaders) -os.makedirs(DestLibs) - -## Step 1 -for f in os.listdir(SourceRoot): - if f != "CMakeLists.txt": - CopyFile(SourceRoot, DestRoot, f) - -## Step 2 -for d in BoostSpecialFolders: - CopyDir(SourceRoot, DestRoot, d) - -## Step 3 -SourceLibs = os.path.join(SourceRoot, BoostLibs) -for f in os.listdir(SourceLibs): - CopyFile(SourceLibs, DestLibs, f) - -## Step 4 -BoostSubProjects = set() -for f in os.listdir(SourceLibs): - if os.path.isdir(os.path.join(SourceLibs, f)): - if os.path.isfile(os.path.join(SourceLibs, f, "meta", "libraries.json")): - BoostSubProjects.add(f) - elif os.path.isdir(os.path.join(SourceLibs, f, "include")): - BoostSubProjects.add(f) - elif f == "headers": - BoostSubProjects.add(f) - elif os.path.isfile(os.path.join(SourceLibs, f, "sublibs")): - for s in os.listdir(os.path.join(SourceLibs, f)): - if os.path.isdir(os.path.join(SourceLibs, f, s)): - if os.path.isfile( - os.path.join(SourceLibs, f, s, "meta", "libraries.json") - ): - BoostSubProjects.add((f, s)) - elif os.path.isdir(os.path.join(SourceLibs, f, s, "include")): - BoostSubProjects.add((f, s)) - -for p in BoostSubProjects: - if isinstance(p, six.string_types): - CopySubProject(SourceLibs, DestLibs, DestHeaders, p) + # Step 4: For each subproject, copy everything except "include" into $DEST/libs + # 4a) Aggregate subprojects to copy + boost_subprojects = set() + for f in os.listdir(source_libs_dir): + if os.path.isdir(os.path.join(source_libs_dir, f)): + if os.path.isfile( + os.path.join(source_libs_dir, f, "meta", "libraries.json") + ): + boost_subprojects.add(f) + elif os.path.isdir(os.path.join(source_libs_dir, f, "include")): + boost_subprojects.add(f) + elif f == "headers": + boost_subprojects.add(f) + elif os.path.isfile(os.path.join(source_libs_dir, f, "sublibs")): + for s in os.listdir(os.path.join(source_libs_dir, f)): + if os.path.isdir(os.path.join(source_libs_dir, f, s)): + if os.path.isfile( + os.path.join( + source_libs_dir, f, s, "meta", "libraries.json" + ) + ): + boost_subprojects.add((f, s)) + elif os.path.isdir( + os.path.join(source_libs_dir, f, s, "include") + ): + boost_subprojects.add((f, s)) + + if not cmake_distro: + # Step 4b) Copy each subproject + # copy the contents of the "includes" folder into $DEST/boost + # copy the contents of the other folders into $DEST/libs + headers_dir_name = "boost" + dest_headers = os.path.join(dest_root, headers_dir_name) + os.makedirs(dest_headers) + for p in boost_subprojects: + if isinstance(p, six.string_types): + copy_distro_subproject( + source_libs_dir, + dest_libs_dir, + dest_headers, + p, + include_source, + include_docs, + ) + else: + nested_source_dir = os.path.join(src_root, "libs", p[0]) + nested_dest_dir = os.path.join(dest_root, "libs", p[0]) + nested_dest_headers_dir = os.path.join(dest_root, "boost") + if not os.path.exists(nested_dest_dir): + os.makedirs(nested_dest_dir) + if not os.path.exists(nested_dest_headers_dir): + os.makedirs(nested_dest_headers_dir) + for f in os.listdir(nested_source_dir): + copy_distro_file( + nested_source_dir, + nested_dest_dir, + f, + include_source, + include_docs, + ) + copy_distro_nested_project( + nested_source_dir, + nested_dest_dir, + nested_dest_headers_dir, + p, + include_source, + include_docs, + ) else: - NestedSource = os.path.join(SourceRoot, "libs", p[0]) - NestedDest = os.path.join(DestRoot, "libs", p[0]) - NestedHeaders = os.path.join(DestRoot, "boost") - if not os.path.exists(NestedDest): - os.makedirs(NestedDest) - if not os.path.exists(NestedHeaders): - os.makedirs(NestedHeaders) - for f in os.listdir(NestedSource): - CopyFile(NestedSource, NestedDest, f) - CopyNestedProject(NestedSource, NestedDest, NestedHeaders, p) + # Step 4b) Clean tests dir from cmake distro + def clean_test_dir(test_dir): + if not os.path.exists(test_dir) or not os.path.exists(test_dir): + return + for base_path in os.listdir(test_dir): + abs_path = os.path.join(test_dir, base_path) + if os.path.isdir(abs_path): + clean_test_dir(abs_path) + elif base_path == "CMakeLists.txt" or base_path.endswith(".cmake"): + with open(abs_path, "w") as fw: + fw.write("# Placeholder \n") + else: + os.remove(abs_path) + + for p in boost_subprojects: + if isinstance(p, six.string_types): + clean_test_dir(os.path.join(dest_root, "libs", p, "test")) + else: + clean_test_dir(os.path.join(dest_root, "libs", p[0], "docs")) + clean_test_dir(os.path.join(dest_root, "libs", p[0], p[1], "docs")) + + # Step 5: Copy any source files referred linked in the docs + if not include_source and include_docs: + href_pattern = re.compile(r'href=["\'](.*?)["\']') + # Iterate html files + for root, dirs, files in os.walk(dest_root): + for file in files: + filepath = os.path.join(root, file) + if filepath.endswith(".html") or filepath.endswith(".htm"): + # extract href contents + with open(filepath, "r") as f: + dir = os.path.dirname(filepath) + try: + contents = f.read() + hrefs = href_pattern.findall(contents) + for href in hrefs: + if href.startswith("http:") or href.startswith( + "https:" + ): + continue + p = href.rfind("#") + if p != -1: + href = href[:p] + if href == "" or is_doc_file(href): + continue + dest = os.path.normpath(os.path.join(dir, href)) + if not dest.startswith(dest_root): + continue + if os.path.exists(dest): + continue + src = os.path.join( + src_root, os.path.relpath(dest, dest_root) + ) + if not src.startswith(src_root): + continue + if not os.path.exists(src): + continue + destdir = os.path.dirname(dest) + if not os.path.exists(destdir): + os.makedirs(destdir) + shutil.copy2(src, dest) + except UnicodeDecodeError: + pass + + # Step 6: remove any empty subdirs + if not (include_source and include_docs): + remove_empty_subdirs(dest_root) + + +if __name__ == "__main__": + include_source = True + include_docs = True + cmake_distro = False + if "--source" in sys.argv[3:]: + include_source = True + if "--no-source" in sys.argv[3:]: + include_source = False + if "--docs" in sys.argv[3:]: + include_docs = True + if "--no-docs" in sys.argv[3:]: + include_docs = False + if "--cmake" in sys.argv[3:]: + cmake_distro = True + if "--no-cmake" in sys.argv[3:]: + cmake_distro = False + main( + sys.argv[1], + sys.argv[2], + include_source, + include_docs, + cmake_distro, + ) diff --git a/ci_boost_release.py b/ci_boost_release.py index 717fd5e..59a8af6 100755 --- a/ci_boost_release.py +++ b/ci_boost_release.py @@ -185,10 +185,10 @@ def names_to_string(names): for library in allmetadata: if category in allmetadata[library]["category"]: boostlibrarycategories[category]["libraries"].append(library) - # sort libraries in this category by name - boostlibrarycategories[category]["libraries"].sort( - key=lambda x: allmetadata[x]["name"].lower() - ) + # sort libraries in this category by name + boostlibrarycategories[category]["libraries"].sort( + key=lambda x: allmetadata[x]["name"].lower() + ) # sort allmetadata by name sorted_tuples = sorted(allmetadata.items(), key=lambda x: x[1]["name"].lower()) @@ -668,7 +668,20 @@ def command_build(self): import MakeBoostDistro os.chdir(self.releases_dir) - MakeBoostDistro.main(self.root_dir, self.boost_release_name) + # Full: Include source / Include docs / B2 layout + MakeBoostDistro.main(self.root_dir, self.boost_release_name, True, True) + # Source: Include source / Exclude docs / B2 layout + MakeBoostDistro.main( + self.root_dir, "%s-source" % self.boost_release_name, True, False + ) + # Docs: Exclude source / Include docs / B2 layout + MakeBoostDistro.main( + self.root_dir, "%s-docs" % self.boost_release_name, False, True + ) + # CMake: Include source / Exclude docs / CMake layout + MakeBoostDistro.main( + self.root_dir, "%s-cmake" % self.boost_release_name, True, False, True + ) else: os.chdir(os.path.join(self.build_dir)) utils.check_call( @@ -678,101 +691,150 @@ def command_build(self): "MakeBoostDistro.py", ) utils.check_call("chmod", "+x", "MakeBoostDistro.py") - os.chdir(os.path.dirname(self.root_dir)) + os.chdir(self.releases_dir) + # Full: Include source / Include docs / B2 layout utils.check_call( pythonbinary, os.path.join(self.build_dir, "MakeBoostDistro.py"), self.root_dir, self.boost_release_name, + "--source", + "--docs", + "--no-cmake", ) - - # Patch release with the html generate in-place - for sourcefilename in ["index.html", "libs/libraries.htm"]: - shutil.copyfile( - os.path.join(self.build_dir, sourcefilename), - os.path.join( - self.releases_dir, self.boost_release_name, sourcefilename - ), + # Source: Include source / Exclude docs / B2 layout + utils.check_call( + pythonbinary, + os.path.join(self.build_dir, "MakeBoostDistro.py"), + self.root_dir, + "%s-source" % self.boost_release_name, + "--source", + "--no-docs", + "--no-cmake", + ) + # Docs: Exclude source / Include docs / B2 layout + utils.check_call( + pythonbinary, + os.path.join(self.build_dir, "MakeBoostDistro.py"), + self.root_dir, + "%s-docs" % self.boost_release_name, + "--no-source", + "--docs", + "--no-cmake", + ) + # CMake: Include source / Exclude docs / CMake layout + utils.check_call( + pythonbinary, + os.path.join(self.build_dir, "MakeBoostDistro.py"), + self.root_dir, + "%s-cmake" % self.boost_release_name, + "--source", + "--no-docs", + "--cmake", ) - # Patch release with antora docs - antora_lib_docs = os.path.join(self.build_dir, "antora/build/lib/doc") - release_lib_docs = os.path.join( - self.releases_dir, self.boost_release_name, "libs" - ) - shutil.copytree( - antora_lib_docs, release_lib_docs, symlinks=False, dirs_exist_ok=True - ) + # Patch doc releases with the html generated in-place + for release_name in [ + self.boost_release_name, + "%s-docs" % self.boost_release_name, + ]: + for sourcefilename in ["index.html", "libs/libraries.htm"]: + abs_source = os.path.join(self.build_dir, sourcefilename) + abs_dest = os.path.join(self.releases_dir, release_name, sourcefilename) + print("Copying %s to %s" % (abs_source, abs_dest)) + shutil.copyfile(abs_source, abs_dest) + + # Patch doc releases with antora docs + for release_name in [ + self.boost_release_name, + "%s-docs" % self.boost_release_name, + ]: + antora_lib_docs = os.path.join(self.build_dir, "antora/build/lib/doc") + release_lib_docs = os.path.join(self.releases_dir, release_name, "libs") + ignored_files_pattern = shutil.ignore_patterns( + "[.]*", + "[.]gitattributes", + "[.]gitignore", + "[.]gitmodules", + "[.]travis[.]yml", + "appveyor[.]yml", + "circle[.]yml", + ) + shutil.copytree( + antora_lib_docs, + release_lib_docs, + symlinks=False, + ignore=ignored_files_pattern, + dirs_exist_ok=True, + ) + # Create archive files for all releases packages = [] archive_files = [] - - # Create packages for LF style content. - if self.eol == "LF": - os.chdir(self.releases_dir) - os.environ["GZIP"] = "-9" - os.environ["BZIP2"] = "-9" - archive_files.append( - "%s%s.tar.gz" % (self.boost_release_name, self.archive_tag) - ) - packages.append( - parallel_call( - "tar", - "--exclude=ci_boost_common.py", - "--exclude=ci_boost_release.py", - "-zcf", - "%s%s.tar.gz" % (self.boost_release_name, self.archive_tag), - self.boost_release_name, + for release_name in [ + self.boost_release_name, + "%s-source" % self.boost_release_name, + "%s-docs" % self.boost_release_name, + "%s-cmake" % self.boost_release_name, + ]: + # Create packages for LF style content. + if self.eol == "LF": + os.chdir(self.releases_dir) + os.environ["GZIP"] = "-9" + os.environ["BZIP2"] = "-9" + archive_files.append("%s%s.tar.gz" % (release_name, self.archive_tag)) + packages.append( + parallel_call( + "tar", + "--exclude=ci_boost_common.py", + "--exclude=ci_boost_release.py", + "-zcf", + "%s%s.tar.gz" % (release_name, self.archive_tag), + release_name, + ) ) - ) - archive_files.append( - "%s%s.tar.bz2" % (self.boost_release_name, self.archive_tag) - ) - packages.append( - parallel_call( - "tar", - "--exclude=ci_boost_common.py", - "--exclude=ci_boost_release.py", - "-jcf", - "%s%s.tar.bz2" % (self.boost_release_name, self.archive_tag), - self.boost_release_name, + archive_files.append("%s%s.tar.bz2" % (release_name, self.archive_tag)) + packages.append( + parallel_call( + "tar", + "--exclude=ci_boost_common.py", + "--exclude=ci_boost_release.py", + "-jcf", + "%s%s.tar.bz2" % (release_name, self.archive_tag), + release_name, + ) ) - ) - # Create packages for CRLF style content. - if self.eol == "CRLF": - os.chdir(self.releases_dir) - archive_files.append( - "%s%s.zip" % (self.boost_release_name, self.archive_tag) - ) - packages.append( - parallel_call( - "zip", - "-qr", - "-9", - "%s%s.zip" % (self.boost_release_name, self.archive_tag), - self.boost_release_name, - "-x", - self.boost_release_name + "/ci_boost_common.py", - self.boost_release_name + "/ci_boost_release.py", - ) - ) - archive_files.append( - "%s%s.7z" % (self.boost_release_name, self.archive_tag) - ) - with open("/dev/null") as dev_null: - utils.check_call( - "7z", - "a", - "-bd", - "-mx=7", - "-ms=on", - "-x!" + self.boost_release_name + "/ci_boost_common.py", - "-x!" + self.boost_release_name + "/ci_boost_release.py", - "%s%s.7z" % (self.boost_release_name, self.archive_tag), - self.boost_release_name, - stdout=dev_null, + # Create packages for CRLF style content. + if self.eol == "CRLF": + os.chdir(self.releases_dir) + archive_files.append("%s%s.zip" % (release_name, self.archive_tag)) + packages.append( + parallel_call( + "zip", + "-qr", + "-9", + "%s%s.zip" % (release_name, self.archive_tag), + release_name, + "-x", + release_name + "/ci_boost_common.py", + release_name + "/ci_boost_release.py", + ) ) + archive_files.append("%s%s.7z" % (release_name, self.archive_tag)) + with open("/dev/null") as dev_null: + utils.check_call( + "7z", + "a", + "-bd", + "-mx=7", + "-ms=on", + "-x!" + release_name + "/ci_boost_common.py", + "-x!" + release_name + "/ci_boost_release.py", + "%s%s.7z" % (release_name, self.archive_tag), + release_name, + stdout=dev_null, + ) for package in packages: package.join() @@ -1064,22 +1126,28 @@ def command_after_success(self): if self.mode == "check": return - if self.eol == "LF": - os.chdir(os.path.dirname(self.root_dir)) - self.upload_archives( - "%s%s.tar.gz" % (self.boost_release_name, self.archive_tag), - "%s%s.tar.gz.json" % (self.boost_release_name, self.archive_tag), - "%s%s.tar.bz2" % (self.boost_release_name, self.archive_tag), - "%s%s.tar.bz2.json" % (self.boost_release_name, self.archive_tag), - ) - if self.eol == "CRLF": - os.chdir(os.path.dirname(self.root_dir)) - self.upload_archives( - "%s%s.zip" % (self.boost_release_name, self.archive_tag), - "%s%s.zip.json" % (self.boost_release_name, self.archive_tag), - "%s%s.7z" % (self.boost_release_name, self.archive_tag), - "%s%s.7z.json" % (self.boost_release_name, self.archive_tag), - ) + for release_name in [ + self.boost_release_name, + "%s-source" % self.boost_release_name, + "%s-docs" % self.boost_release_name, + "%s-cmake" % self.boost_release_name, + ]: + if self.eol == "LF": + os.chdir(self.releases_dir) + self.upload_archives( + "%s%s.tar.gz" % (release_name, self.archive_tag), + "%s%s.tar.gz.json" % (release_name, self.archive_tag), + "%s%s.tar.bz2" % (release_name, self.archive_tag), + "%s%s.tar.bz2.json" % (release_name, self.archive_tag), + ) + if self.eol == "CRLF": + os.chdir(self.releases_dir) + self.upload_archives( + "%s%s.zip" % (release_name, self.archive_tag), + "%s%s.zip.json" % (release_name, self.archive_tag), + "%s%s.7z" % (release_name, self.archive_tag), + "%s%s.7z.json" % (release_name, self.archive_tag), + ) self.upload_to_s3()