Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rename 'FastOCI' as 'OverlayBD - Turbo OCIv1' #252

Merged
merged 1 commit into from
Aug 24, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 1 addition & 7 deletions .github/workflows/cmake.yml
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ jobs:
ls obd_mp
sudo diff -r --exclude "lost+found" test_data obd_mp
sudo umount obd_mp
- name: E2E Test FastOCI
- name: E2E Test turboOCIv1
working-directory: ${{github.workspace}}/build
shell: bash
run: |
Expand Down Expand Up @@ -118,9 +118,3 @@ jobs:
working-directory: ${{github.workspace}}/build
shell: bash
run: ctest -j8 --output-on-failure -C $BUILD_TYPE






2 changes: 1 addition & 1 deletion src/image_file.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -384,7 +384,7 @@ LSMT::IFileRW *ImageFile::open_upper(ImageConfigNS::UpperConfig &upper) {
}

if (upper.target() != "") {
LOG_INFO("fastoci upper layer : `, `, `, `", upper.index(), upper.data(), upper.target());
LOG_INFO("turboOCIv1 upper layer : `, `, `, `", upper.index(), upper.data(), upper.target());
target_file = new_sure_file_by_path(upper.target().c_str(), O_RDWR, this);
if (!target_file) {
LOG_ERROR("open(`,flags), `:`", upper.target(), errno, strerror(errno));
Expand Down
2 changes: 1 addition & 1 deletion src/overlaybd/tar/libtar.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ int UnTar::extract_file() {

int UnTar::extract_regfile_meta_only(const char *filename) {
size_t size = get_size();
LOG_DEBUG(" ==> extracting: ` (` bytes) (fastoci index)", filename, size);
LOG_DEBUG(" ==> extracting: ` (` bytes) (turboOCIv1 index)", filename, size);
photon::fs::IFile *fout = fs->open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_NOFOLLOW, 0666);
if (fout == nullptr) {
return -1;
Expand Down
12 changes: 6 additions & 6 deletions src/overlaybd/tar/test/test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -167,10 +167,10 @@ TEST_F(TarTest, tar_meta) {
make_extfs(verify_dev);
auto verify_ext4fs = new_extfs(verify_dev, false);
auto verifyfs = new_subfs(verify_ext4fs, "/", true);
auto fastoci_verify = new UnTar(src_file, verifyfs, 0, 4096, verify_dev, true);
ASSERT_EQ(0, fastoci_verify->extract_all());
auto turboOCI_verify = new UnTar(src_file, verifyfs, 0, 4096, verify_dev, true);
ASSERT_EQ(0, turboOCI_verify->extract_all());
verify_ext4fs->sync();
delete fastoci_verify;
delete turboOCI_verify;
delete verifyfs;

src_file->lseek(0, 0);
Expand All @@ -187,9 +187,9 @@ TEST_F(TarTest, tar_meta) {
make_extfs(imgfile);
auto extfs = new_extfs(imgfile, false);
auto target = new_subfs(extfs, "/", true);
auto fastoci_mock = new UnTar(tar_idx, target, TAR_IGNORE_CRC, 4096, imgfile, true, true);
auto ret = fastoci_mock->extract_all();
delete fastoci_mock;
auto turboOCI_mock = new UnTar(tar_idx, target, TAR_IGNORE_CRC, 4096, imgfile, true, true);
auto ret = turboOCI_mock->extract_all();
delete turboOCI_mock;
delete target;

ASSERT_EQ(0, ret);
Expand Down
10 changes: 5 additions & 5 deletions src/tools/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,17 @@ target_include_directories(overlaybd-apply PUBLIC ${PHOTON_INCLUDE_DIR} ${rapidj
target_link_libraries(overlaybd-apply photon_static overlaybd_lib overlaybd_image_lib)
set_target_properties(overlaybd-apply PROPERTIES INSTALL_RPATH "/opt/overlaybd/lib")

add_executable(fastoci-apply fastoci-apply.cpp comm_func.cpp)
target_include_directories(fastoci-apply PUBLIC ${PHOTON_INCLUDE_DIR} ${rapidjson_SOURCE_DIR}/include)
target_link_libraries(fastoci-apply photon_static overlaybd_lib overlaybd_image_lib)
set_target_properties(fastoci-apply PROPERTIES INSTALL_RPATH "/opt/overlaybd/lib")
add_executable(turboOCI-apply turboOCI-apply.cpp comm_func.cpp)
target_include_directories(turboOCI-apply PUBLIC ${PHOTON_INCLUDE_DIR} ${rapidjson_SOURCE_DIR}/include)
target_link_libraries(turboOCI-apply photon_static overlaybd_lib overlaybd_image_lib)
set_target_properties(turboOCI-apply PROPERTIES INSTALL_RPATH "/opt/overlaybd/lib")


install(TARGETS
overlaybd-commit
overlaybd-create
overlaybd-zfile
overlaybd-apply
fastoci-apply
turboOCI-apply
DESTINATION /opt/overlaybd/bin
)
2 changes: 1 addition & 1 deletion src/tools/comm_func.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ photon::fs::IFileSystem *create_ext4fs(photon::fs::IFile *imgfile, bool mkfs,
exit(-1);
}
}
// for now, buffer_file can't be used with fastoci
// for now, buffer_file can't be used with turboOCI
auto extfs = new_extfs(imgfile, enable_buffer);
if (!extfs) {
fprintf(stderr, "new extfs failed, %s\n", strerror(errno));
Expand Down
8 changes: 4 additions & 4 deletions src/tools/overlaybd-apply.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ int main(int argc, char **argv) {

app.add_flag("--verbose", verbose, "output debug info")->default_val(false);
app.add_option("--service_config_path", config_path, "overlaybd image service config path")->type_name("FILEPATH")->check(CLI::ExistingFile)->default_val("/etc/overlaybd/overlaybd.json");
app.add_option("--gz_index_path", gz_index_path, "build gzip index if layer is gzip, only used with fastoci")->type_name("FILEPATH");
app.add_option("--gz_index_path", gz_index_path, "build gzip index if layer is gzip, only used with turboOCIv1")->type_name("FILEPATH");
app.add_option("--checksum", sha256_checksum, "sha256 checksum for origin uncompressed data");
app.add_option("input_path", input_path, "input OCIv1 tar layer path")->type_name("FILEPATH")->check(CLI::ExistingFile)->required();

Expand All @@ -81,9 +81,9 @@ int main(int argc, char **argv) {
delete imgfile;
delete imgservice;
});
bool gen_fastoci = (gz_index_path != "" );
bool gen_turboOCI = (gz_index_path != "" );

auto target = create_ext4fs(imgfile, mkfs, !gen_fastoci, "/");
auto target = create_ext4fs(imgfile, mkfs, !gen_turboOCI, "/");
DEFER({ delete target; });

photon::fs::IFile* src_file = nullptr;
Expand All @@ -109,7 +109,7 @@ int main(int argc, char **argv) {

photon::fs::IFile* base_file = raw ? nullptr : ((ImageFile *)imgfile)->get_base();

auto tar = new UnTar(src_file, target, 0, 4096, base_file, gen_fastoci);
auto tar = new UnTar(src_file, target, 0, 4096, base_file, gen_turboOCI);

if (tar->extract_all() < 0) {
fprintf(stderr, "failed to extract\n");
Expand Down
8 changes: 5 additions & 3 deletions src/tools/overlaybd-commit.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ int main(int argc, char **argv) {
int block_size = -1;
std::string data_file_path, index_file_path, commit_file_path, remote_mapping_file;
bool compress_zfile = false;
bool build_turboOCI = false;
bool build_fastoci = false;
bool tar = false, rm_old = false, seal = false, commit_sealed = false;
bool verbose = false;
Expand All @@ -69,15 +70,16 @@ int main(int argc, char **argv) {
app.add_option(
"--bs", block_size,
"The size of a data block in KB. Must be a power of two between 4K~64K [4/8/16/32/64](default 4)");
app.add_flag("--fastoci", build_fastoci, "commit using fastoci format")->default_val(false);
app.add_flag("--turboOCI", build_turboOCI, "commit using turboOCIv1 format")->default_val(false);
app.add_flag("--fastoci", build_fastoci, "commit using turboOCIv1 format (depracated)")->default_val(false);
app.add_option("data_file", data_file_path, "data file path")->type_name("FILEPATH")->check(CLI::ExistingFile)->required();
app.add_option("index_file", index_file_path, "index file path")->type_name("FILEPATH");
app.add_option("commit_file", commit_file_path, "commit file path")->type_name("FILEPATH");
app.add_flag("--seal", seal, "seal only, data_file is output itself")->default_val(false);
app.add_flag("--commit_sealed", commit_sealed, "commit sealed, index_file is output")->default_val(false);
app.add_flag("--verbose", verbose, "output debug info")->default_val(false);
CLI11_PARSE(app, argc, argv);

build_turboOCI = build_turboOCI || build_fastoci;
set_log_output_level(verbose ? 0 : 1);
photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_DEFAULT);
DEFER({photon::fini();});
Expand All @@ -86,7 +88,7 @@ int main(int argc, char **argv) {

IFile* fdata = open_file(lfs, data_file_path.c_str(), O_RDWR, 0);
IFileRW* fin = nullptr;
if (build_fastoci) {
if (build_turboOCI) {
LOG_INFO("commit LSMTWarpFile with args: {index_file: `, fsmeta: `}",
index_file_path, data_file_path);
IFile* findex = open_file(lfs, index_file_path.c_str(), O_RDONLY, 0);
Expand Down
8 changes: 6 additions & 2 deletions src/tools/overlaybd-create.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -50,21 +50,25 @@ int main(int argc, char **argv) {
string parent_uuid;
bool sparse = false;
std::string data_file_path, index_file_path, warp_index_path;
bool build_turboOCI = false;
bool build_fastoci = false;
bool mkfs = false;
bool verbose = false;

CLI::App app{"this is overlaybd-create"};
app.add_option("-u", parent_uuid, "parent uuid");
app.add_flag("-s", sparse, "create sparse RW layer")->default_val(false);
app.add_flag("--fastoci", build_fastoci, "commit using fastoci format")->default_val(false);
app.add_flag("--turboOCI", build_turboOCI, "commit using turboOCI format")->default_val(false);
app.add_flag("--fastoci", build_fastoci, "commit using turboOCI format(depracated)")->default_val(false);
app.add_flag("--mkfs", mkfs, "mkfs after create")->default_val(false);
app.add_option("data_file", data_file_path, "data file path")->type_name("FILEPATH")->required();
app.add_option("index_file", index_file_path, "index file path")->type_name("FILEPATH")->required();
app.add_option("vsize", vsize, "virtual size(GB)")->type_name("INT")->check(CLI::PositiveNumber)->required();
app.add_flag("--verbose", verbose, "output debug info")->default_val(false);
CLI11_PARSE(app, argc, argv);

build_turboOCI = build_turboOCI || build_fastoci;

set_log_output_level(verbose ? 0 : 1);
photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_DEFAULT);
DEFER({photon::fini();});
Expand All @@ -76,7 +80,7 @@ int main(int argc, char **argv) {
IFile* findex = open_file(index_file_path.c_str(), flag, mode);
IFile* file = nullptr;

if (build_fastoci) {
if (build_turboOCI) {
LSMT::WarpFileArgs args(findex, fdata, nullptr);
args.virtual_size = vsize;
file = LSMT::create_warpfile(args, false);
Expand Down
14 changes: 7 additions & 7 deletions src/tools/fastoci-apply.cpp → src/tools/turboOCI-apply.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -65,18 +65,18 @@ int main(int argc, char **argv) {
bool raw = false, mkfs = false, verbose = false;
bool export_tar_headers = false, import_tar_headers = false;

CLI::App app{"this is fastoci-apply, apply OCIv1 tar layer to overlaybd-fastoci format"};
CLI::App app{"this is turboOCI-apply, apply OCIv1 tar layer to 'Overlaybd-TurboOCI v1' format"};
app.add_flag("--mkfs", mkfs, "mkfs before apply")->default_val(false);
app.add_flag("--verbose", verbose, "output debug info")->default_val(false);
app.add_option("--service_config_path", config_path, "overlaybd image service config path")
->type_name("FILEPATH")
->check(CLI::ExistingFile)
->default_val("/etc/overlaybd/overlaybd.json");
app.add_option("--gz_index_path", gz_index_path,
"build gzip index if layer is gzip, only used with fastoci")
"build gzip index if layer is gzip, only used with turboOCI")
->type_name("FILEPATH")
->default_val("gzip.meta");
app.add_flag("--import", import_tar_headers, "generate fastoci file from <input_path>")
app.add_flag("--import", import_tar_headers, "generate turboOCI file from <input_path>")
->default_val(false);
app.add_flag("--export", export_tar_headers, "export tar meta from <input_path>")
->default_val(false);
Expand Down Expand Up @@ -121,20 +121,20 @@ int main(int argc, char **argv) {
delete imgservice;
});

// for now, buffer_file can't be used with fastoci
// for now, buffer_file can't be used with turboOCI
auto target = create_ext4fs(imgfile, mkfs, false, "/");
DEFER({ delete target; });

photon::fs::IFile *base_file = raw ? nullptr : ((ImageFile *)imgfile)->get_base();
bool gen_fastoci = true;
bool gen_turboOCI = true;
int option = (import_tar_headers ? TAR_IGNORE_CRC : 0);
auto tar =
new UnTar(src_file, target, option, 4096, base_file, gen_fastoci, import_tar_headers);
new UnTar(src_file, target, option, 4096, base_file, gen_turboOCI, import_tar_headers);

if (tar->extract_all() < 0) {
fprintf(stderr, "failed to extract\n");
exit(-1);
}
fprintf(stdout, "fastoci-apply done\n");
fprintf(stdout, "turboOCI-apply done\n");
return 0;
}