-
Notifications
You must be signed in to change notification settings - Fork 12
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: create an exportable version of plan loading #91
Changes from all commits
0ea3c42
6758f3f
af4fb1e
bfe2ff6
44188b6
20acf0b
77fcd30
df1b42c
58bed47
73bec1f
74bf673
80b2c3e
347fbe2
e4485db
a72a827
d0c187d
320053c
125a8cb
db12b3d
12b54fe
3d1e40e
e49c11a
00de8bc
e754a1c
1b191ea
f5ce243
755c4ef
c28f6f0
dac8ea9
5c3d5f2
dd6b5c3
8fc79fb
7f8a7da
fbe9231
e487313
8ea2556
0d0246d
06cd5c9
9be359b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
# SPDX-License-Identifier: Apache-2.0 | ||
|
||
add_subdirectory(planloader) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
# SPDX-License-Identifier: Apache-2.0 | ||
|
||
if(NOT BUILD_SUBDIR_NAME EQUAL "release") | ||
message( | ||
SEND_ERROR, | ||
"The planloader library does not work in Debug mode due to its dependencies." | ||
) | ||
endif() | ||
|
||
add_library(planloader SHARED planloader.cpp) | ||
|
||
add_dependencies(planloader substrait_io) | ||
target_link_libraries(planloader substrait_io) | ||
|
||
install( | ||
TARGETS planloader | ||
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} | ||
PRIVATE_HEADER DESTINATION ${CMAKE_INSTALL_INCDIR}) | ||
|
||
if(${SUBSTRAIT_CPP_BUILD_TESTING}) | ||
add_subdirectory(tests) | ||
endif() |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
/* SPDX-License-Identifier: Apache-2.0 */ | ||
|
||
#include "planloader.h" | ||
|
||
#include <limits> | ||
#include <substrait/common/Io.h> | ||
|
||
extern "C" { | ||
|
||
// Load a Substrait plan (in any format) from disk. | ||
// Stores the Substrait plan in planBuffer in serialized form. | ||
// Returns a SerializedPlan structure containing either the serialized plan or | ||
// an error message. error_message is nullptr upon success. | ||
SerializedPlan* load_substrait_plan(const char* filename) { | ||
auto newPlan = new SerializedPlan(); | ||
newPlan->buffer = nullptr; | ||
newPlan->size = 0; | ||
newPlan->error_message = nullptr; | ||
|
||
auto planOrError = io::substrait::loadPlan(filename); | ||
if (!planOrError.ok()) { | ||
auto errMsg = planOrError.status().message(); | ||
newPlan->error_message = new char[errMsg.length()+1]; | ||
strncpy(newPlan->error_message, errMsg.data(), errMsg.length()+1); | ||
return newPlan; | ||
} | ||
::substrait::proto::Plan plan = *planOrError; | ||
std::string text = plan.SerializeAsString(); | ||
newPlan->buffer = new unsigned char[text.length()+1]; | ||
memcpy(newPlan->buffer, text.data(), text.length()+1); | ||
newPlan->size = static_cast<int32_t>( | ||
text.length() & | ||
std::numeric_limits<int32_t>::max()); | ||
return newPlan; | ||
} | ||
|
||
void free_substrait_plan(SerializedPlan* plan) { | ||
delete[] plan->buffer; | ||
delete[] plan->error_message; | ||
delete plan; | ||
} | ||
|
||
// Write a serialized Substrait plan to disk in the specified format. | ||
// On error returns a non-empty error message. | ||
// On success a nullptr is returned. | ||
const char* save_substrait_plan( | ||
const unsigned char* plan_data, | ||
int32_t plan_data_length, | ||
const char* filename, | ||
io::substrait::PlanFileFormat format) { | ||
::substrait::proto::Plan plan; | ||
std::string data((const char*) plan_data, plan_data_length); | ||
plan.ParseFromString(data); | ||
auto result = io::substrait::savePlan(plan, filename, format); | ||
if (!result.ok()) { | ||
return result.message().data(); | ||
} | ||
return nullptr; | ||
} | ||
|
||
} // extern "C" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,46 @@ | ||
/* SPDX-License-Identifier: Apache-2.0 */ | ||
|
||
#include <substrait/common/Io.h> | ||
|
||
extern "C" { | ||
|
||
// Since this is actually C code, stick to C style names for exporting. | ||
// NOLINTBEGIN(readability-identifier-naming) | ||
|
||
using SerializedPlan = struct { | ||
// If set, contains a serialized ::substrait::proto::Plan object. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I've become used to There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm not sure I want to formalize this interface. If someone wants to use the C++ code they should be using the recently exposed substrait_io library. This version is for wrapping purposes only so I'd like it to be more obscure. |
||
unsigned char *buffer; | ||
// If buffer is set, this is the size of the buffer. | ||
int32_t size; | ||
// If null the buffer is valid, otherwise this points to a null terminated | ||
// error string. | ||
char *error_message; | ||
}; | ||
|
||
// Load a Substrait plan (in any format) from disk. | ||
// | ||
// Accepts filename as a null-terminated C string. | ||
// Returns a SerializedPlan structure containing either the serialized plan or | ||
// an error message. This SerializedPlan should be freed using | ||
// free_substrait_plan. | ||
SerializedPlan* load_substrait_plan(const char* filename); | ||
EpsilonPrime marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
// Frees a SerializedPlan that was returned from load_substrait_plan. | ||
void free_substrait_plan(SerializedPlan* plan); | ||
EpsilonPrime marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
// Write a serialized Substrait plan to disk in the specified format. | ||
// | ||
// plan_data is a Substrait Plan serialized into a byte array with length | ||
// plan_data_length. | ||
// Filename is a null-terminated C string. | ||
// On error returns a non-empty error message. | ||
// On success an empty string is returned. | ||
const char* save_substrait_plan( | ||
const unsigned char* plan_data, | ||
int32_t plan_data_length, | ||
const char* filename, | ||
io::substrait::PlanFileFormat format); | ||
|
||
// NOLINTEND(readability-identifier-naming) | ||
|
||
} // extern "C" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
# SPDX-License-Identifier: Apache-2.0 | ||
|
||
cmake_path(GET CMAKE_CURRENT_BINARY_DIR PARENT_PATH | ||
CMAKE_CURRENT_BINARY_PARENT_DIR) | ||
|
||
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_PARENT_DIR}) | ||
|
||
add_test_case( | ||
planloader_test | ||
SOURCES | ||
PlanLoaderTest.cpp | ||
EXTRA_LINK_LIBS | ||
planloader | ||
gmock | ||
gtest | ||
gtest_main) | ||
|
||
set(TEXTPLAN_SOURCE_DIR "${CMAKE_SOURCE_DIR}/src/substrait/textplan") | ||
|
||
add_custom_command( | ||
TARGET planloader_test | ||
POST_BUILD | ||
COMMAND ${CMAKE_COMMAND} -E echo "Copying unit test data.." | ||
COMMAND ${CMAKE_COMMAND} -E make_directory | ||
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/tests/data" | ||
COMMAND | ||
${CMAKE_COMMAND} -E copy | ||
"${TEXTPLAN_SOURCE_DIR}/converter/data/q6_first_stage.json" | ||
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/tests/data/q6_first_stage.json") | ||
|
||
message( | ||
STATUS "test data will be here: ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/tests/data" | ||
) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
/* SPDX-License-Identifier: Apache-2.0 */ | ||
|
||
#include <gmock/gmock-matchers.h> | ||
#include <gtest/gtest.h> | ||
#include <functional> | ||
|
||
#include "../planloader.h" | ||
#include "substrait/proto/plan.pb.h" | ||
|
||
namespace io::substrait::textplan { | ||
namespace { | ||
|
||
TEST(PlanLoaderTest, LoadAndSave) { | ||
auto serializedPlan = load_substrait_plan("data/q6_first_stage.json"); | ||
ASSERT_EQ(serializedPlan->error_message, nullptr); | ||
|
||
::substrait::proto::Plan plan; | ||
bool parseStatus = | ||
plan.ParseFromArray(serializedPlan->buffer, serializedPlan->size); | ||
ASSERT_TRUE(parseStatus) << "Failed to parse the plan."; | ||
|
||
const char* saveStatus = save_substrait_plan( | ||
serializedPlan->buffer, | ||
serializedPlan->size, | ||
"outfile.splan", | ||
PlanFileFormat::kText); | ||
ASSERT_EQ(saveStatus, nullptr); | ||
|
||
free_substrait_plan(serializedPlan); | ||
} | ||
|
||
TEST(PlanLoaderTest, LoadMissingFile) { | ||
auto serializedPlan = load_substrait_plan("no_such_file.json"); | ||
ASSERT_THAT( | ||
serializedPlan->error_message, | ||
::testing::StartsWith("Failed to open file no_such_file.json")); | ||
|
||
free_substrait_plan(serializedPlan); | ||
} | ||
|
||
} // namespace | ||
} // namespace io::substrait::textplan |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Why not? Does this mean you can't build the planloader in debug mode (annoying but ok)? Or does this mean that someone linking to the planloader cannot build in debug (probably an issue)?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The problem is all of the sanitization stuff that we're including in debug mode. An unfettered debug mode would be fine.