diff options
author | Abhilash Raju <abhilash.kollam@gmail.com> | 2023-11-09 07:32:44 +0300 |
---|---|---|
committer | Ed Tanous <ed@tanous.net> | 2024-01-22 22:15:20 +0300 |
commit | b5f288d294e17719f30e32acc40e07681baf04b9 (patch) | |
tree | 7ccfd67ef408e0fc5d508ac9266040d3d4a879fe | |
parent | ee192c065cc658b2489d04c791e2fb956a331699 (diff) | |
download | bmcweb-b5f288d294e17719f30e32acc40e07681baf04b9.tar.xz |
Make use of filebody for dump offload
Logservice has been rewritten to use file_body to offload dump files
from BMC.
There are two kind of dump files, BMC dump and System dump.While BMC
dump just requires default support from beast::file_body, System dump
requires base64 encoding support from beast. But beast::file_body do not
have ready-made support for base64 encoding. So a custom file_body has
been written for the base64 encoding.
The openFile apis in crow::Response do not have support for unix file
descriptor. Since dump files are accesses via descriptors, added new
openFile api that accepts descriptors.
Tested:
Functionality test have been executed to verify the bmc dump offload.
Did sanity test by invoking bmcweb pages via browser.
Change-Id: I24192657c03d8b2f0394d31e7424c6796ba3227a
Signed-off-by: Abhilash Raju <abhilash.kollam@gmail.com>
-rw-r--r-- | http/http_file_body.hpp | 136 | ||||
-rw-r--r-- | http/http_response.hpp | 34 | ||||
-rw-r--r-- | redfish-core/lib/log_services.hpp | 81 | ||||
-rw-r--r-- | test/http/http_response_test.cpp | 156 |
4 files changed, 354 insertions, 53 deletions
diff --git a/http/http_file_body.hpp b/http/http_file_body.hpp new file mode 100644 index 0000000000..eaafd5dbeb --- /dev/null +++ b/http/http_file_body.hpp @@ -0,0 +1,136 @@ +#pragma once + +#include "utility.hpp" + +#include <boost/beast/core/file_posix.hpp> +#include <boost/beast/http/message.hpp> +#include <boost/system/error_code.hpp> + +namespace bmcweb +{ +struct FileBody +{ + class writer; + class value_type; + + static std::uint64_t size(const value_type& body); +}; + +enum class EncodingType +{ + Raw, + Base64, +}; + +class FileBody::value_type +{ + boost::beast::file_posix fileHandle; + + std::uint64_t fileSize = 0; + + public: + EncodingType encodingType = EncodingType::Raw; + + ~value_type() = default; + value_type() = default; + explicit value_type(EncodingType enc) : encodingType(enc) {} + value_type(value_type&& other) = default; + value_type& operator=(value_type&& other) = default; + value_type(const value_type& other) = delete; + value_type& operator=(const value_type& other) = delete; + + boost::beast::file_posix& file() + { + return fileHandle; + } + + std::uint64_t size() const + { + return fileSize; + } + + void open(const char* path, boost::beast::file_mode mode, + boost::system::error_code& ec) + { + fileHandle.open(path, mode, ec); + fileSize = fileHandle.size(ec); + } + + void setFd(int fd, boost::system::error_code& ec) + { + fileHandle.native_handle(fd); + fileSize = fileHandle.size(ec); + } +}; + +inline std::uint64_t FileBody::size(const value_type& body) +{ + return body.size(); +} + +class FileBody::writer +{ + public: + using const_buffers_type = boost::asio::const_buffer; + + private: + std::string buf; + crow::utility::Base64Encoder encoder; + + value_type& body; + std::uint64_t remain; + constexpr static size_t readBufSize = 4096; + std::array<char, readBufSize> fileReadBuf{}; + + public: + template <bool IsRequest, class Fields> + writer(boost::beast::http::header<IsRequest, Fields>& /*header*/, + value_type& bodyIn) : + body(bodyIn), + remain(body.size()) + {} + + static void init(boost::beast::error_code& ec) + { + ec = {}; + } + + boost::optional<std::pair<const_buffers_type, bool>> + get(boost::beast::error_code& ec) + { + size_t toRead = fileReadBuf.size(); + if (remain < toRead) + { + toRead = static_cast<size_t>(remain); + } + size_t read = body.file().read(fileReadBuf.data(), toRead, ec); + if (read != toRead || ec) + { + return boost::none; + } + remain -= read; + + std::string_view chunkView(fileReadBuf.data(), read); + + std::pair<const_buffers_type, bool> ret; + ret.second = remain > 0; + if (body.encodingType == EncodingType::Base64) + { + buf.clear(); + buf.reserve( + crow::utility::Base64Encoder::encodedSize(chunkView.size())); + encoder.encode(chunkView, buf); + if (!ret.second) + { + encoder.finalize(buf); + } + ret.first = const_buffers_type(buf.data(), buf.size()); + } + else + { + ret.first = const_buffers_type(chunkView.data(), chunkView.size()); + } + return ret; + } +}; +} // namespace bmcweb diff --git a/http/http_response.hpp b/http/http_response.hpp index ec54a90c4b..40158992e4 100644 --- a/http/http_response.hpp +++ b/http/http_response.hpp @@ -1,8 +1,8 @@ #pragma once +#include "http_file_body.hpp" #include "logging.hpp" #include "utils/hex_utils.hpp" -#include <boost/beast/http/file_body.hpp> #include <boost/beast/http/message.hpp> #include <boost/beast/http/message_generator.hpp> #include <boost/beast/http/string_body.hpp> @@ -27,7 +27,7 @@ struct Response friend class crow::Connection; using string_response = http::response<http::string_body>; - using file_response = http::response<http::file_body>; + using file_response = http::response<bmcweb::FileBody>; // Use boost variant2 because it doesn't have valueless by exception boost::variant2::variant<string_response, file_response> response; @@ -356,25 +356,45 @@ struct Response response); } - bool openFile(const std::filesystem::path& path) + bool openFile(const std::filesystem::path& path, + bmcweb::EncodingType enc = bmcweb::EncodingType::Raw) { - http::file_body::value_type file; + file_response::body_type::value_type body(enc); boost::beast::error_code ec; - file.open(path.c_str(), boost::beast::file_mode::read, ec); + body.open(path.c_str(), boost::beast::file_mode::read, ec); if (ec) { return false; } + updateFileBody(std::move(body)); + return true; + } + + bool openFd(int fd, bmcweb::EncodingType enc = bmcweb::EncodingType::Raw) + { + file_response::body_type::value_type body(enc); + boost::beast::error_code ec; + body.setFd(fd, ec); + if (ec) + { + BMCWEB_LOG_ERROR("Failed to set fd"); + return false; + } + updateFileBody(std::move(body)); + return true; + } + + private: + void updateFileBody(file_response::body_type::value_type file) + { // store the headers on stack temporarily so we can reconstruct the new // base with the old headers copied in. http::header<false> headTemp = std::move(fields()); file_response& fileResponse = response.emplace<file_response>(std::move(headTemp)); fileResponse.body() = std::move(file); - return true; } - private: std::optional<std::string> expectedHash; bool completed = false; std::function<void(Response&)> completeRequestHandler; diff --git a/redfish-core/lib/log_services.hpp b/redfish-core/lib/log_services.hpp index 371ae44867..24f0251ee1 100644 --- a/redfish-core/lib/log_services.hpp +++ b/redfish-core/lib/log_services.hpp @@ -45,6 +45,7 @@ #include <array> #include <charconv> +#include <cstddef> #include <filesystem> #include <iterator> #include <optional> @@ -745,7 +746,35 @@ inline void deleteDumpEntry(const std::shared_ptr<bmcweb::AsyncResp>& asyncResp, std::format("{}/entry/{}", getDumpPath(dumpType), entryID), "xyz.openbmc_project.Object.Delete", "Delete"); } +inline bool checkSizeLimit(int fd, crow::Response& res) +{ + long long int size = lseek(fd, 0, SEEK_END); + if (size <= 0) + { + BMCWEB_LOG_ERROR("Failed to get size of file, lseek() returned {}", + size); + messages::internalError(res); + return false; + } + // Arbitrary max size of 20MB to accommodate BMC dumps + constexpr long long int maxFileSize = 20LL * 1024LL * 1024LL; + if (size > maxFileSize) + { + BMCWEB_LOG_ERROR("File size {} exceeds maximum allowed size of {}", + size, maxFileSize); + messages::internalError(res); + return false; + } + off_t rc = lseek(fd, 0, SEEK_SET); + if (rc < 0) + { + BMCWEB_LOG_ERROR("Failed to reset file offset to 0"); + messages::internalError(res); + return false; + } + return true; +} inline void downloadEntryCallback(const std::shared_ptr<bmcweb::AsyncResp>& asyncResp, const std::string& entryID, @@ -781,59 +810,29 @@ inline void messages::internalError(asyncResp->res); return; } - - long long int size = lseek(fd, 0, SEEK_END); - if (size <= 0) + if (!checkSizeLimit(fd, asyncResp->res)) { - BMCWEB_LOG_ERROR("Failed to get size of file, lseek() returned {}", - size); - messages::internalError(asyncResp->res); close(fd); return; } - - // Arbitrary max size of 20MB to accommodate BMC dumps - constexpr int maxFileSize = 20 * 1024 * 1024; - if (size > maxFileSize) - { - BMCWEB_LOG_ERROR("File size {} exceeds maximum allowed size of {}", - size, maxFileSize); - messages::internalError(asyncResp->res); - close(fd); - return; - } - long long int rc = lseek(fd, 0, SEEK_SET); - if (rc < 0) + if (downloadEntryType == "System") { - BMCWEB_LOG_ERROR("Failed to reset file offset to 0"); - messages::internalError(asyncResp->res); - close(fd); + if (!asyncResp->res.openFd(fd, bmcweb::EncodingType::Base64)) + { + messages::internalError(asyncResp->res); + close(fd); + return; + } + asyncResp->res.addHeader( + boost::beast::http::field::content_transfer_encoding, "Base64"); return; } - - std::string body; - body.resize(static_cast<size_t>(size), '\0'); - rc = read(fd, body.data(), body.size()); - if ((rc == -1) || (rc != size)) + if (!asyncResp->res.openFd(fd)) { - BMCWEB_LOG_ERROR("Failed to read in file"); messages::internalError(asyncResp->res); close(fd); return; } - close(fd); - if (downloadEntryType == "System") - { - // Base64 encode response. - asyncResp->res.write(crow::utility::base64encode(body)); - asyncResp->res.addHeader( - boost::beast::http::field::content_transfer_encoding, "Base64"); - } - else - { - asyncResp->res.write(std::move(body)); - } - asyncResp->res.addHeader(boost::beast::http::field::content_type, "application/octet-stream"); } diff --git a/test/http/http_response_test.cpp b/test/http/http_response_test.cpp index c644ea9c05..1ee3853f78 100644 --- a/test/http/http_response_test.cpp +++ b/test/http/http_response_test.cpp @@ -22,17 +22,76 @@ void verifyHeaders(crow::Response& res) EXPECT_EQ(res.result(), boost::beast::http::status::ok); } -std::string makeFile() +std::string makeFile(std::string_view sampleData) { std::filesystem::path path = std::filesystem::temp_directory_path(); path /= "bmcweb_http_response_test_XXXXXXXXXXX"; std::string stringPath = path.string(); int fd = mkstemp(stringPath.data()); EXPECT_GT(fd, 0); - std::string_view sample = "sample text"; - EXPECT_EQ(write(fd, sample.data(), sample.size()), sample.size()); + EXPECT_EQ(write(fd, sampleData.data(), sampleData.size()), + sampleData.size()); + close(fd); return stringPath; } + +void readHeader(boost::beast::http::serializer<false, bmcweb::FileBody>& sr) +{ + while (!sr.is_header_done()) + { + boost::system::error_code ec; + sr.next(ec, [&sr](const boost::system::error_code& ec2, + const auto& buffer) { + ASSERT_FALSE(ec2); + sr.consume(boost::beast::buffer_bytes(buffer)); + }); + ASSERT_FALSE(ec); + } +} + +std::string collectFromBuffers( + const auto& buffer, + boost::beast::http::serializer<false, bmcweb::FileBody>& sr) +{ + std::string ret; + + for (auto iter = boost::asio::buffer_sequence_begin(buffer); + iter != boost::asio::buffer_sequence_end(buffer); ++iter) + { + const auto& innerBuf = *iter; + auto view = std::string_view(static_cast<const char*>(innerBuf.data()), + innerBuf.size()); + ret += view; + sr.consume(innerBuf.size()); + } + return ret; +} + +std::string + readBody(boost::beast::http::serializer<false, bmcweb::FileBody>& sr) +{ + std::string ret; + while (!sr.is_done()) + { + boost::system::error_code ec; + sr.next(ec, [&sr, &ret](const boost::system::error_code& ec2, + const auto& buffer) { + ASSERT_FALSE(ec2); + ret += collectFromBuffers(buffer, sr); + }); + EXPECT_FALSE(ec); + } + + return ret; +} +std::string getData(crow::Response::file_response& m) +{ + boost::beast::http::serializer<false, bmcweb::FileBody> sr{m}; + std::stringstream ret; + sr.split(true); + readHeader(sr); + return readBody(sr); +} TEST(HttpResponse, Defaults) { crow::Response res; @@ -60,17 +119,41 @@ TEST(HttpResponse, FileBody) { crow::Response res; addHeaders(res); - std::string path = makeFile(); + std::string path = makeFile("sample text"); res.openFile(path); verifyHeaders(res); std::filesystem::remove(path); } +TEST(HttpResponse, FileBodyWithFd) +{ + crow::Response res; + addHeaders(res); + std::string path = makeFile("sample text"); + FILE* fd = fopen(path.c_str(), "r+"); + res.openFd(fileno(fd)); + verifyHeaders(res); + fclose(fd); + std::filesystem::remove(path); +} + +TEST(HttpResponse, Base64FileBodyWithFd) +{ + crow::Response res; + addHeaders(res); + std::string path = makeFile("sample text"); + FILE* fd = fopen(path.c_str(), "r+"); + res.openFd(fileno(fd), bmcweb::EncodingType::Base64); + verifyHeaders(res); + fclose(fd); + std::filesystem::remove(path); +} + TEST(HttpResponse, BodyTransitions) { crow::Response res; addHeaders(res); - std::string path = makeFile(); + std::string path = makeFile("sample text"); res.openFile(path); EXPECT_EQ(boost::variant2::holds_alternative<crow::Response::file_response>( @@ -88,4 +171,67 @@ TEST(HttpResponse, BodyTransitions) verifyHeaders(res); std::filesystem::remove(path); } + +void testFileData(crow::Response& res, const std::string& data) +{ + auto& fb = + boost::variant2::get<crow::Response::file_response>(res.response); + EXPECT_EQ(getData(fb), data); +} + +TEST(HttpResponse, Base64FileBodyWriter) +{ + crow::Response res; + std::string data = "sample text"; + std::string path = makeFile(data); + FILE* f = fopen(path.c_str(), "r+"); + res.openFd(fileno(f), bmcweb::EncodingType::Base64); + testFileData(res, crow::utility::base64encode(data)); + fclose(f); + std::filesystem::remove(path); +} + +std::string generateBigdata() +{ + std::string result; + while (result.size() < 10000) + { + result += "sample text"; + } + return result; +} + +TEST(HttpResponse, Base64FileBodyWriterLarge) +{ + crow::Response res; + std::string data = generateBigdata(); + std::string path = makeFile(data); + { + boost::beast::file_posix file; + boost::system::error_code ec; + file.open(path.c_str(), boost::beast::file_mode::read, ec); + EXPECT_EQ(ec.value(), 0); + res.openFd(file.native_handle(), bmcweb::EncodingType::Base64); + testFileData(res, crow::utility::base64encode(data)); + } + + std::filesystem::remove(path); +} + +TEST(HttpResponse, FileBodyWriterLarge) +{ + crow::Response res; + std::string data = generateBigdata(); + std::string path = makeFile(data); + { + boost::beast::file_posix file; + boost::system::error_code ec; + file.open(path.c_str(), boost::beast::file_mode::read, ec); + EXPECT_EQ(ec.value(), 0); + res.openFd(file.native_handle()); + testFileData(res, data); + } + std::filesystem::remove(path); +} + } // namespace |