From a8e86dec5e5ae0462e6694f95ff2f8a3928bf24a Mon Sep 17 00:00:00 2001 From: Chunting Gu Date: Thu, 28 Mar 2019 17:23:48 +0800 Subject: [PATCH] Post/upload files support. --- examples/http_client.cc | 31 +++++++++- webcc/globals.cc | 12 ++-- webcc/globals.h | 15 +++-- webcc/http_client_session.cc | 2 +- webcc/http_message.cc | 2 +- webcc/http_message.h | 4 ++ webcc/http_request_builder.cc | 110 ++++++++++++++++++++++++++++++---- webcc/http_request_builder.h | 37 +++++++++++- webcc/utility.cc | 10 ++++ webcc/utility.h | 2 + webcc/zlib_wrapper.cc | 8 +-- 11 files changed, 201 insertions(+), 32 deletions(-) diff --git a/examples/http_client.cc b/examples/http_client.cc index 60dd8f4..ba84373 100644 --- a/examples/http_client.cc +++ b/examples/http_client.cc @@ -91,7 +91,7 @@ void ExampleCompression() { } // Get an image from HttpBin.org and save to the given file path. -// E.g., ExampleImage("E:\\example.jpeg") +// E.g., ExampleImage("E:\\example.jpg") void ExampleImage(const std::string& path) { HttpClientSession session; @@ -105,6 +105,35 @@ void ExampleImage(const std::string& path) { ofs << r->content(); } +// Post/upload files. +void ExamplePostFiles() { + HttpClientSession session; + + auto r = session.Request(HttpRequestBuilder{} + .Post() + .url("http://httpbin.org/post") + .file_data("file1", "report.xls", "", "application/vnd.ms-excel") + .file_data("file2", "report.xml", "", "text/xml")()); + + std::cout << r->content() << std::endl; +} + +// Post/upload files by file path. +void ExamplePostFiles(const std::string& name, + const std::string& file_name, + const std::string& file_path, + const std::string& content_type) { + HttpClientSession session; + + auto r = + session.Request(HttpRequestBuilder{} + .Post() + .url("http://httpbin.org/post") + .file(name, file_name, file_path, content_type)()); + + std::cout << r->content() << std::endl; +} + int main() { WEBCC_LOG_INIT("", LOG_CONSOLE); diff --git a/webcc/globals.cc b/webcc/globals.cc index 535c750..660a129 100644 --- a/webcc/globals.cc +++ b/webcc/globals.cc @@ -31,6 +31,8 @@ const char* DescribeError(Error error) { return "HTTP error"; case kServerError: return "Server error"; + case kFileIOError: + return "File IO error"; case kXmlError: return "XML error"; default: @@ -38,14 +40,14 @@ const char* DescribeError(Error error) { } } -Exception::Exception(Error error, bool timeout, const std::string& details) - : error_(error), timeout_(timeout), msg_(DescribeError(error)) { - if (timeout) { - msg_ += " (timeout)"; - } +Exception::Exception(Error error, const std::string& details, bool timeout) + : error_(error), msg_(DescribeError(error)), timeout_(timeout) { if (!details.empty()) { msg_ += " (" + details + ")"; } + if (timeout) { + msg_ += " (timeout)"; + } } } // namespace webcc diff --git a/webcc/globals.h b/webcc/globals.h index 86fda3f..04b7b00 100644 --- a/webcc/globals.h +++ b/webcc/globals.h @@ -145,6 +145,9 @@ enum Error { // E.g., HTTP status 500 + SOAP Fault element. kServerError, + // File read/write error. + kFileIOError, + // XML parsing error. kXmlError, }; @@ -154,25 +157,25 @@ const char* DescribeError(Error error); class Exception : public std::exception { public: - explicit Exception(Error error = kNoError, bool timeout = false, - const std::string& details = ""); + explicit Exception(Error error, const std::string& details = "", + bool timeout = false); + + Error error() const { return error_; } // Note that `noexcept` is required by GCC. const char* what() const noexcept override { return msg_.c_str(); } - Error error() const { return error_; } - bool timeout() const { return timeout_; } private: Error error_; + std::string msg_; + // If the error was caused by timeout or not. bool timeout_; - - std::string msg_; }; } // namespace webcc diff --git a/webcc/http_client_session.cc b/webcc/http_client_session.cc index b5d3325..ce25f17 100644 --- a/webcc/http_client_session.cc +++ b/webcc/http_client_session.cc @@ -172,7 +172,7 @@ HttpResponsePtr HttpClientSession::Send(HttpRequestPtr request) { // Remove the failed connection from pool. pool_.Remove(key); } - throw Exception(client->error(), client->timed_out()); + throw Exception(client->error(), "", client->timed_out()); } // Update connection pool. diff --git a/webcc/http_message.cc b/webcc/http_message.cc index d1f9fce..f461962 100644 --- a/webcc/http_message.cc +++ b/webcc/http_message.cc @@ -179,7 +179,7 @@ void HttpMessage::Dump(std::ostream& os, std::size_t indent, } else { // Split by EOL to achieve more readability. std::vector splitted; - boost::split(splitted, content_, boost::is_any_of(kCRLF)); + boost::split(splitted, content_, boost::is_any_of("\n")); std::size_t size = 0; diff --git a/webcc/http_message.h b/webcc/http_message.h index f4b7643..175373a 100644 --- a/webcc/http_message.h +++ b/webcc/http_message.h @@ -108,6 +108,10 @@ public: // Return true if header Accept-Encoding contains "gzip". bool AcceptEncodingGzip() const; + void SetContentType(const std::string& content_type) { + SetHeader(http::headers::kContentType, content_type); + } + // E.g., "text/html", "application/json; charset=utf-8", etc. void SetContentType(const std::string& media_type, const std::string& charset); diff --git a/webcc/http_request_builder.cc b/webcc/http_request_builder.cc index 68f9c49..cf0bd8f 100644 --- a/webcc/http_request_builder.cc +++ b/webcc/http_request_builder.cc @@ -1,10 +1,32 @@ #include "webcc/http_request_builder.h" +#include + #include "webcc/logger.h" +#include "webcc/utility.h" #include "webcc/zlib_wrapper.h" namespace webcc { +// ----------------------------------------------------------------------------- + +// Read entire file into string. +static bool ReadFile(const std::string& path, std::string* output) { + std::ifstream ifs{path, std::ios::binary | std::ios::ate}; + if (!ifs) { + return false; + } + + auto size = ifs.tellg(); + output->resize(size, '\0'); + ifs.seekg(0); + ifs.read(&(*output)[0], size); // TODO: Error handling + + return true; +} + +// ----------------------------------------------------------------------------- + HttpRequestPtr HttpRequestBuilder::operator()() { assert(parameters_.size() % 2 == 0); assert(headers_.size() % 2 == 0); @@ -25,26 +47,90 @@ HttpRequestPtr HttpRequestBuilder::operator()() { } if (!data_.empty()) { - if (gzip_ && data_.size() > kGzipThreshold) { - std::string compressed; - if (Compress(data_, &compressed)) { - request->SetContent(std::move(compressed), true); - request->SetHeader(http::headers::kContentEncoding, "gzip"); - } else { - LOG_WARN("Cannot compress the content data!"); - request->SetContent(std::move(data_), true); - } - } else { - request->SetContent(std::move(data_), true); - } + SetContent(request, std::move(data_)); // TODO: Request-level charset. if (json_) { request->SetContentType(http::media_types::kApplicationJson, ""); } + } else if (!files_.empty()) { + // Another choice to generate the boundary is what Apache does, see: + // https://stackoverflow.com/a/5686863 + const std::string boundary = RandomUuid(); + + request->SetContentType("multipart/form-data; boundary=" + boundary); + + std::string data; + CreateFormData(&data, boundary); + + // Ingore gzip since most servers don't support it. + request->SetContent(std::move(data), true); } return request; } +HttpRequestBuilder& HttpRequestBuilder::file(const std::string& name, + const std::string& file_name, + const std::string& file_path, + const std::string& content_type) { + std::string file_data; + if (!ReadFile(file_path, &file_data)) { + throw Exception(kFileIOError, "Cannot read the file."); + } + + files_.push_back({name, file_name, std::move(file_data), content_type}); + + return *this; +} + +void HttpRequestBuilder::SetContent(HttpRequestPtr request, + std::string&& data) { + if (gzip_ && data.size() > kGzipThreshold) { + std::string compressed; + if (Compress(data, &compressed)) { + request->SetContent(std::move(compressed), true); + request->SetHeader(http::headers::kContentEncoding, "gzip"); + return; + } + + LOG_WARN("Cannot compress the content data!"); + } + + request->SetContent(std::move(data), true); +} + +void HttpRequestBuilder::CreateFormData(std::string* data, + const std::string& boundary) { + for (File& file : files_) { + data->append("--" + boundary + kCRLF); + + // Content-Disposition header + data->append("Content-Disposition: form-data"); + if (!file.name.empty()) { + data->append("; name=\"" + file.name + "\""); + } + if (!file.file_name.empty()) { + data->append("; filename=\"" + file.file_name + "\""); + } + data->append(kCRLF); + + // Content-Type header + if (!file.content_type.empty()) { + data->append("Content-Type: " + file.content_type); + data->append(kCRLF); + } + + data->append(kCRLF); + + // Payload + data->append(file.file_data); + + data->append(kCRLF); + } + + data->append("--" + boundary + "--"); + data->append(kCRLF); +} + } // namespace webcc diff --git a/webcc/http_request_builder.h b/webcc/http_request_builder.h index 8bc7712..26e2f67 100644 --- a/webcc/http_request_builder.h +++ b/webcc/http_request_builder.h @@ -51,12 +51,27 @@ public: return *this; } - HttpRequestBuilder& json(bool json) { + HttpRequestBuilder& json(bool json = true) { json_ = json; return *this; } - HttpRequestBuilder& gzip(bool gzip) { + // Upload a file with its path. + HttpRequestBuilder& file(const std::string& name, + const std::string& file_name, + const std::string& file_path, // TODO: UNICODE + const std::string& content_type = ""); + + // Upload a file with its data. + HttpRequestBuilder& file_data(const std::string& name, + const std::string& file_name, + std::string&& file_data, + const std::string& content_type = "") { + files_.push_back({name, file_name, file_data, content_type}); + return *this; + } + + HttpRequestBuilder& gzip(bool gzip = true) { gzip_ = gzip; return *this; } @@ -73,6 +88,11 @@ public: return *this; } +private: + void SetContent(HttpRequestPtr request, std::string&& data); + + void CreateFormData(std::string* data, const std::string& boundary); + private: std::string method_; @@ -87,6 +107,19 @@ private: // Is the data to send a JSON string? bool json_ = false; + // Examples: + // { "images", "example.jpg", "BinaryData", "image/jpeg" } + // { "file", "report.csv", "BinaryData", "" } + struct File { + std::string name; + std::string file_name; + std::string file_data; // Binary file data + std::string content_type; + }; + + // Files to upload for a POST (or PUT?) request. + std::vector files_; + // Compress the request content. // NOTE: Most servers don't support compressed requests. // Even the requests module from Python doesn't have a built-in support. diff --git a/webcc/utility.cc b/webcc/utility.cc index 93e780d..75b65bd 100644 --- a/webcc/utility.cc +++ b/webcc/utility.cc @@ -6,6 +6,9 @@ #include #include +#include "boost/uuid/random_generator.hpp" +#include "boost/uuid/uuid_io.hpp" + #include "webcc/logger.h" using tcp = boost::asio::ip::tcp; @@ -44,4 +47,11 @@ std::string GetHttpDateTimestamp() { return ss.str(); } +std::string RandomUuid() { + boost::uuids::uuid u = boost::uuids::random_generator()(); + std::stringstream ss; + ss << u; + return ss.str(); +} + } // namespace webcc diff --git a/webcc/utility.h b/webcc/utility.h index 68e5656..bb91b32 100644 --- a/webcc/utility.h +++ b/webcc/utility.h @@ -22,6 +22,8 @@ std::string EndpointToString(const TcpEndpoint& endpoint); // See: https://tools.ietf.org/html/rfc7231#section-7.1.1.2 std::string GetHttpDateTimestamp(); +std::string RandomUuid(); + } // namespace webcc #endif // WEBCC_UTILITY_H_ diff --git a/webcc/zlib_wrapper.cc b/webcc/zlib_wrapper.cc index dbf5567..1a10efd 100644 --- a/webcc/zlib_wrapper.cc +++ b/webcc/zlib_wrapper.cc @@ -39,9 +39,7 @@ bool Compress(const std::string& input, std::string* output) { int err = deflate(&stream, Z_FINISH); - assert(err != Z_STREAM_ERROR); - - if (err != Z_OK) { + if (err != Z_OK && err != Z_STREAM_END) { deflateEnd(&stream); if (stream.msg != nullptr) { LOG_ERRO("zlib deflate error: %s", stream.msg); @@ -51,6 +49,7 @@ bool Compress(const std::string& input, std::string* output) { std::size_t size = buf.size() - stream.avail_out; output->insert(output->end(), buf.data(), buf.data() + size); + } while (stream.avail_out == 0); if (deflateEnd(&stream) != Z_OK) { @@ -105,7 +104,8 @@ bool Decompress(const std::string& input, std::string* output) { if (err == Z_STREAM_END) { break; - } else if (err != Z_OK) { + } + if (err != Z_OK) { inflateEnd(&stream); if (stream.msg != nullptr) { LOG_ERRO("zlib inflate error: %s", stream.msg);