From 2a868c4dc1038488aff89cd5e37b8284f8a0ed08 Mon Sep 17 00:00:00 2001 From: Chunting Gu Date: Mon, 22 Apr 2019 14:43:46 +0800 Subject: [PATCH] Add an option to disable GZIP. --- CMakeLists.txt | 23 ++++++++------- autotest/CMakeLists.txt | 13 +++++---- autotest/client_autotest.cc | 11 ++++++- examples/CMakeLists.txt | 12 ++++---- unittest/CMakeLists.txt | 18 +++++++----- unittest/parser_unittest.cc | 3 +- webcc/CMakeLists.txt | 4 +++ webcc/client_session.cc | 5 ++++ webcc/config.h.example | 5 +++- webcc/config.h.in | 3 ++ webcc/globals.h | 2 ++ webcc/{zlib_wrapper.cc => gzip.cc} | 4 ++- webcc/{zlib_wrapper.h => gzip.h} | 8 +++-- webcc/parser.cc | 20 +++++++++++-- webcc/request.cc | 5 ---- webcc/request.h | 1 - webcc/request_builder.cc | 9 ++++-- webcc/rest_request_handler.cc | 47 +++++++++++++++++------------- webcc/rest_request_handler.h | 3 ++ webcc/utility.h | 6 ---- 20 files changed, 129 insertions(+), 73 deletions(-) rename webcc/{zlib_wrapper.cc => gzip.cc} (98%) rename webcc/{zlib_wrapper.h => gzip.h} (75%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 90a68ac..02372b2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,7 @@ set(WEBCC_ENABLE_LOG 1 CACHE STRING "Enable logging? (1:Yes, 0:No)") set(WEBCC_LOG_LEVEL 2 CACHE STRING "Log level (0:VERB, 1:INFO, 2:WARN, 3:ERRO or 4:FATA)") set(WEBCC_ENABLE_SSL 0 CACHE STRING "Enable SSL/HTTPS (need OpenSSL)? (1:Yes, 0:No)") +set(WEBCC_ENABLE_GZIP 0 CACHE STRING "Enable gzip compression (need Zlib)? (1:Yes, 0:No)") if(WEBCC_ENABLE_UNITTEST) enable_testing() @@ -111,18 +112,20 @@ endif() include_directories(${THIRD_PARTY_DIR}/src) -# For using zlib on Windows. -if(WIN32) - add_subdirectory(${THIRD_PARTY_DIR}/src/zlib) +if(WEBCC_ENABLE_GZIP) + # For using zlib on Windows. + if(WIN32) + add_subdirectory(${THIRD_PARTY_DIR}/src/zlib) - include_directories(${THIRD_PARTY_DIR}/src/zlib) + include_directories(${THIRD_PARTY_DIR}/src/zlib) - # For including CMake generated zconf.h. - include_directories(${PROJECT_BINARY_DIR}/third_party/src/zlib) -else() - find_package(ZLIB REQUIRED) - if(ZLIB_FOUND) - include_directories(${ZLIB_INCLUDE_DIRS}) + # For including CMake generated zconf.h. + include_directories(${PROJECT_BINARY_DIR}/third_party/src/zlib) + else() + find_package(ZLIB REQUIRED) + if(ZLIB_FOUND) + include_directories(${ZLIB_INCLUDE_DIRS}) + endif() endif() endif() diff --git a/autotest/CMakeLists.txt b/autotest/CMakeLists.txt index ff416c9..4cc6d0c 100644 --- a/autotest/CMakeLists.txt +++ b/autotest/CMakeLists.txt @@ -10,14 +10,15 @@ set(AT_TARGET_NAME webcc_autotest) set(AT_LIBS webcc ${Boost_LIBRARIES} "${CMAKE_THREAD_LIBS_INIT}") if(WEBCC_ENABLE_SSL) - set(AT_LIBS ${AT_LIBS} ${OPENSSL_LIBRARIES}) + set(AT_LIBS ${AT_LIBS} ${OPENSSL_LIBRARIES} crypt32) endif() -if(WIN32) - # TODO: crypt32 should be ssl related. - set(AT_LIBS ${AT_LIBS} zlibstatic crypt32) -else() - set(AT_LIBS ${AT_LIBS} ${ZLIB_LIBRARIES}) +if(WEBCC_ENABLE_GZIP) + if(WIN32) + set(AT_LIBS ${AT_LIBS} zlibstatic) + else() + set(AT_LIBS ${AT_LIBS} ${ZLIB_LIBRARIES}) + endif() endif() if(UNIX) diff --git a/autotest/client_autotest.cc b/autotest/client_autotest.cc index 70aec7a..262b3fa 100644 --- a/autotest/client_autotest.cc +++ b/autotest/client_autotest.cc @@ -48,8 +48,13 @@ static void AssertGet(webcc::ResponsePtr r) { Json::Value headers = json["headers"]; EXPECT_EQ("application/json", headers["Accept"].asString()); - EXPECT_EQ("gzip, deflate", headers["Accept-Encoding"].asString()); EXPECT_EQ("httpbin.org", headers["Host"].asString()); + +#if WEBCC_ENABLE_GZIP + EXPECT_EQ("gzip, deflate", headers["Accept-Encoding"].asString()); +#else + EXPECT_EQ("identity", headers["Accept-Encoding"].asString()); +#endif // WEBCC_ENABLE_GZIP } TEST(ClientTest, Get_RequestFunc) { @@ -110,6 +115,8 @@ TEST(ClientTest, Get_SSL) { // ----------------------------------------------------------------------------- +#if WEBCC_ENABLE_GZIP + // Test Gzip compressed response. TEST(ClientTest, Compression_Gzip) { webcc::ClientSession session; @@ -142,6 +149,8 @@ TEST(ClientTest, Compression_Deflate) { } } +#endif // WEBCC_ENABLE_GZIP + // ----------------------------------------------------------------------------- // Test persistent (keep-alive) connections. diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 6bab5d8..2bb9b1c 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -4,13 +4,15 @@ set(EXAMPLE_LIBS webcc ${Boost_LIBRARIES} "${CMAKE_THREAD_LIBS_INIT}") if(WEBCC_ENABLE_SSL) - set(EXAMPLE_LIBS ${EXAMPLE_LIBS} ${OPENSSL_LIBRARIES}) + set(EXAMPLE_LIBS ${EXAMPLE_LIBS} ${OPENSSL_LIBRARIES} crypt32) endif() -if(WIN32) - set(EXAMPLE_LIBS ${EXAMPLE_LIBS} zlibstatic crypt32) -else() - set(EXAMPLE_LIBS ${EXAMPLE_LIBS} ${ZLIB_LIBRARIES}) +if(WEBCC_ENABLE_GZIP) + if(WIN32) + set(EXAMPLE_LIBS ${EXAMPLE_LIBS} zlibstatic) + else() + set(EXAMPLE_LIBS ${EXAMPLE_LIBS} ${ZLIB_LIBRARIES}) + endif() endif() if(UNIX) diff --git a/unittest/CMakeLists.txt b/unittest/CMakeLists.txt index 9525efe..f1f5b61 100644 --- a/unittest/CMakeLists.txt +++ b/unittest/CMakeLists.txt @@ -10,24 +10,26 @@ set(UT_SRCS set(UT_TARGET_NAME webcc_unittest) # Common libraries to link. -set(TEST_LIBS webcc ${Boost_LIBRARIES} "${CMAKE_THREAD_LIBS_INIT}") +set(UT_LIBS webcc ${Boost_LIBRARIES} "${CMAKE_THREAD_LIBS_INIT}") if(WEBCC_ENABLE_SSL) - set(TEST_LIBS ${TEST_LIBS} ${OPENSSL_LIBRARIES}) + set(UT_LIBS ${UT_LIBS} ${OPENSSL_LIBRARIES} crypt32) endif() -if(WIN32) - set(TEST_LIBS ${TEST_LIBS} zlibstatic crypt32) -else() - set(TEST_LIBS ${TEST_LIBS} ${ZLIB_LIBRARIES}) +if(WEBCC_ENABLE_GZIP) + if(WIN32) + set(UT_LIBS ${UT_LIBS} zlibstatic) + else() + set(UT_LIBS ${UT_LIBS} ${ZLIB_LIBRARIES}) + endif() endif() if(UNIX) # Add `-ldl` for Linux to avoid "undefined reference to `dlopen'". - set(TEST_LIBS ${TEST_LIBS} ${CMAKE_DL_LIBS}) + set(UT_LIBS ${UT_LIBS} ${CMAKE_DL_LIBS}) endif() add_executable(${UT_TARGET_NAME} ${UT_SRCS}) -target_link_libraries(${UT_TARGET_NAME} webcc gtest ${TEST_LIBS}) +target_link_libraries(${UT_TARGET_NAME} webcc gtest ${UT_LIBS}) add_test(${UT_TARGET_NAME} ${UT_TARGET_NAME}) diff --git a/unittest/parser_unittest.cc b/unittest/parser_unittest.cc index 072c759..f47bd97 100644 --- a/unittest/parser_unittest.cc +++ b/unittest/parser_unittest.cc @@ -8,7 +8,7 @@ #include // ----------------------------------------------------------------------------- -#if 0 + // HTTP GET request parser test fixture. class GetRequestParserTest : public testing::Test { protected: @@ -145,7 +145,6 @@ TEST_F(PostRequestParserTest, ParseByteWise) { CheckResult(); } -#endif // ----------------------------------------------------------------------------- diff --git a/webcc/CMakeLists.txt b/webcc/CMakeLists.txt index 3ebaf55..4dc8772 100644 --- a/webcc/CMakeLists.txt +++ b/webcc/CMakeLists.txt @@ -18,6 +18,10 @@ file(GLOB SRCS ${CMAKE_CURRENT_SOURCE_DIR}/*.cc ${CMAKE_CURRENT_SOURCE_DIR}/*.h) +if(NOT WEBCC_ENABLE_GZIP) + list(REMOVE_ITEM SRCS "gzip.h" "gzip.cc") +endif() + set(TARGET webcc) add_library(${TARGET} STATIC ${SRCS}) diff --git a/webcc/client_session.cc b/webcc/client_session.cc index 0e3bf2b..b1fc564 100644 --- a/webcc/client_session.cc +++ b/webcc/client_session.cc @@ -158,6 +158,7 @@ void ClientSession::InitHeaders() { // Content-Encoding Tokens: // (https://en.wikipedia.org/wiki/HTTP_compression) + // // * compress ¨C UNIX "compress" program method (historic; deprecated in most // applications and replaced by gzip or deflate); // * deflate ¨C compression based on the deflate algorithm, a combination of @@ -177,7 +178,11 @@ void ClientSession::InitHeaders() { // the raw deflate compressed data format. // Simply put, "deflate" is not recommended for HTTP 1.1 encoding. +#if WEBCC_ENABLE_GZIP headers_.Set(kAcceptEncoding, "gzip, deflate"); +#else + headers_.Set(kAcceptEncoding, "identity"); +#endif // WEBCC_ENABLE_GZIP headers_.Set(kAccept, "*/*"); diff --git a/webcc/config.h.example b/webcc/config.h.example index b1a234e..e3c6777 100644 --- a/webcc/config.h.example +++ b/webcc/config.h.example @@ -12,6 +12,9 @@ #endif // Set 1/0 to enable/disable SSL/HTTPS. -#define WEBCC_ENABLE_SSL 1 +#define WEBCC_ENABLE_SSL 0 + +// Set 1/0 to enable/disable GZIP compression. +#define WEBCC_ENABLE_GZIP 0 #endif // WEBCC_CONFIG_H_ diff --git a/webcc/config.h.in b/webcc/config.h.in index 1a41d62..91a0789 100644 --- a/webcc/config.h.in +++ b/webcc/config.h.in @@ -16,4 +16,7 @@ // Set 1/0 to enable/disable SSL/HTTPS. #define WEBCC_ENABLE_SSL @WEBCC_ENABLE_SSL@ +// Set 1/0 to enable/disable GZIP compression. +#define WEBCC_ENABLE_GZIP @WEBCC_ENABLE_GZIP@ + #endif // WEBCC_CONFIG_H_ diff --git a/webcc/globals.h b/webcc/globals.h index 9e43866..6b9cb40 100644 --- a/webcc/globals.h +++ b/webcc/globals.h @@ -5,6 +5,8 @@ #include #include +#include "webcc/config.h" + // ----------------------------------------------------------------------------- // Macros diff --git a/webcc/zlib_wrapper.cc b/webcc/gzip.cc similarity index 98% rename from webcc/zlib_wrapper.cc rename to webcc/gzip.cc index 1a10efd..ed18281 100644 --- a/webcc/zlib_wrapper.cc +++ b/webcc/gzip.cc @@ -1,4 +1,4 @@ -#include "webcc/zlib_wrapper.h" +#include "webcc/gzip.h" #include #include // std::move @@ -8,6 +8,7 @@ #include "webcc/logger.h" namespace webcc { +namespace gzip { bool Compress(const std::string& input, std::string* output) { output->clear(); @@ -125,4 +126,5 @@ bool Decompress(const std::string& input, std::string* output) { return true; } +} // namespace gzip } // namespace webcc diff --git a/webcc/zlib_wrapper.h b/webcc/gzip.h similarity index 75% rename from webcc/zlib_wrapper.h rename to webcc/gzip.h index 26b3c9d..e728649 100644 --- a/webcc/zlib_wrapper.h +++ b/webcc/gzip.h @@ -1,9 +1,10 @@ -#ifndef WEBCC_ZLIB_WRAPPER_H_ -#define WEBCC_ZLIB_WRAPPER_H_ +#ifndef WEBCC_GZIP_H_ +#define WEBCC_GZIP_H_ #include namespace webcc { +namespace gzip { // Compress the input string to gzip format output. bool Compress(const std::string& input, std::string* output); @@ -12,6 +13,7 @@ bool Compress(const std::string& input, std::string* output); // formats. bool Decompress(const std::string& input, std::string* output); +} // namespace gzip } // namespace webcc -#endif // WEBCC_ZLIB_WRAPPER_H_ +#endif // WEBCC_GZIP_H_ diff --git a/webcc/parser.cc b/webcc/parser.cc index b0f37e0..a678f75 100644 --- a/webcc/parser.cc +++ b/webcc/parser.cc @@ -5,7 +5,10 @@ #include "webcc/logger.h" #include "webcc/message.h" #include "webcc/utility.h" -#include "webcc/zlib_wrapper.h" + +#if WEBCC_ENABLE_GZIP +#include "webcc/gzip.h" +#endif namespace webcc { @@ -323,16 +326,29 @@ bool Parser::Finish() { return true; } +#if WEBCC_ENABLE_GZIP + LOG_INFO("Decompress the HTTP content..."); std::string decompressed; - if (!Decompress(content_, &decompressed)) { + if (!gzip::Decompress(content_, &decompressed)) { LOG_ERRO("Cannot decompress the HTTP content!"); return false; } message_->SetContent(std::move(decompressed), false); + return true; + +#else + + LOG_WARN("Compressed HTTP content remains untouched."); + + message_->SetContent(std::move(content_), false); + + return true; + +#endif // WEBCC_ENABLE_GZIP } void Parser::AppendContent(const char* data, std::size_t count) { diff --git a/webcc/request.cc b/webcc/request.cc index 01353fd..15f94f7 100644 --- a/webcc/request.cc +++ b/webcc/request.cc @@ -2,7 +2,6 @@ #include "webcc/logger.h" #include "webcc/utility.h" -#include namespace webcc { @@ -74,10 +73,6 @@ void Request::Prepare() { // Append payload of content data. payload_.insert(payload_.end(), data_payload.begin(), data_payload.end()); - - std::string str; - CopyPayload(&str); - std::cout << str; } void Request::CreateStartLine() { diff --git a/webcc/request.h b/webcc/request.h index c57f599..5daeb83 100644 --- a/webcc/request.h +++ b/webcc/request.h @@ -78,7 +78,6 @@ private: Url url_; - // Files to upload for a POST request. std::vector form_parts_; std::string boundary_; diff --git a/webcc/request_builder.cc b/webcc/request_builder.cc index 51cd73f..ad15f37 100644 --- a/webcc/request_builder.cc +++ b/webcc/request_builder.cc @@ -3,7 +3,10 @@ #include "webcc/base64.h" #include "webcc/logger.h" #include "webcc/utility.h" -#include "webcc/zlib_wrapper.h" + +#if WEBCC_ENABLE_GZIP +#include "webcc/gzip.h" +#endif namespace webcc { @@ -76,9 +79,10 @@ RequestBuilder& RequestBuilder::AuthToken(const std::string& token) { } void RequestBuilder::SetContent(RequestPtr request, std::string&& data) { +#if WEBCC_ENABLE_GZIP if (gzip_ && data.size() > kGzipThreshold) { std::string compressed; - if (Compress(data, &compressed)) { + if (gzip::Compress(data, &compressed)) { request->SetContent(std::move(compressed), true); request->SetHeader(headers::kContentEncoding, "gzip"); return; @@ -86,6 +90,7 @@ void RequestBuilder::SetContent(RequestPtr request, std::string&& data) { LOG_WARN("Cannot compress the content data!"); } +#endif // WEBCC_ENABLE_GZIP request->SetContent(std::move(data), true); } diff --git a/webcc/rest_request_handler.cc b/webcc/rest_request_handler.cc index 7ad1804..b3d3331 100644 --- a/webcc/rest_request_handler.cc +++ b/webcc/rest_request_handler.cc @@ -5,7 +5,10 @@ #include "webcc/logger.h" #include "webcc/url.h" -#include "webcc/zlib_wrapper.h" + +#if WEBCC_ENABLE_GZIP +#include "webcc/gzip.h" +#endif namespace webcc { @@ -15,12 +18,11 @@ bool RestRequestHandler::Bind(RestServicePtr service, const std::string& url, } void RestRequestHandler::HandleConnection(ConnectionPtr connection) { - RequestPtr http_request = connection->request(); - assert(http_request); + RequestPtr request = connection->request(); - const Url& url = http_request->url(); + const Url& url = request->url(); - RestRequest rest_request{ http_request }; + RestRequest rest_request{ request }; // Get service by URL path. std::string path = "/" + url.path(); @@ -35,29 +37,34 @@ void RestRequestHandler::HandleConnection(ConnectionPtr connection) { RestResponse rest_response; service->Handle(rest_request, &rest_response); - auto http_response = std::make_shared(rest_response.status); + auto response = std::make_shared(rest_response.status); if (!rest_response.content.empty()) { if (!rest_response.media_type.empty()) { - http_response->SetContentType(rest_response.media_type, - rest_response.charset); + response->SetContentType(rest_response.media_type, rest_response.charset); } + SetContent(request, response, std::move(rest_response.content)); + } + + // Send response back to client. + connection->SendResponse(response); +} - // Only support gzip for response compression. - if (rest_response.content.size() > kGzipThreshold && - http_request->AcceptEncodingGzip()) { - std::string compressed; - if (Compress(rest_response.content, &compressed)) { - http_response->SetHeader(headers::kContentEncoding, "gzip"); - http_response->SetContent(std::move(compressed), true); - } - } else { - http_response->SetContent(std::move(rest_response.content), true); +void RestRequestHandler::SetContent(RequestPtr request, ResponsePtr response, + std::string&& content) { +#if WEBCC_ENABLE_GZIP + // Only support gzip (no deflate) for response compression. + if (content.size() > kGzipThreshold && request->AcceptEncodingGzip()) { + std::string compressed; + if (gzip::Compress(content, &compressed)) { + response->SetHeader(headers::kContentEncoding, "gzip"); + response->SetContent(std::move(compressed), true); + return; } } +#endif // WEBCC_ENABLE_GZIP - // Send response back to client. - connection->SendResponse(http_response); + response->SetContent(std::move(content), true); } } // namespace webcc diff --git a/webcc/rest_request_handler.h b/webcc/rest_request_handler.h index 1854e11..40b9993 100644 --- a/webcc/rest_request_handler.h +++ b/webcc/rest_request_handler.h @@ -21,6 +21,9 @@ public: private: void HandleConnection(ConnectionPtr connection) final; + void SetContent(RequestPtr request, ResponsePtr response, + std::string&& content); + private: RestServiceManager service_manager_; }; diff --git a/webcc/utility.h b/webcc/utility.h index 4adca9e..f4e6fcc 100644 --- a/webcc/utility.h +++ b/webcc/utility.h @@ -1,16 +1,10 @@ #ifndef WEBCC_UTILITY_H_ #define WEBCC_UTILITY_H_ -#include #include namespace webcc { -// Get the timestamp for HTTP Date header field. -// E.g., Wed, 21 Oct 2015 07:28:00 GMT -// See: https://tools.ietf.org/html/rfc7231#section-7.1.1.2 -std::string GetHttpDateTimestamp(); - std::string RandomUuid(); } // namespace webcc