Support request and response compression.

examples/github_client.cc

@@ -103,6 +103,22 @@ void ListAuthUserFollowers(webcc::HttpClientSession& session,
   }
 }
 
+void CreateAuthorization(webcc::HttpClientSession& session,
+                         const std::string& auth) {
+  try {
+
+    std::string data = "{'note': 'Webcc test', 'scopes': ['public_repo', 'repo', 'repo:status', 'user']}";
+
+    auto r = session.Post(kUrlRoot + "/authorizations", std::move(data), true,
+                          {"Authorization", auth});
+
+    std::cout << r->content() << std::endl;
+
+  } catch (const webcc::Exception& e) {
+    std::cout << e.what() << std::endl;
+  }
+}
+
 // -----------------------------------------------------------------------------
 
 int main() {

examples/rest_book_server.cc

@@ -98,7 +98,10 @@ void BookListService::Get(const webcc::UrlQuery& /*query*/,
     json.append(BookToJson(book));
   }
 
+  // TODO: Simplify
   response->content = JsonToString(json);
+  response->media_type = webcc::http::media_types::kApplicationJson;
+  response->charset = "utf-8";
   response->status = webcc::http::Status::kOK;
 }
 
@@ -114,6 +117,8 @@ void BookListService::Post(const std::string& request_content,
     json["id"] = id;
 
     response->content = JsonToString(json);
+    response->media_type = webcc::http::media_types::kApplicationJson;
+    response->charset = "utf-8";
     response->status = webcc::http::Status::kCreated;
   } else {
     // Invalid JSON
@@ -144,6 +149,8 @@ void BookDetailService::Get(const webcc::UrlMatches& url_matches,
   }
 
   response->content = BookToJsonString(book);
+  response->media_type = webcc::http::media_types::kApplicationJson;
+  response->charset = "utf-8";
   response->status = webcc::http::Status::kOK;
 }
 

webcc/globals.h

@@ -28,6 +28,12 @@ const std::size_t kBufferSize = 1024;
 const char* const kPort80 = "80";
 const char* const kPort443 = "443";
 
+// Why 1400? See the following page:
+// https://www.itworld.com/article/2693941/why-it-doesn-t-make-sense-to-
+// gzip-all-content-from-your-web-server.html
+// TODO: Configurable
+const std::size_t kGzipThreshold = 1400;
+
 // -----------------------------------------------------------------------------
 
 // HTTP headers.
@@ -64,6 +70,7 @@ namespace headers {
 // NOTE: Field names are case-insensitive.
 //   See https://stackoverflow.com/a/5259004 for more details.
 const char* const kHost = "Host";
+const char* const kDate = "Date";
 const char* const kContentType = "Content-Type";
 const char* const kContentLength = "Content-Length";
 const char* const kContentEncoding = "Content-Encoding";
@@ -72,6 +79,7 @@ const char* const kTransferEncoding = "Transfer-Encoding";
 const char* const kAccept = "Accept";
 const char* const kAcceptEncoding = "Accept-Encoding";
 const char* const kUserAgent = "User-Agent";
+const char* const kServer = "Server";
 
 }  // namespace headers
 
@@ -95,6 +103,12 @@ const char* const kUtf8 = "utf-8";
 
 }  // namespace charsets
 
+enum class ContentEncoding {
+  kUnknown,
+  kGzip,
+  kDeflate,
+};
+
 // Return default user agent for HTTP headers.
 const std::string& UserAgent();
 

webcc/http_client_session.cc

@@ -1,6 +1,7 @@
 #include "webcc/http_client_session.h"
 
 #include "webcc/url.h"
+#include "webcc/zlib_wrapper.h"
 
 namespace webcc {
 
@@ -33,7 +34,6 @@ std::size_t GetBufferSize(std::size_t session_buffer_size,
 
 // -----------------------------------------------------------------------------
 
-
 HttpClientSession::HttpClientSession() {
   InitHeaders();
 }
@@ -48,17 +48,6 @@ HttpResponsePtr HttpClientSession::Request(HttpRequestArgs&& args) {
     request.AddParameter(args.parameters_[i - 1], args.parameters_[i]);
   }
 
-  if (!args.data_.empty()) {
-    request.SetContent(std::move(args.data_), true);
-
-    // TODO: Request-level charset.
-    if (args.json_) {
-      request.SetContentType(http::media_types::kApplicationJson, charset_);
-    } else if (!content_type_.empty()) {
-      request.SetContentType(content_type_, charset_);
-    }
-  }
-
   // Apply the session-level headers.
   for (const HttpHeader& h : headers_.data()) {
     request.SetHeader(h.first, h.second);
@@ -76,6 +65,28 @@ HttpResponsePtr HttpClientSession::Request(HttpRequestArgs&& args) {
     request.SetHeader(http::headers::kConnection, "Close");
   }
 
+  if (!args.data_.empty()) {
+    if (gzip_ && args.data_.size() > kGzipThreshold) {
+      std::string compressed;
+      if (Compress(args.data_, &compressed)) {
+        request.SetContent(std::move(compressed), true);
+        request.SetHeader(http::headers::kContentEncoding, "gzip");
+      } else {
+        LOG_WARN("Cannot compress the content data!");
+        request.SetContent(std::move(args.data_), true);
+      }
+    } else {
+      request.SetContent(std::move(args.data_), true);
+    }
+
+    // TODO: Request-level charset.
+    if (args.json_) {
+      request.SetContentType(http::media_types::kApplicationJson, charset_);
+    } else if (!content_type_.empty()) {
+      request.SetContentType(content_type_, charset_);
+    }
+  }
+
   request.Prepare();
 
   bool ssl_verify = GetSslVerify(ssl_verify_, args.ssl_verify_);

webcc/http_client_session.h

@@ -41,6 +41,10 @@ public:
     }
   }
 
+  void set_gzip(bool gzip) {
+    gzip_ = gzip;
+  }
+
   void AddHeader(const std::string& key, const std::string& value) {
     headers_.Add(key, value);
   }
@@ -87,6 +91,12 @@ private:
   // Timeout in seconds for receiving response.
   int timeout_ = 0;
 
+  // Compress the request content.
+  // NOTE: Most servers don't support compressed requests.
+  // Even the requests module from Python doesn't have a built-in support.
+  // See: https://github.com/kennethreitz/requests/issues/1753
+  bool gzip_ = false;
+
   // Connection pool for keep-alive.
   HttpClientPool pool_;
 };

webcc/http_connection.cc

@@ -32,17 +32,21 @@ void HttpConnection::Close() {
   }
 }
 
-void HttpConnection::SetResponseContent(std::string&& content,
+void HttpConnection::SendResponse(HttpResponsePtr response) {
-                                        const std::string& media_type,
+  assert(response);
-                                        const std::string& charset) {
+
-  response_.SetContent(std::move(content), true);
+  response_ = response;
-  response_.SetContentType(media_type, charset);
+
+  // TODO: Support keep-alive.
+  response_->SetHeader(http::headers::kConnection, "Close");
+
+  response_->Prepare();
+
+  DoWrite();
 }
 
 void HttpConnection::SendResponse(http::Status status) {
-  response_.set_status(status);
+  SendResponse(std::make_shared<HttpResponse>(status));
-  response_.Prepare();
-  DoWrite();
 }
 
 void HttpConnection::DoRead() {
@@ -83,9 +87,9 @@ void HttpConnection::OnRead(boost::system::error_code ec, std::size_t length) {
 }
 
 void HttpConnection::DoWrite() {
-  LOG_VERB("HTTP response:\n%s", response_.Dump(4, "> ").c_str());
+  LOG_VERB("HTTP response:\n%s", response_->Dump(4, "> ").c_str());
 
-  boost::asio::async_write(socket_, response_.ToBuffers(),
+  boost::asio::async_write(socket_, response_->ToBuffers(),
                            std::bind(&HttpConnection::OnWrite, shared_from_this(),
                                      std::placeholders::_1,
                                      std::placeholders::_2));

webcc/http_connection.h

@@ -39,11 +39,9 @@ public:
   // Close the socket.
   void Close();
 
-  void SetResponseContent(std::string&& content,
+  // Send response to client.
-                          const std::string& media_type,
+  void SendResponse(HttpResponsePtr response);
-                          const std::string& charset);
 
-  // Send response to client with the given status.
   void SendResponse(http::Status status);
 
 private:
@@ -72,7 +70,7 @@ private:
   HttpRequestParser request_parser_;
 
   // The response to be sent back to the client.
-  HttpResponse response_;
+  HttpResponsePtr response_;
 };
 
 }  // namespace webcc

webcc/http_message.cc

@@ -87,6 +87,23 @@ bool HttpMessage::IsConnectionKeepAlive() const {
   return false;
 }
 
+http::ContentEncoding HttpMessage::GetContentEncoding() const {
+  const std::string& encoding = GetHeader(http::headers::kContentEncoding);
+  if (encoding == "gzip") {
+    return http::ContentEncoding::kGzip;
+  }
+  if (encoding == "deflate") {
+    return http::ContentEncoding::kDeflate;
+  }
+  return http::ContentEncoding::kUnknown;
+}
+
+bool HttpMessage::AcceptEncodingGzip() const {
+  using http::headers::kAcceptEncoding;
+
+  return GetHeader(kAcceptEncoding).find("gzip") != std::string::npos;
+}
+
 // See: https://tools.ietf.org/html/rfc7231#section-3.1.1.1
 void HttpMessage::SetContentType(const std::string& media_type,
                                  const std::string& charset) {

webcc/http_message.h

@@ -97,6 +97,11 @@ public:
     return headers_.Get(key, existed);
   }
 
+  http::ContentEncoding GetContentEncoding() const;
+
+  // Return true if header Accept-Encoding contains "gzip".
+  bool AcceptEncodingGzip() const;
+
   // E.g., "text/html", "application/json; charset=utf-8", etc.
   void SetContentType(const std::string& media_type,
                       const std::string& charset);

webcc/http_parser.cc

@@ -295,7 +295,7 @@ bool HttpParser::Finish() {
   LOG_INFO("Decompress the HTTP content...");
 
   std::string decompressed;
-  if (!Decompress(content_, decompressed)) {
+  if (!Decompress(content_, &decompressed)) {
     LOG_ERRO("Cannot decompress the HTTP content!");
     return false;
   }
@@ -318,19 +318,7 @@ bool HttpParser::IsContentFull() const {
 }
 
 bool HttpParser::IsContentCompressed() const {
-  using http::headers::kContentEncoding;
+  return message_->GetContentEncoding() != http::ContentEncoding::kUnknown;
-
-  const std::string& encoding = message_->GetHeader(kContentEncoding);
-
-  if (encoding.find("gzip") != std::string::npos) {
-    return true;
-  }
-
-  if (encoding.find("deflate") != std::string::npos) {
-    return true;
-  }
-
-  return false;
 }
 
 }  // namespace webcc

webcc/http_request.h

@@ -59,7 +59,7 @@ public:
 
   // Prepare payload.
   // Compose start line, set Host header, etc.
-  bool Prepare() override;
+  bool Prepare() final;
 
 private:
   std::string method_;

webcc/http_response.cc

@@ -60,22 +60,21 @@ const std::string& ToString(int status) {
 bool HttpResponse::Prepare() {
   start_line_ = status_strings::ToString(status_);
 
-  SetHeader("Server", http::UserAgent());
+  SetHeader(http::headers::kServer, http::UserAgent());
-  SetHeader("Date", GetHttpDateTimestamp());
+  SetHeader(http::headers::kDate, GetHttpDateTimestamp());
-
-  // TODO: Support Keep-Alive.
-  SetHeader(http::headers::kConnection, "Close");
 
   return true;
 }
 
-HttpResponse HttpResponse::Fault(http::Status status) {
+HttpResponsePtr HttpResponse::Fault(http::Status status) {
   assert(status != http::Status::kOK);
 
-  HttpResponse response;
+  auto response = std::make_shared<HttpResponse>(status);
-  response.set_status(status);
+
+  // TODO
+  response->SetHeader(http::headers::kConnection, "Close");
 
-  response.Prepare();
+  //response->Prepare();
 
   return response;
 }

webcc/http_response.h

@@ -8,9 +8,14 @@
 
 namespace webcc {
 
+class HttpResponse;
+typedef std::shared_ptr<HttpResponse> HttpResponsePtr;
+
 class HttpResponse : public HttpMessage {
 public:
-  HttpResponse() : status_(http::Status::kOK) {}
+  explicit HttpResponse(http::Status status = http::Status::kOK)
+      : status_(status) {
+  }
 
   ~HttpResponse() override = default;
 
@@ -19,18 +24,15 @@ public:
   void set_status(int status) { status_ = status; }
 
   // Set start line according to status code.
-  bool Prepare() override;
+  bool Prepare() final;
 
   // Get a fault response when HTTP status is not OK.
-  // TODO: Avoid copy.
+  static HttpResponsePtr Fault(http::Status status);
-  static HttpResponse Fault(http::Status status);
 
 private:
   int status_;
 };
 
-typedef std::shared_ptr<HttpResponse> HttpResponsePtr;
-
 }  // namespace webcc
 
 #endif  // WEBCC_HTTP_RESPONSE_H_

webcc/rest_request_handler.cc

@@ -5,6 +5,7 @@
 
 #include "webcc/logger.h"
 #include "webcc/url.h"
+#include "webcc/zlib_wrapper.h"
 
 namespace webcc {
 
@@ -32,18 +33,32 @@ void RestRequestHandler::HandleConnection(HttpConnectionPtr connection) {
     return;
   }
 
-  // TODO: Let the service to provide the media-type and charset.
   RestResponse rest_response;
   service->Handle(rest_request, &rest_response);
 
+  auto http_response = std::make_shared<HttpResponse>(rest_response.status);
+
   if (!rest_response.content.empty()) {
-    connection->SetResponseContent(std::move(rest_response.content),
+    if (!rest_response.media_type.empty()) {
-                                   http::media_types::kApplicationJson,
+      http_response->SetContentType(rest_response.media_type,
-                                   http::charsets::kUtf8);
+                                    rest_response.charset);
+    }
+
+    // Only support gzip for response compression.
+    if (rest_response.content.size() > kGzipThreshold &&
+        http_request.AcceptEncodingGzip()) {
+      std::string compressed;
+      if (Compress(rest_response.content, &compressed)) {
+        http_response->SetHeader(http::headers::kContentEncoding, "gzip");
+        http_response->SetContent(std::move(compressed), true);
+      }
+    } else {
+      http_response->SetContent(std::move(rest_response.content), true);
+    }
   }
 
   // Send response back to client.
-  connection->SendResponse(rest_response.status);
+  connection->SendResponse(http_response);
 }
 
 }  // namespace webcc

webcc/rest_service.h

@@ -39,7 +39,11 @@ struct RestRequest {
 
 struct RestResponse {
   http::Status status;
+
   std::string content;
+
+  std::string media_type;
+  std::string charset;
 };
 
 // -----------------------------------------------------------------------------

webcc/soap_request_handler.cc

@@ -16,18 +16,24 @@ bool SoapRequestHandler::Bind(SoapServicePtr service, const std::string& url) {
 }
 
 void SoapRequestHandler::HandleConnection(HttpConnectionPtr connection) {
-  std::string path = "/" + connection->request().url().path();
+  auto http_response = std::make_shared<HttpResponse>();
+
+  // TODO: Support keep-alive.
+  http_response->SetHeader(http::headers::kConnection, "Close");
 
+  std::string path = "/" + connection->request().url().path();
   SoapServicePtr service = GetServiceByUrl(path);
   if (!service) {
-    connection->SendResponse(http::Status::kBadRequest);
+    http_response->set_status(http::Status::kBadRequest);
+    connection->SendResponse(http_response);
     return;
   }
 
   // Parse the SOAP request XML.
   SoapRequest soap_request;
   if (!soap_request.FromXml(connection->request().content())) {
-    connection->SendResponse(http::Status::kBadRequest);
+    http_response->set_status(http::Status::kBadRequest);
+    connection->SendResponse(http_response);
     return;
   }
 
@@ -42,24 +48,26 @@ void SoapRequestHandler::HandleConnection(HttpConnectionPtr connection) {
   }
 
   if (!service->Handle(soap_request, &soap_response)) {
-    connection->SendResponse(http::Status::kBadRequest);
+    http_response->set_status(http::Status::kBadRequest);
+    connection->SendResponse(http_response);
     return;
   }
 
   std::string content;
   soap_response.ToXml(format_raw_, indent_str_, &content);
 
+  // TODO: Let the service provide charset.
   if (soap_version_ == kSoapV11) {
-    connection->SetResponseContent(std::move(content),
+    http_response->SetContentType(http::media_types::kTextXml,
-                                http::media_types::kTextXml,
                                   http::charsets::kUtf8);
   } else {
-    connection->SetResponseContent(std::move(content),
+    http_response->SetContentType(http::media_types::kApplicationSoapXml,
-                                http::media_types::kApplicationSoapXml,
                                   http::charsets::kUtf8);
   }
 
-  connection->SendResponse(http::Status::kOK);
+  http_response->set_status(http::Status::kOK);
+
+  connection->SendResponse(http_response);
 }
 
 SoapServicePtr SoapRequestHandler::GetServiceByUrl(const std::string& url) {

webcc/zlib_wrapper.cc

@@ -1,5 +1,6 @@
 #include "webcc/zlib_wrapper.h"
 
+#include <cassert>
 #include <utility>  // std::move
 
 #include "zlib.h"
@@ -8,11 +9,61 @@
 
 namespace webcc {
 
+bool Compress(const std::string& input, std::string* output) {
+  output->clear();
+
+  if (input.empty()) {
+    return true;
+  }
+
+  z_stream stream;
+  stream.next_in = (Bytef*)input.data();
+  stream.avail_in = (uInt)input.size();
+  stream.zalloc = Z_NULL;
+  stream.zfree = Z_NULL;
+  stream.opaque = Z_NULL;
+
+  int ret = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
+                         MAX_WBITS + 16, 8, Z_DEFAULT_STRATEGY);
+  if (ret != Z_OK) {
+    return false;
+  }
+
+  std::string buf;
+  buf.resize(input.size() / 2);  // TODO
+
+  // Run deflate() on input until output buffer is not full.
+  do {
+    stream.avail_out = (uInt)buf.size();
+    stream.next_out = (Bytef*)buf.data();
+
+    int err = deflate(&stream, Z_FINISH);
+
+    assert(err != Z_STREAM_ERROR);
+
+    if (err != Z_OK) {
+      deflateEnd(&stream);
+      if (stream.msg != nullptr) {
+        LOG_ERRO("zlib deflate error: %s", stream.msg);
+      }
+      return false;
+    }
+
+    std::size_t size = buf.size() - stream.avail_out;
+    output->insert(output->end(), buf.data(), buf.data() + size);
+  } while (stream.avail_out == 0);
+
+  if (deflateEnd(&stream) != Z_OK) {
+    return false;
+  }
+
+  return true;
+}
+
 // Modified from:
 //   http://windrealm.org/tutorials/decompress-gzip-stream.php
-
+bool Decompress(const std::string& input, std::string* output) {
-bool Decompress(const std::string& input, std::string& output) {
+  output->clear();
-  output.clear();
 
   if (input.empty()) {
     return true;
@@ -22,12 +73,12 @@ bool Decompress(const std::string& input, std::string& output) {
   std::string buf;
   buf.resize(input.size());
 
-  z_stream strm;
+  z_stream stream;
-  strm.next_in = (Bytef*)input.c_str();
+  stream.next_in = (Bytef*)input.data();
-  strm.avail_in = (uInt)input.size();
+  stream.avail_in = (uInt)input.size();
-  strm.total_out = 0;
+  stream.total_out = 0;
-  strm.zalloc = Z_NULL;
+  stream.zalloc = Z_NULL;
-  strm.zfree = Z_NULL;
+  stream.zfree = Z_NULL;
 
   // About the windowBits paramter:
   //   (https://stackoverflow.com/a/1838702)
@@ -35,45 +86,41 @@ bool Decompress(const std::string& input, std::string& output) {
   // windowBits can also be greater than 15 for optional gzip decoding. Add 32
   // to windowBits to enable zlib and gzip decoding with automatic header
   // detection, or add 16 to decode only the gzip format (the zlib format will
-  // return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is
+  // return a Z_DATA_ERROR).
-  // a crc32 instead of an adler32.
+  if (inflateInit2(&stream, MAX_WBITS + 32) != Z_OK) {
-  if (inflateInit2(&strm, (32 + MAX_WBITS)) != Z_OK) {
     return false;
   }
 
   while (true) {
     // Enlarge the output buffer if it's too small.
-    if (strm.total_out >= buf.size()) {
+    if (stream.total_out >= buf.size()) {
       buf.resize(buf.size() + input.size() / 2);
     }
 
-    strm.next_out = (Bytef*)(buf.c_str() + strm.total_out);
+    stream.next_out = (Bytef*)(buf.data() + stream.total_out);
-    strm.avail_out = (uInt)buf.size() - strm.total_out;
+    stream.avail_out = (uInt)buf.size() - stream.total_out;
 
     // Inflate another chunk.
-    //int err = inflate(&strm, Z_SYNC_FLUSH);
+    int err = inflate(&stream, Z_SYNC_FLUSH);
-    int err = inflate(&strm, Z_FULL_FLUSH);
 
     if (err == Z_STREAM_END) {
       break;
     } else if (err != Z_OK) {
-      inflateEnd(&strm);
+      inflateEnd(&stream);
-      if (strm.msg != nullptr) {
+      if (stream.msg != nullptr) {
-        LOG_ERRO("zlib inflate error: %s", strm.msg);
+        LOG_ERRO("zlib inflate error: %s", stream.msg);
       }
       return false;
     }
   }
 
-  if (inflateEnd(&strm) != Z_OK) {
+  if (inflateEnd(&stream) != Z_OK) {
     return false;
   }
 
-  // Remove the unused buffer.
+  // Remove the unused part then move to the output
-  buf.erase(strm.total_out);
+  buf.erase(stream.total_out);
-
+  *output = std::move(buf);
-  // Move the buffer to the output.
-  output = std::move(buf);
 
   return true;
 }

webcc/zlib_wrapper.h

@@ -5,7 +5,12 @@
 
 namespace webcc {
 
-bool Decompress(const std::string& input, std::string& output);
+// Compress the input string to gzip format output.
+bool Compress(const std::string& input, std::string* output);
+
+// Decompress the input string with auto detecting both gzip and zlib (deflate)
+// formats.
+bool Decompress(const std::string& input, std::string* output);
 
 }  // namespace webcc