Add zlib to support content-encoding (gzip, deflate)

webcc/CMakeLists.txt

@@ -37,6 +37,7 @@ set(HEADERS
     url.h
     utility.h
     version.h
+    zlib_wrapper.h
     )
 
 set(SOURCES
@@ -60,6 +61,7 @@ set(SOURCES
     rest_service.cc
     url.cc
     utility.cc
+    zlib_wrapper.cc
     )
 
 if(WEBCC_ENABLE_SOAP)

webcc/globals.h

@@ -66,6 +66,7 @@ namespace headers {
 const char* const kHost = "Host";
 const char* const kContentType = "Content-Type";
 const char* const kContentLength = "Content-Length";
+const char* const kContentEncoding = "Content-Encoding";
 const char* const kConnection = "Connection";
 const char* const kTransferEncoding = "Transfer-Encoding";
 const char* const kAccept = "Accept";

webcc/http_client_session.cc

@@ -123,14 +123,36 @@ HttpResponsePtr HttpClientSession::Delete(const std::string& url,
 }
 
 void HttpClientSession::InitHeaders() {
-  headers_.Add(http::headers::kUserAgent, http::UserAgent());
+  using namespace http::headers;
 
-  // TODO: Support gzip, deflate
-  headers_.Add(http::headers::kAcceptEncoding, "identity");
+  headers_.Add(kUserAgent, http::UserAgent());
 
-  headers_.Add(http::headers::kAccept, "*/*");
+  // Content-Encoding Tokens:
+  //   (https://en.wikipedia.org/wiki/HTTP_compression)
+  // * compress ¨C UNIX "compress" program method (historic; deprecated in most
+  //              applications and replaced by gzip or deflate);
+  // * deflate  ¨C compression based on the deflate algorithm, a combination of
+  //              the LZ77 algorithm and Huffman coding, wrapped inside the
+  //              zlib data format;
+  // * gzip     ¨C GNU zip format. Uses the deflate algorithm for compression,
+  //              but the data format and the checksum algorithm differ from
+  //              the "deflate" content-encoding. This method is the most
+  //              broadly supported as of March 2011.
+  // * identity ¨C No transformation is used. This is the default value for
+  //              content coding.
+  //
+  // A note about "deflate":
+  //   (https://www.zlib.net/zlib_faq.html#faq39)
+  // "gzip" is the gzip format, and "deflate" is the zlib format. They should
+  // probably have called the second one "zlib" instead to avoid confusion with
+  // the raw deflate compressed data format.
+  // Simply put, "deflate" is not recommended for HTTP 1.1 encoding.
+  //
+  headers_.Add(kAcceptEncoding, "gzip, deflate");
 
-  headers_.Add(http::headers::kConnection, "Keep-Alive");
+  headers_.Add(kAccept, "*/*");
+
+  headers_.Add(kConnection, "Keep-Alive");
 }
 
 }  // namespace webcc

webcc/http_parser.cc

@@ -4,6 +4,7 @@
 
 #include "webcc/http_message.h"
 #include "webcc/logger.h"
+#include "webcc/zlib_wrapper.h"
 
 namespace webcc {
 
@@ -279,11 +280,32 @@ bool HttpParser::ParseChunkSize() {
   return true;
 }
 
-void HttpParser::Finish() {
-  if (!content_.empty()) {
-    message_->SetContent(std::move(content_), /*set_length*/false);
-  }
+bool HttpParser::Finish() {
   finished_ = true;
+
+  if (content_.empty()) {
+    return true;
+  }
+
+  if (!IsContentCompressed()) {
+    message_->SetContent(std::move(content_), false);
+    return true;
+  }
+
+  LOG_INFO("Decompress the HTTP content...");
+
+  // TODO (Potential issues with gzip + chuncked):
+  // See the last section about HTTP in the following page:
+  //   https://www.bolet.org/~pornin/deflate-flush-fr.html
+  // Also see: https://stackoverflow.com/questions/5280633/gzip-compression-of-chunked-encoding-response
+  std::string decompressed;
+  if (!Decompress(content_, decompressed)) {
+    LOG_ERRO("Cannot decompress the HTTP content!", );
+    return false;
+  }
+
+  message_->SetContent(std::move(decompressed), false);
+  return true;
 }
 
 void HttpParser::AppendContent(const char* data, std::size_t count) {
@@ -299,4 +321,20 @@ bool HttpParser::IsContentFull() const {
          content_length_ <= content_.length();
 }
 
+bool HttpParser::IsContentCompressed() const {
+  using http::headers::kContentEncoding;
+
+  const std::string& encoding = message_->GetHeader(kContentEncoding);
+
+  if (encoding.find("gzip") != std::string::npos) {
+    return true;
+  }
+
+  if (encoding.find("deflate") != std::string::npos) {
+    return true;
+  }
+
+  return false;
+}
+
 }  // namespace webcc

webcc/http_parser.h

@@ -25,7 +25,7 @@ public:
 
   bool Parse(const char* data, std::size_t length);
 
-public:
+protected:
   // Parse headers from pending data.
   // Return false only on syntax errors.
   bool ParseHeaders();
@@ -45,13 +45,18 @@ public:
   bool ParseChunkedContent();
   bool ParseChunkSize();
 
-  void Finish();
+  // Return false if the compressed content cannot be decompressed.
+  bool Finish();
 
   void AppendContent(const char* data, std::size_t count);
   void AppendContent(const std::string& data);
 
   bool IsContentFull() const;
 
+  // Check header Content-Encoding to see if the content is compressed.
+  bool IsContentCompressed() const;
+
+protected:
   // The result HTTP message.
   HttpMessage* message_;
 

webcc/zlib_wrapper.cc

@@ -0,0 +1,81 @@
+#include "webcc/zlib_wrapper.h"
+
+#include <utility>  // std::move
+
+#include "zlib.h"
+
+#include "webcc/logger.h"
+
+namespace webcc {
+
+// Modified from:
+//   http://windrealm.org/tutorials/decompress-gzip-stream.php
+
+bool Decompress(const std::string& input, std::string& output) {
+  output.clear();
+
+  if (input.empty()) {
+    return true;
+  }
+
+  // Initialize the output buffer with the same size as the input.
+  std::string buf;
+  buf.resize(input.size());
+
+  z_stream strm;
+  strm.next_in = (Bytef*)input.c_str();
+  strm.avail_in = (uInt)input.size();
+  strm.total_out = 0;
+  strm.zalloc = Z_NULL;
+  strm.zfree = Z_NULL;
+
+  // About the windowBits paramter:
+  //   (https://stackoverflow.com/a/1838702)
+  //   (http://www.zlib.net/manual.html)
+  // windowBits can also be greater than 15 for optional gzip decoding. Add 32
+  // to windowBits to enable zlib and gzip decoding with automatic header
+  // detection, or add 16 to decode only the gzip format (the zlib format will
+  // return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is
+  // a crc32 instead of an adler32.
+  if (inflateInit2(&strm, (32 + MAX_WBITS)) != Z_OK) {
+    return false;
+  }
+
+  while (true) {
+    // Enlarge the output buffer if it's too small.
+    if (strm.total_out >= buf.size()) {
+      buf.resize(buf.size() + input.size() / 2);
+    }
+
+    strm.next_out = (Bytef*)(buf.c_str() + strm.total_out);
+    strm.avail_out = (uInt)buf.size() - strm.total_out;
+
+    // Inflate another chunk.
+    //int err = inflate(&strm, Z_SYNC_FLUSH);
+    int err = inflate(&strm, Z_FULL_FLUSH);
+
+    if (err == Z_STREAM_END) {
+      break;
+    } else if (err != Z_OK) {
+      inflateEnd(&strm);
+      if (strm.msg != nullptr) {
+        LOG_ERRO("zlib inflate error: %s", strm.msg);
+      }
+      return false;
+    }
+  }
+
+  if (inflateEnd(&strm) != Z_OK) {
+    return false;
+  }
+
+  // Remove the unused buffer.
+  buf.erase(strm.total_out);
+
+  // Move the buffer to the output.
+  output = std::move(buf);
+
+  return true;
+}
+
+}  // namespace webcc

webcc/zlib_wrapper.h

@@ -0,0 +1,12 @@
+#ifndef WEBCC_ZLIB_WRAPPER_H_
+#define WEBCC_ZLIB_WRAPPER_H_
+
+#include <string>
+
+namespace webcc {
+
+bool Decompress(const std::string& input, std::string& output);
+
+}  // namespace webcc
+
+#endif  // WEBCC_ZLIB_WRAPPER_H_