Add buffer size to HttpClientSession

examples/http_client.cc

@@ -55,7 +55,7 @@ void ExampleWrappers() {
               {"Accept", "application/json"},
               HttpRequestArgs{}.buffer_size(1000));
 
-  session.Post("http://httpbin.org/post", "{ 'key': 'value' }", true,
+  session.Post("http://httpbin.org/post", "{'key': 'value'}", true,
                {"Accept", "application/json"});
 }
 
@@ -94,7 +94,7 @@ void ExampleKeepAlive(const std::string& url) {
   // Close
   session.Request(webcc::HttpRequestArgs("GET").url(url).
                   ssl_verify(kSslVerify).
-                  headers({ "Connection", "Close" }));
+                  headers({"Connection", "Close"}));
 
   // Keep-Alive
   session.Request(webcc::HttpRequestArgs("GET").url(url).
@@ -114,11 +114,9 @@ void ExampleCompression() {
 int main() {
   WEBCC_LOG_INIT("", LOG_CONSOLE);
 
-  // Note that the exception handling is mandatory.
   try {
 
-    // ExampleBasic();
-    ExampleCompression();
+    ExampleBasic();
 
   } catch (const Exception& e) {
     std::cout << "Exception: " << e.what() << std::endl;

webcc/globals.h

@@ -11,9 +11,6 @@ namespace webcc {
 
 const char* const kCRLF = "\r\n";
 
-// Default buffer size for socket reading.
-const std::size_t kBufferSize = 1024;
-
 const std::size_t kInvalidLength = std::string::npos;
 
 // Default timeout for reading response.
@@ -24,6 +21,9 @@ const int kMaxReadSeconds = 30;
 // when dumped/logged.
 const std::size_t kMaxDumpSize = 2048;
 
+// Default buffer size for socket reading.
+const std::size_t kBufferSize = 1024;
+
 // Default ports.
 const char* const kPort80 = "80";
 const char* const kPort443 = "443";

webcc/http_client.cc

@@ -1,6 +1,5 @@
 #include "webcc/http_client.h"
 
-#include "boost/algorithm/string.hpp"  // TODO: Remove
 #include "boost/date_time/posix_time/posix_time.hpp"
 
 #include "webcc/logger.h"
@@ -11,24 +10,16 @@ using boost::asio::ip::tcp;
 namespace webcc {
 
 HttpClient::HttpClient(std::size_t buffer_size, bool ssl_verify)
-    : buffer_(buffer_size == 0 ? kBufferSize : buffer_size),
+    : buffer_size_(buffer_size == 0 ? kBufferSize : buffer_size),
       timer_(io_context_),
       ssl_verify_(ssl_verify),
-      timeout_seconds_(kMaxReadSeconds),
+      timeout_(kMaxReadSeconds),
       closed_(false),
       timed_out_(false),
       error_(kNoError) {
 }
 
-void HttpClient::SetTimeout(int seconds) {
-  if (seconds > 0) {
-    timeout_seconds_ = seconds;
-  }
-}
-
-bool HttpClient::Request(const HttpRequest& request,
-                         std::size_t buffer_size,
-                         bool connect) {
+bool HttpClient::Request(const HttpRequest& request, bool connect) {
   io_context_.restart();
 
   response_.reset(new HttpResponse());
@@ -38,7 +29,10 @@ bool HttpClient::Request(const HttpRequest& request,
   timed_out_ = false;
   error_ = kNoError;
 
-  BufferResizer buffer_resizer(&buffer_, buffer_size);
+  if (buffer_.size() != buffer_size_) {
+    LOG_VERB("Resize buffer: %u -> %u.", buffer_.size(), buffer_size_);
+    buffer_.resize(buffer_size_);
+  }
 
   if (connect) {
     // No existing socket connection was specified, create a new one.
@@ -142,9 +136,9 @@ Error HttpClient::WriteReqeust(const HttpRequest& request) {
 }
 
 Error HttpClient::ReadResponse() {
-  LOG_VERB("Read response (timeout: %ds)...", timeout_seconds_);
+  LOG_VERB("Read response (timeout: %ds)...", timeout_);
 
-  timer_.expires_from_now(boost::posix_time::seconds(timeout_seconds_));
+  timer_.expires_from_now(boost::posix_time::seconds(timeout_));
   DoWaitTimer();
 
   Error error = kNoError;

webcc/http_client.h

@@ -27,8 +27,6 @@ typedef std::shared_ptr<HttpClient> HttpClientPtr;
 // Please don't use the same client object in multiple threads.
 class HttpClient {
 public:
-  // The |buffer_size| is the bytes of the buffer for reading response.
-  // 0 means default value (e.g., 1024) will be used.
   explicit HttpClient(std::size_t buffer_size = 0, bool ssl_verify = true);
 
   virtual ~HttpClient() = default;
@@ -36,16 +34,23 @@ public:
   HttpClient(const HttpClient&) = delete;
   HttpClient& operator=(const HttpClient&) = delete;
 
-  // Set the timeout seconds for reading response.
-  // The |seconds| is only effective when greater than 0.
-  void SetTimeout(int seconds);
+  void set_buffer_size(std::size_t buffer_size) {
+    buffer_size_ = (buffer_size == 0 ? kBufferSize : buffer_size);
+  }
+
+  void set_ssl_verify(bool ssl_verify) {
+    ssl_verify_ = ssl_verify;
+  }
+
+  // Set the timeout (in seconds) for reading response.
+  void set_timeout(int timeout)  {
+    if (timeout > 0) {
+      timeout_ = timeout;
+    }
+  }
 
   // Connect to server, send request, wait until response is received.
-  // Set |buffer_size| to non-zero to use a different buffer size for this
-  // specific request.
-  bool Request(const HttpRequest& request,
-               std::size_t buffer_size = 0,
-               bool connect = true);
+  bool Request(const HttpRequest& request, bool connect = true);
 
   // Close the socket.
   void Close();
@@ -90,21 +95,26 @@ private:
   // Socket connection.
   std::unique_ptr<HttpSocketBase> socket_;
 
-  std::vector<char> buffer_;
-
   HttpResponsePtr response_;
   std::unique_ptr<HttpResponseParser> response_parser_;
 
   // Timer for the timeout control.
   boost::asio::deadline_timer timer_;
 
+  // The buffer for reading response.
+  std::vector<char> buffer_;
+
+  // The size of the buffer for reading response.
+  // Set 0 for using default value (e.g., 1024).
+  std::size_t buffer_size_;
+
   // Verify the certificate of the peer (remote server) or not.
   // HTTPS only.
   bool ssl_verify_;
 
   // Maximum seconds to wait before the client cancels the operation.
   // Only for reading response from server.
-  int timeout_seconds_;
+  int timeout_;
 
   // Connection closed.
   bool closed_;

webcc/http_client_session.cc

@@ -4,6 +4,36 @@
 
 namespace webcc {
 
+namespace {
+
+// -----------------------------------------------------------------------------
+// Helper functions.
+
+bool GetSslVerify(const boost::optional<bool>& session_ssl_verify,
+                  const boost::optional<bool>& request_ssl_verify) {
+  if (request_ssl_verify) {
+    return request_ssl_verify.value();
+  } else if (session_ssl_verify) {
+    return session_ssl_verify.value();
+  }
+  return true;
+}
+
+std::size_t GetBufferSize(std::size_t session_buffer_size,
+                          std::size_t request_buffer_size) {
+  if (request_buffer_size != 0) {
+    return request_buffer_size;
+  } else if (session_buffer_size != 0) {
+    return session_buffer_size;
+  }
+  return 0;
+}
+
+}  // namespace
+
+// -----------------------------------------------------------------------------
+
+
 HttpClientSession::HttpClientSession() {
   InitHeaders();
 }
@@ -43,32 +73,31 @@ HttpResponsePtr HttpClientSession::Request(HttpRequestArgs&& args) {
 
   request.Prepare();
 
-  // Determine SSL verify flag.
-  bool ssl_verify = true;
-  if (args.ssl_verify_) {
-    ssl_verify = args.ssl_verify_.value();
-  } else if (ssl_verify_) {
-    ssl_verify = ssl_verify_.value();
-  }
+  bool ssl_verify = GetSslVerify(ssl_verify_, args.ssl_verify_);
+  std::size_t buffer_size = GetBufferSize(buffer_size_, args.buffer_size_);
 
+  const HttpClientPool::Key key{request.url()};
+
+  // Reuse a connection or not.
   bool reuse = false;
-  const HttpClientPool::Key key{ request.url() };
 
   HttpClientPtr client = pool_.Get(key);
   if (!client) {
-    client.reset(new HttpClient{ 0, ssl_verify });
+    client.reset(new HttpClient{buffer_size, ssl_verify});
     reuse = false;
   } else {
-    // TODO: Apply args.ssl_verify even if reuse a client.
-    reuse = false;
+    client->set_buffer_size(buffer_size);
+    client->set_ssl_verify(ssl_verify);
+    reuse = true;
     LOG_VERB("Reuse an existing connection.");
   }
 
-  if (!client->Request(request, args.buffer_size_, !reuse)) {
+  if (!client->Request(request, !reuse)) {
     throw Exception(client->error(), client->timed_out());
   }
 
-  // Update pool.
+  // Update connection pool.
+
   if (reuse) {
     if (client->closed()) {
       pool_.Remove(key);

webcc/http_client_session.h

@@ -35,6 +35,10 @@ public:
     ssl_verify_.emplace(ssl_verify);
   }
 
+  void set_buffer_size(std::size_t buffer_size) {
+    buffer_size_ = buffer_size;
+  }
+
   HttpResponsePtr Request(HttpRequestArgs&& args);
 
   HttpResponsePtr Get(const std::string& url,
@@ -70,6 +74,10 @@ private:
   // Verify the certificate of the peer or not.
   boost::optional<bool> ssl_verify_;
 
+  // The bytes of the buffer for reading response.
+  // 0 means default value will be used.
+  std::size_t buffer_size_ = 0;
+
   // Connection pool for keep-alive.
   HttpClientPool pool_;
 };

webcc/http_parser.cc

@@ -294,10 +294,6 @@ bool HttpParser::Finish() {
 
   LOG_INFO("Decompress the HTTP content...");
 
-  // TODO (Potential issues with gzip + chuncked):
-  // See the last section about HTTP in the following page:
-  //   https://www.bolet.org/~pornin/deflate-flush-fr.html
-  // Also see: https://stackoverflow.com/questions/5280633/gzip-compression-of-chunked-encoding-response
   std::string decompressed;
   if (!Decompress(content_, decompressed)) {
     LOG_ERRO("Cannot decompress the HTTP content!");

webcc/http_request_args.h

@@ -20,7 +20,6 @@ class HttpRequestArgs {
 public:
   explicit HttpRequestArgs(const std::string& method = "")
       : method_(method), json_(false), buffer_size_(0) {
-    LOG_VERB("HttpRequestArgs()");
   }
 
   HttpRequestArgs(const HttpRequestArgs&) = default;

webcc/logger.h

@@ -53,6 +53,9 @@ void LogWrite(int level, const char* file, int line, const char* format, ...);
 #if (defined(WIN32) || defined(_WIN64))
 
 // See: https://stackoverflow.com/a/8488201
+// ISSUE: The last path separator of __FILE__ in a header file becomes "/"
+//        instead of "\". The result is that __FILENAME__ will contain a
+//        prefix of "webcc/". So don't log from a header file!
 #define __FILENAME__ std::strrchr("\\" __FILE__, '\\') + 1
 
 #if WEBCC_LOG_LEVEL <= WEBCC_VERB

webcc/soap_client.h

@@ -23,7 +23,7 @@ public:
   SoapClient& operator=(const SoapClient&) = delete;
 
   void SetTimeout(int seconds) {
-    http_client_.SetTimeout(seconds);
+    http_client_.set_timeout(seconds);
   }
 
   void set_service_ns(const SoapNamespace& service_ns) {