Fix URL encoding issues; remove request shortcuts.

This commit is contained in:
Chunting Gu
2019-08-23 14:51:41 +08:00
parent 9b31ac855a
commit 1a21989b2e
16 changed files with 372 additions and 365 deletions
-98
View File
@@ -47,104 +47,6 @@ ResponsePtr ClientSession::Request(RequestPtr request, bool stream) {
return Send(request, stream);
}
static void SetHeaders(const Strings& headers, RequestBuilder* builder) {
assert(headers.size() % 2 == 0);
for (std::size_t i = 1; i < headers.size(); i += 2) {
builder->Header(headers[i - 1], headers[i]);
}
}
ResponsePtr ClientSession::Get(const std::string& url,
const Strings& parameters,
const Strings& headers) {
RequestBuilder builder;
builder.Get(url);
assert(parameters.size() % 2 == 0);
for (std::size_t i = 1; i < parameters.size(); i += 2) {
builder.Query(parameters[i - 1], parameters[i]);
}
SetHeaders(headers, &builder);
return Request(builder());
}
ResponsePtr ClientSession::Head(const std::string& url,
const Strings& parameters,
const Strings& headers) {
RequestBuilder builder;
builder.Head(url);
assert(parameters.size() % 2 == 0);
for (std::size_t i = 1; i < parameters.size(); i += 2) {
builder.Query(parameters[i - 1], parameters[i]);
}
SetHeaders(headers, &builder);
return Request(builder());
}
ResponsePtr ClientSession::Post(const std::string& url, std::string&& data,
bool json, const Strings& headers) {
RequestBuilder builder;
builder.Post(url);
SetHeaders(headers, &builder);
builder.Body(std::move(data));
if (json) {
builder.Json();
}
return Request(builder());
}
ResponsePtr ClientSession::Put(const std::string& url, std::string&& data,
bool json, const Strings& headers) {
RequestBuilder builder;
builder.Put(url);
SetHeaders(headers, &builder);
builder.Body(std::move(data));
if (json) {
builder.Json();
}
return Request(builder());
}
ResponsePtr ClientSession::Delete(const std::string& url,
const Strings& headers) {
RequestBuilder builder;
builder.Delete(url);
SetHeaders(headers, &builder);
return Request(builder());
}
ResponsePtr ClientSession::Patch(const std::string& url, std::string&& data,
bool json, const Strings& headers) {
RequestBuilder builder;
builder.Patch(url);
SetHeaders(headers, &builder);
builder.Body(std::move(data));
if (json) {
builder.Json();
}
return Request(builder());
}
void ClientSession::InitHeaders() {
using namespace headers;
+1 -24
View File
@@ -59,33 +59,10 @@ public:
// Please use RequestBuilder to build the request.
// If |stream| is true, the response data will be written into a temp file,
// the response body will be FileBody, and you can easily move the temp file
// to another path with FileBody::Move(). So |stream| is useful for
// to another path with FileBody::Move(). So, |stream| is really useful for
// downloading files (JPEG, etc.) or saving memory for huge data responses.
ResponsePtr Request(RequestPtr request, bool stream = false);
// Shortcut for GET request.
ResponsePtr Get(const std::string& url, const Strings& parameters = {},
const Strings& headers = {});
// Shortcut for HEAD request.
ResponsePtr Head(const std::string& url, const Strings& parameters = {},
const Strings& headers = {});
// Shortcut for POST request.
ResponsePtr Post(const std::string& url, std::string&& data, bool json,
const Strings& headers = {});
// Shortcut for PUT request.
ResponsePtr Put(const std::string& url, std::string&& data, bool json,
const Strings& headers = {});
// Shortcut for DELETE request.
ResponsePtr Delete(const std::string& url, const Strings& headers = {});
// Shortcut for PATCH request.
ResponsePtr Patch(const std::string& url, std::string&& data, bool json,
const Strings& headers = {});
private:
void InitHeaders();
+1 -4
View File
@@ -31,10 +31,7 @@ void Request::Prepare() {
target += url_.query();
}
start_line_ = method_;
start_line_ += " ";
start_line_ += target;
start_line_ += " HTTP/1.1";
start_line_ = method_ + " " + target + " HTTP/1.1";
if (url_.port().empty()) {
SetHeader(headers::kHost, url_.host());
+20 -2
View File
@@ -51,7 +51,25 @@ RequestPtr RequestBuilder::operator()() {
return request;
}
RequestBuilder& RequestBuilder::File(const Path& path, bool infer_media_type,
RequestBuilder& RequestBuilder::Url(const std::string& url, bool encode) {
url_ = webcc::Url{ url, encode };
return *this;
}
RequestBuilder& RequestBuilder::Path(const std::string& path, bool encode) {
url_.AppendPath(path, encode);
return *this;
}
RequestBuilder& RequestBuilder::Query(const std::string& key,
const std::string& value,
bool encode) {
url_.AppendQuery(key, value, encode);
return *this;
}
RequestBuilder& RequestBuilder::File(const webcc::Path& path,
bool infer_media_type,
std::size_t chunk_size) {
body_.reset(new FileBody{ path, chunk_size });
@@ -63,7 +81,7 @@ RequestBuilder& RequestBuilder::File(const Path& path, bool infer_media_type,
}
RequestBuilder& RequestBuilder::FormFile(const std::string& name,
const Path& path,
const webcc::Path& path,
const std::string& media_type) {
assert(!name.empty());
return Form(FormPart::NewFile(name, path, media_type));
+40 -28
View File
@@ -7,6 +7,24 @@
#include "webcc/request.h"
#include "webcc/url.h"
// -----------------------------------------------------------------------------
// Handy macros for creating a RequestBuilder.
#define WEBCC_GET(url) webcc::RequestBuilder{}.Get(url, false)
#define WEBCC_GET_ENC(url) webcc::RequestBuilder{}.Get(url, true)
#define WEBCC_HEAD(url) webcc::RequestBuilder{}.Head(url, false)
#define WEBCC_HEAD_ENC(url) webcc::RequestBuilder{}.Head(url, true)
#define WEBCC_POST(url) webcc::RequestBuilder{}.Post(url, false)
#define WEBCC_POST_ENC(url) webcc::RequestBuilder{}.Post(url, true)
#define WEBCC_PUT(url) webcc::RequestBuilder{}.Put(url, false)
#define WEBCC_PUT_ENC(url) webcc::RequestBuilder{}.Put(url, true)
#define WEBCC_DELETE(url) webcc::RequestBuilder{}.Delete(url, false)
#define WEBCC_DELETE_ENC(url) webcc::RequestBuilder{}.Delete(url, true)
#define WEBCC_PATCH(url) webcc::RequestBuilder{}.Patch(url, false)
#define WEBCC_PATCH_ENC(url) webcc::RequestBuilder{}.Patch(url, true)
// -----------------------------------------------------------------------------
namespace webcc {
class RequestBuilder {
@@ -16,52 +34,46 @@ public:
RequestBuilder(const RequestBuilder&) = delete;
RequestBuilder& operator=(const RequestBuilder&) = delete;
// Build the request.
// Build
RequestPtr operator()();
// NOTE:
// The naming convention doesn't follow Google C++ Style for
// consistency and simplicity.
RequestBuilder& Method(const std::string& method) {
method_ = method;
return *this;
}
RequestBuilder& Url(const std::string& url) {
url_.Init(url);
return *this;
RequestBuilder& Get(const std::string& url, bool encode = false) {
return Method(methods::kGet).Url(url, encode);
}
RequestBuilder& Get(const std::string& url) {
return Method(methods::kGet).Url(url);
RequestBuilder& Head(const std::string& url, bool encode = false) {
return Method(methods::kHead).Url(url, encode);
}
RequestBuilder& Head(const std::string& url) {
return Method(methods::kHead).Url(url);
RequestBuilder& Post(const std::string& url, bool encode = false) {
return Method(methods::kPost).Url(url, encode);
}
RequestBuilder& Post(const std::string& url) {
return Method(methods::kPost).Url(url);
RequestBuilder& Put(const std::string& url, bool encode = false) {
return Method(methods::kPut).Url(url, encode);
}
RequestBuilder& Put(const std::string& url) {
return Method(methods::kPut).Url(url);
RequestBuilder& Delete(const std::string& url, bool encode = false) {
return Method(methods::kDelete).Url(url, encode);
}
RequestBuilder& Delete(const std::string& url) {
return Method(methods::kDelete).Url(url);
RequestBuilder& Patch(const std::string& url, bool encode = false) {
return Method(methods::kPatch).Url(url, encode);
}
RequestBuilder& Patch(const std::string& url) {
return Method(methods::kPatch).Url(url);
}
RequestBuilder& Url(const std::string& url, bool encode = false);
// Add a query parameter.
RequestBuilder& Query(const std::string& key, const std::string& value) {
url_.AddQuery(key, value);
return *this;
}
// Append a piece to the path.
RequestBuilder& Path(const std::string& path, bool encode = false);
// Append a parameter to the query.
RequestBuilder& Query(const std::string& key, const std::string& value,
bool encode = false);
RequestBuilder& MediaType(const std::string& media_type) {
media_type_ = media_type;
@@ -97,7 +109,7 @@ public:
// Use the file content as body.
// NOTE: Error::kFileError might be thrown.
RequestBuilder& File(const Path& path, bool infer_media_type = true,
RequestBuilder& File(const webcc::Path& path, bool infer_media_type = true,
std::size_t chunk_size = 1024);
// Add a form part.
@@ -107,7 +119,7 @@ public:
}
// Add a form part of file.
RequestBuilder& FormFile(const std::string& name, const Path& path,
RequestBuilder& FormFile(const std::string& name, const webcc::Path& path,
const std::string& media_type = "");
// Add a form part of string data.
+1 -5
View File
@@ -22,13 +22,9 @@ public:
ResponseBuilder(const ResponseBuilder&) = delete;
ResponseBuilder& operator=(const ResponseBuilder&) = delete;
// Build the response.
// Build
ResponsePtr operator()();
// NOTE:
// The naming convention doesn't follow Google C++ Style for
// consistency and simplicity.
// Some shortcuts for different status codes:
ResponseBuilder& OK() {
+125 -93
View File
@@ -1,15 +1,16 @@
#include "webcc/url.h"
#include <algorithm>
#include <cctype>
#include <functional>
#include <sstream>
#include "boost/algorithm/string.hpp"
#include "boost/algorithm/string/trim.hpp"
#include "webcc/utility.h"
namespace webcc {
// -----------------------------------------------------------------------------
// Helper functions to decode URL string.
namespace {
@@ -63,58 +64,59 @@ bool Decode(const std::string& encoded, std::string* raw) {
return true;
}
// Encodes all characters not in given set determined by given function.
std::string EncodeImpl(const std::string& raw,
// Unsafe decode.
// Return the original string on failure.
std::string DecodeUnsafe(const std::string& encoded) {
std::string raw;
if (Decode(encoded, &raw)) {
return raw;
}
return encoded;
}
// Encode all characters which should be encoded.
std::string EncodeImpl(const std::string& raw, // UTF8
std::function<bool(int)> should_encode) {
const char* const hex = "0123456789ABCDEF";
const char kHex[] = "0123456789ABCDEF";
std::string encoded;
for (auto iter = raw.begin(); iter != raw.end(); ++iter) {
for (auto i = raw.begin(); i != raw.end(); ++i) {
// For UTF8 encoded string, char ASCII can be greater than 127.
int ch = static_cast<unsigned char>(*iter);
int c = static_cast<int>(*i);
// |ch| should be the same under both UTF8 and UTF16.
if (should_encode(ch)) {
if (should_encode(c)) {
encoded.push_back('%');
encoded.push_back(hex[(ch >> 4) & 0xF]);
encoded.push_back(hex[ch & 0xF]);
encoded.push_back(kHex[(c >> 4) & 0xF]);
encoded.push_back(kHex[c & 0xF]);
} else {
// ASCII doesn't need to be encoded, it should be the same under both
// UTF8 and UTF16.
encoded.push_back(static_cast<char>(ch));
encoded.push_back(static_cast<char>(c));
}
}
return encoded;
}
// Our own implementation of alpha numeric instead of std::isalnum to avoid
// taking global lock for performance reasons.
inline bool IsAlphaNumeric(char c) {
return (c >= '0' && c <= '9') ||
(c >= 'A' && c <= 'Z') ||
(c >= 'a' && c <= 'z');
// Characters that are allowed in a URI but do not have a reserved purpose are
// are called unreserved. These include uppercase and lowercase letters, decimal
// digits, hyphen, period, underscore, and tilde.
inline bool IsUnreserved(int c) {
return std::isalnum(c) || c == '-' || c == '.' || c == '_' || c == '~';
}
// Unreserved characters are those that are allowed in a URL/URI but do not have
// a reserved purpose. They include:
// - A-Z
// - a-z
// - 0-9
// - '-' (hyphen)
// - '.' (period)
// - '_' (underscore)
// - '~' (tilde)
inline bool IsUnreserved(int c) {
return IsAlphaNumeric((char)c) ||
c == '-' || c == '.' || c == '_' || c == '~';
// General delimiters serve as the delimiters between different uri components.
// General delimiters include:
// - All of these :/?#[]@
inline bool IsGenDelim(int c) {
return c == ':' || c == '/' || c == '?' || c == '#' || c == '[' || c == ']' ||
c == '@';
}
// Sub-delimiters are those characters that may have a defined meaning within
// component of a URL/URI for a particular scheme. They do not serve as
// delimiters in any case between URL/URI segments. Sub-delimiters include:
// - All of these !$&'()*+,;=
inline bool SubDelimiter(int c) {
bool IsSubDelim(int c) {
switch (c) {
case '!':
case '$':
@@ -133,8 +135,20 @@ inline bool SubDelimiter(int c) {
}
}
// Reserved characters includes the general delimiters and sub delimiters. Some
// characters are neither reserved nor unreserved, and must be percent-encoded.
inline bool IsReserved(int c) {
return IsGenDelim(c) || IsSubDelim(c);
}
// Legal characters in the path portion include:
// - Any unreserved character
// - The percent character ('%'), and thus any percent-encoded octet
// - The sub-delimiters
// - ':' (colon)
// - '@' (at sign)
inline bool IsPathChar(int c) {
return IsUnreserved(c) || SubDelimiter(c) ||
return IsUnreserved(c) || IsSubDelim(c) ||
c == '%' || c == '/' || c == ':' || c == '@';
}
@@ -145,57 +159,75 @@ inline bool IsQueryChar(int c) {
return IsPathChar(c) || c == '?';
}
// Encode the URL query string.
inline std::string EncodeQuery(const std::string& query) {
return EncodeImpl(query, [](int c) {
return !IsQueryChar(c) || c == '%' || c == '+';
});
}
bool SplitKeyValue(const std::string& kv, std::string* key,
std::string* value) {
std::size_t i = kv.find_first_of('=');
if (i == std::string::npos || i == 0) {
return false;
}
*key = kv.substr(0, i);
*value = kv.substr(i + 1);
return true;
}
} // namespace
// -----------------------------------------------------------------------------
Url::Url(const std::string& str, bool decode) {
Init(str, decode);
std::string Url::EncodeHost(const std::string& utf8_str) {
return EncodeImpl(utf8_str, [](int c) -> bool { return c > 127; });
}
void Url::Init(const std::string& str, bool decode, bool clear) {
if (clear) {
Clear();
}
std::string Url::EncodePath(const std::string& utf8_str) {
return EncodeImpl(utf8_str, [](int c) -> bool {
return !IsPathChar(c) || c == '%' || c == '+';
});
}
if (!decode || str.find('%') == std::string::npos) {
std::string Url::EncodeQuery(const std::string& utf8_str) {
return EncodeImpl(utf8_str, [](int c) -> bool {
return !IsQueryChar(c) || c == '%' || c == '+';
});
}
std::string Url::EncodeFull(const std::string& utf8_str) {
return EncodeImpl(utf8_str, [](int c) -> bool {
return !IsUnreserved(c) && !IsReserved(c);
});
}
// -----------------------------------------------------------------------------
Url::Url(const std::string& str, bool encode) {
if (encode) {
Parse(Url::EncodeFull(str));
} else {
Parse(str);
}
}
void Url::AppendPath(const std::string& piece, bool encode) {
if (piece.empty() || piece == "/") {
return;
}
std::string decoded;
if (Decode(str, &decoded)) {
Parse(decoded);
if (path_.empty() || path_ == "/") {
path_.clear();
if (piece.front() != '/') {
path_.push_back('/');
}
} else if (path_.back() == '/' && piece.front() == '/') {
path_.pop_back();
} else if (path_.back() != '/' && piece.front() != '/') {
path_.push_back('/');
}
if (encode) {
path_.append(Url::EncodePath(piece));
} else {
// TODO: Exception?
Parse(str);
path_.append(piece);
}
}
void Url::AddQuery(const std::string& key, const std::string& value) {
void Url::AppendQuery(const std::string& key, const std::string& value,
bool encode) {
if (!query_.empty()) {
query_ += "&";
}
query_ += key + "=" + value;
if (encode) {
query_ += Url::EncodeQuery(key) + "=" + Url::EncodeQuery(value);
} else {
query_ += key + "=" + value;
}
}
void Url::Parse(const std::string& str) {
@@ -253,34 +285,43 @@ void Url::Clear() {
// -----------------------------------------------------------------------------
UrlQuery::UrlQuery(const std::string& str) {
if (!str.empty()) {
UrlQuery::UrlQuery(const std::string& encoded_str) {
if (!encoded_str.empty()) {
// Split into key value pairs separated by '&'.
for (std::size_t i = 0; i != std::string::npos;) {
std::size_t j = str.find_first_of('&', i);
std::size_t j = encoded_str.find_first_of('&', i);
std::string kv;
if (j == std::string::npos) {
kv = str.substr(i);
kv = encoded_str.substr(i);
i = std::string::npos;
} else {
kv = str.substr(i, j - i);
kv = encoded_str.substr(i, j - i);
i = j + 1;
}
std::string key;
std::string value;
if (SplitKeyValue(kv, &key, &value)) {
Add(std::move(key), std::move(value));
if (utility::SplitKV(kv, '=', &key, &value, false)) {
parameters_.push_back({ DecodeUnsafe(key), DecodeUnsafe(value) });
}
}
}
}
void UrlQuery::Add(std::string&& key, std::string&& value) {
if (!Has(key)) {
parameters_.push_back({ std::move(key), std::move(value) });
const std::string& UrlQuery::Get(const std::string& key) const {
auto it = Find(key);
if (it != parameters_.end()) {
return it->second;
}
static const std::string kEmptyValue;
return kEmptyValue;
}
const UrlQuery::Parameter& UrlQuery::Get(std::size_t index) const {
assert(index < Size());
return parameters_[index];
}
void UrlQuery::Add(const std::string& key, const std::string& value) {
@@ -296,30 +337,21 @@ void UrlQuery::Remove(const std::string& key) {
}
}
const std::string& UrlQuery::Get(const std::string& key) const {
auto it = Find(key);
if (it != parameters_.end()) {
return it->second;
}
static const std::string kEmptyValue;
return kEmptyValue;
}
std::string UrlQuery::ToString() const {
if (parameters_.empty()) {
return "";
}
std::string str = parameters_[0].first + "=" + parameters_[0].second;
std::string str;
for (std::size_t i = 1; i < parameters_.size(); ++i) {
str += "&";
for (std::size_t i = 0; i < parameters_.size(); ++i) {
if (i != 0) {
str += "&";
}
str += parameters_[i].first + "=" + parameters_[i].second;
}
str = EncodeQuery(str);
return str;
return Url::EncodeQuery(str);
}
UrlQuery::ConstIterator UrlQuery::Find(const std::string& key) const {
+37 -24
View File
@@ -12,12 +12,20 @@ namespace webcc {
// -----------------------------------------------------------------------------
// A simplified implementation of URL (or URI).
// A simple implementation of URL (or URI).
// TODO: Encoding of path
class Url {
public:
// Encode URL different components.
static std::string EncodeHost(const std::string& utf8_str);
static std::string EncodePath(const std::string& utf8_str);
static std::string EncodeQuery(const std::string& utf8_str);
static std::string EncodeFull(const std::string& utf8_str);
public:
Url() = default;
explicit Url(const std::string& str, bool decode = true);
explicit Url(const std::string& str, bool encode = false);
#if WEBCC_DEFAULT_MOVE_COPY_ASSIGN
@@ -47,8 +55,6 @@ public:
#endif // WEBCC_DEFAULT_MOVE_COPY_ASSIGN
void Init(const std::string& str, bool decode = true, bool clear = true);
const std::string& scheme() const {
return scheme_;
}
@@ -69,15 +75,19 @@ public:
return query_;
}
// Add a query parameter.
void AddQuery(const std::string& key, const std::string& value);
// Append a piece of path.
void AppendPath(const std::string& piece, bool encode = false);
// Append a query parameter.
void AppendQuery(const std::string& key, const std::string& value,
bool encode = false);
private:
void Parse(const std::string& str);
void Clear();
// TODO: Support auth & fragment.
private:
std::string scheme_;
std::string host_;
std::string port_;
@@ -87,35 +97,38 @@ private:
// -----------------------------------------------------------------------------
// URL query parameters.
// For accessing URL query parameters.
class UrlQuery {
public:
using Parameter = std::pair<std::string, std::string>;
UrlQuery() = default;
// The query string should be key-value pairs separated by '&'.
explicit UrlQuery(const std::string& encoded_str);
// The query string should be key value pairs separated by '&'.
explicit UrlQuery(const std::string& str);
bool Empty() const {
return parameters_.empty();
}
void Add(const std::string& key, const std::string& value);
void Add(std::string&& key, std::string&& value);
void Remove(const std::string& key);
// Get a value by key.
// Return empty string if the key doesn't exist.
const std::string& Get(const std::string& key) const;
std::size_t Size() const {
return parameters_.size();
}
bool Has(const std::string& key) const {
return Find(key) != parameters_.end();
}
bool IsEmpty() const {
return parameters_.empty();
}
// Get a value by key.
// Return empty string if the key doesn't exist.
const std::string& Get(const std::string& key) const;
// Return key-value pairs concatenated by '&'.
// Get a key-value pair by index.
const Parameter& Get(std::size_t index) const;
void Add(const std::string& key, const std::string& value);
void Remove(const std::string& key);
// Return encoded query string joined with '&'.
// E.g., "item=12731&color=blue&size=large".
std::string ToString() const;
+8 -6
View File
@@ -35,18 +35,20 @@ std::string GetTimestamp() {
return ss.str();
}
bool SplitKV(const std::string& str, char delimiter,
std::string* part1, std::string* part2) {
bool SplitKV(const std::string& str, char delimiter, std::string* key,
std::string* value, bool trim) {
std::size_t pos = str.find(delimiter);
if (pos == std::string::npos) {
return false;
}
*part1 = str.substr(0, pos);
*part2 = str.substr(pos + 1);
*key = str.substr(0, pos);
*value = str.substr(pos + 1);
boost::trim(*part1);
boost::trim(*part2);
if (trim) {
boost::trim(*key);
boost::trim(*value);
}
return true;
}
+2 -2
View File
@@ -21,8 +21,8 @@ std::string GetTimestamp();
// Split a key-value string.
// E.g., split "Connection: Keep-Alive".
bool SplitKV(const std::string& str, char delimiter,
std::string* key, std::string* value);
bool SplitKV(const std::string& str, char delimiter, std::string* key,
std::string* value, bool trim = true);
// Convert string to size_t.
bool ToSize(const std::string& str, int base, std::size_t* size);