From 34b19507dba9c9fa909bbdebd47b53cc10cb5f90 Mon Sep 17 00:00:00 2001 From: Tindy X <49061470+tindy2013@users.noreply.github.com> Date: Mon, 22 Jun 2020 21:40:18 +0800 Subject: [PATCH] Enhancements Fix ruleset types other than Surge Ruleset not correctly expanded to rules. Fix trying to limit upload file size. Fix when no GEOIP rule is added, the Clash Script will not be able to generate. Add "mixed" target to generate subscriptions with all types of standard subscriptions. Add passing X-User-Agent and X-Client-IP header to subscription downloader. Optimize codes. --- src/interfaces.cpp | 65 ++++++-------- src/misc.h | 3 + src/nodemanip.cpp | 4 +- src/nodemanip.h | 2 +- src/subexport.cpp | 171 +++++++++++-------------------------- src/subexport.h | 5 +- src/templates.cpp | 8 +- src/webget.cpp | 110 +++++++++++++++--------- src/webget.h | 20 ++++- src/webserver_libevent.cpp | 15 +++- 10 files changed, 191 insertions(+), 212 deletions(-) diff --git a/src/interfaces.cpp b/src/interfaces.cpp index 039b418..bd0947a 100644 --- a/src/interfaces.cpp +++ b/src/interfaces.cpp @@ -330,7 +330,7 @@ int importItems(string_array &target, bool scope_limit = true) { string_array result; std::stringstream ss; - std::string path, content, strLine, dummy; + std::string path, content, strLine; unsigned int itemCount = 0; for(std::string &x : target) { @@ -347,7 +347,7 @@ int importItems(string_array &target, bool scope_limit = true) if(fileExist(path)) content = fileGet(path, scope_limit); else if(isLink(path)) - content = webGet(path, proxy, dummy, cache_config); + content = webGet(path, proxy, cache_config); else writeLog(0, "File not found or not a valid URL: " + path, LOG_LEVEL_ERROR); if(!content.size()) @@ -1286,7 +1286,7 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS) std::string target = getUrlArg(argument, "target"); switch(hash_(target)) { - case "clash"_hash: case "clashr"_hash: case "surge"_hash: case "quan"_hash: case "quanx"_hash: case "loon"_hash: case "surfboard"_hash: case "mellow"_hash: case "ss"_hash: case "ssd"_hash: case "ssr"_hash: case "sssub"_hash: case "v2ray"_hash: case "trojan"_hash: + case "clash"_hash: case "clashr"_hash: case "surge"_hash: case "quan"_hash: case "quanx"_hash: case "loon"_hash: case "surfboard"_hash: case "mellow"_hash: case "ss"_hash: case "ssd"_hash: case "ssr"_hash: case "sssub"_hash: case "v2ray"_hash: case "trojan"_hash: case "mixed"_hash: break; default: *status_code = 400; @@ -1526,7 +1526,7 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS) { x = regTrim(x); writeLog(0, "Fetching node data from url '" + x + "'.", LOG_LEVEL_INFO); - if(addNodes(x, insert_nodes, groupID, proxy, exclude_remarks, include_remarks, stream_temp, time_temp, subInfo, authorized) == -1) + if(addNodes(x, insert_nodes, groupID, proxy, exclude_remarks, include_remarks, stream_temp, time_temp, subInfo, authorized, request.headers) == -1) { *status_code = 400; return std::string("The following link doesn't contain any valid node info: " + x); @@ -1541,7 +1541,7 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS) x = regTrim(x); //std::cerr<<"Fetching node data from url '"< nodes; - std::string base_content, url = argument.size() <= 5 ? "" : argument.substr(5), dummy; + std::string base_content, url = argument.size() <= 5 ? "" : argument.substr(5); const std::string proxygroup_name = clash_use_new_field_name ? "proxy-groups" : "Proxy Group", rule_name = clash_use_new_field_name ? "rules" : "Rule"; ini.store_any_line = true; @@ -1962,7 +1953,7 @@ std::string surgeConfToClash(RESPONSE_CALLBACK_ARGS) { //std::cerr<<"Fetching node data from url '"< class __defer_struct final {private: T fn; bool __cancelle //#define defer(x) std::unique_ptr DO_CONCAT(__defer_deleter_,__LINE__) (nullptr, [&](...){x}); #define defer(x) __defer_struct DO_CONCAT(__defer_deleter,__COUNTER__) ([&](...){x;}); +#define getbit(x,n) (((int)x < 1) ? 0 : ((x >> (n - 1)) & 1)) +#define setbit(x,n,v) x ^= (-v ^ x) & (1UL << (n - 1)) + typedef std::string::size_type string_size; typedef std::vector string_array; typedef std::map string_map; diff --git a/src/nodemanip.cpp b/src/nodemanip.cpp index e29fec4..1cc1544 100644 --- a/src/nodemanip.cpp +++ b/src/nodemanip.cpp @@ -22,7 +22,7 @@ void copyNodes(std::vector &source, std::vector &dest) } } -int addNodes(std::string link, std::vector &allNodes, int groupID, std::string proxy, string_array &exclude_remarks, string_array &include_remarks, string_array &stream_rules, string_array &time_rules, std::string &subInfo, bool authorized) +int addNodes(std::string link, std::vector &allNodes, int groupID, std::string proxy, string_array &exclude_remarks, string_array &include_remarks, string_array &stream_rules, string_array &time_rules, std::string &subInfo, bool authorized, string_map &request_headers) { int linkType = -1; std::vector nodes; @@ -85,7 +85,7 @@ int addNodes(std::string link, std::vector &allNodes, int groupID, std writeLog(LOG_TYPE_INFO, "Downloading subscription data..."); if(startsWith(link, "surge:///install-config")) //surge config link link = UrlDecode(getUrlArg(link, "url")); - strSub = webGet(link, proxy, extra_headers, cache_subscription); + strSub = webGet(link, proxy, cache_subscription, &extra_headers, &request_headers); /* if(strSub.size() == 0) { diff --git a/src/nodemanip.h b/src/nodemanip.h index 668ce04..c114bb2 100644 --- a/src/nodemanip.h +++ b/src/nodemanip.h @@ -6,6 +6,6 @@ #include "nodeinfo.h" -int addNodes(std::string link, std::vector &allNodes, int groupID, std::string proxy, string_array &exclude_remarks, string_array &include_remarks, string_array &stream_rules, string_array &time_rules, std::string &subInfo, bool authorized); +int addNodes(std::string link, std::vector &allNodes, int groupID, std::string proxy, string_array &exclude_remarks, string_array &include_remarks, string_array &stream_rules, string_array &time_rules, std::string &subInfo, bool authorized, string_map &request_headers); #endif // NODEMANIP_H_INCLUDED diff --git a/src/subexport.cpp b/src/subexport.cpp index e66f42d..55a505f 100644 --- a/src/subexport.cpp +++ b/src/subexport.cpp @@ -651,7 +651,7 @@ void rulesetToClash(YAML::Node &base_rule, std::vector &ruleset total_rules++; continue; } - convertRuleset(retrieved_rules, x.rule_type); + retrieved_rules = convertRuleset(retrieved_rules, x.rule_type); char delimiter = getLineBreak(retrieved_rules); strStrm.clear(); @@ -743,7 +743,7 @@ std::string rulesetToClashStr(YAML::Node &base_rule, std::vector &ruleset_ continue; } - convertRuleset(retrieved_rules, x.rule_type); + retrieved_rules = convertRuleset(retrieved_rules, x.rule_type); char delimiter = getLineBreak(retrieved_rules); strStrm.clear(); @@ -1735,13 +1735,16 @@ std::string netchToSurge(std::vector &nodes, std::string &base_conf, s return ini.ToString(); } -std::string netchToSS(std::vector &nodes, extra_settings &ext) +std::string netchToSingle(std::vector &nodes, int types, extra_settings &ext) { + /// types: SS=1 SSR=2 VMess=4 Trojan=8 rapidjson::Document json; std::string remark, hostname, port, password, method; std::string plugin, pluginopts; std::string protocol, protoparam, obfs, obfsparam; + std::string id, aid, transproto, faketype, host, path, quicsecure, quicsecret; std::string proxyStr, allLinks; + bool tlssecure, ss = getbit(types, 1), ssr = getbit(types, 2), vmess = getbit(types, 3), trojan = getbit(types, 4); for(nodeInfo &x : nodes) { @@ -1762,16 +1765,55 @@ std::string netchToSS(std::vector &nodes, extra_settings &ext) switch(x.linkType) { case SPEEDTEST_MESSAGE_FOUNDSS: - proxyStr = "ss://" + urlsafe_base64_encode(method + ":" + password) + "@" + hostname + ":" + port; - if(plugin.size() && pluginopts.size()) + if(ss) { - proxyStr += "/?plugin=" + UrlEncode(plugin + ";" + pluginopts); + proxyStr = "ss://" + urlsafe_base64_encode(method + ":" + password) + "@" + hostname + ":" + port; + if(plugin.size() && pluginopts.size()) + { + proxyStr += "/?plugin=" + UrlEncode(plugin + ";" + pluginopts); + } + proxyStr += "#" + UrlEncode(remark); } - proxyStr += "#" + UrlEncode(remark); + else if(ssr) + { + if(std::count(ssr_ciphers.begin(), ssr_ciphers.end(), method) > 0 && !GetMember(json, "Plugin").size() && !GetMember(json, "Plugin").size()) + proxyStr = "ssr://" + urlsafe_base64_encode(hostname + ":" + port + ":origin:" + method + ":plain:" + urlsafe_base64_encode(password) \ + + "/?group=" + urlsafe_base64_encode(x.group) + "&remarks=" + urlsafe_base64_encode(remark)); + } + else + continue; break; case SPEEDTEST_MESSAGE_FOUNDSSR: - if(std::count(ss_ciphers.begin(), ss_ciphers.end(), method) > 0 && protocol == "origin" && obfs == "plain") - proxyStr = "ss://" + urlsafe_base64_encode(method + ":" + password) + "@" + hostname + ":" + port + "#" + UrlEncode(remark); + if(ssr) + { + proxyStr = "ssr://" + urlsafe_base64_encode(hostname + ":" + port + ":" + protocol + ":" + method + ":" + obfs + ":" + urlsafe_base64_encode(password) \ + + "/?group=" + urlsafe_base64_encode(x.group) + "&remarks=" + urlsafe_base64_encode(remark) \ + + "&obfsparam=" + urlsafe_base64_encode(obfsparam) + "&protoparam=" + urlsafe_base64_encode(protoparam)); + } + else if(ss) + { + if(std::count(ss_ciphers.begin(), ss_ciphers.end(), method) > 0 && protocol == "origin" && obfs == "plain") + proxyStr = "ss://" + urlsafe_base64_encode(method + ":" + password) + "@" + hostname + ":" + port + "#" + UrlEncode(remark); + } + else + continue; + break; + case SPEEDTEST_MESSAGE_FOUNDVMESS: + if(!vmess) + continue; + id = GetMember(json, "UserID"); + aid = GetMember(json, "AlterID"); + transproto = GetMember(json, "TransferProtocol"); + host = GetMember(json, "Host"); + path = GetMember(json, "Path"); + faketype = GetMember(json, "FakeType"); + tlssecure = GetMember(json, "TLSSecure") == "true"; + proxyStr = "vmess://" + base64_encode(vmessLinkConstruct(remark, hostname, port, faketype, id, aid, transproto, path, host, tlssecure ? "tls" : "")); + break; + case SPEEDTEST_MESSAGE_FOUNDTROJAN: + if(!trojan) + continue; + proxyStr = "trojan://" + password + "@" + hostname + ":" + port + "#" + UrlEncode(remark); break; default: continue; @@ -1857,115 +1899,6 @@ std::string netchToSSSub(std::string &base_conf, std::vector &nodes, e return output_content; } -std::string netchToSSR(std::vector &nodes, extra_settings &ext) -{ - rapidjson::Document json; - std::string remark, hostname, port, password, method; - std::string protocol, protoparam, obfs, obfsparam; - std::string proxyStr, allLinks; - - for(nodeInfo &x : nodes) - { - json.Parse(x.proxyStr.data()); - - remark = x.remarks; - hostname = GetMember(json, "Hostname"); - port = std::to_string((unsigned short)stoi(GetMember(json, "Port"))); - password = GetMember(json, "Password"); - method = GetMember(json, "EncryptMethod"); - protocol = GetMember(json, "Protocol"); - protoparam = GetMember(json, "ProtocolParam"); - obfs = GetMember(json, "OBFS"); - obfsparam = GetMember(json, "OBFSParam"); - - switch(x.linkType) - { - case SPEEDTEST_MESSAGE_FOUNDSSR: - proxyStr = "ssr://" + urlsafe_base64_encode(hostname + ":" + port + ":" + protocol + ":" + method + ":" + obfs + ":" + urlsafe_base64_encode(password) \ - + "/?group=" + urlsafe_base64_encode(x.group) + "&remarks=" + urlsafe_base64_encode(remark) \ - + "&obfsparam=" + urlsafe_base64_encode(obfsparam) + "&protoparam=" + urlsafe_base64_encode(protoparam)); - break; - case SPEEDTEST_MESSAGE_FOUNDSS: - if(std::count(ssr_ciphers.begin(), ssr_ciphers.end(), method) > 0 && !GetMember(json, "Plugin").size() && !GetMember(json, "Plugin").size()) - proxyStr = "ssr://" + urlsafe_base64_encode(hostname + ":" + port + ":origin:" + method + ":plain:" + urlsafe_base64_encode(password) \ - + "/?group=" + urlsafe_base64_encode(x.group) + "&remarks=" + urlsafe_base64_encode(remark)); - break; - default: - continue; - } - allLinks += proxyStr + "\n"; - } - - return base64_encode(allLinks); -} - -std::string netchToVMess(std::vector &nodes, extra_settings &ext) -{ - rapidjson::Document json; - std::string remark, hostname, port, method; - std::string id, aid, transproto, faketype, host, path, quicsecure, quicsecret; - std::string proxyStr, allLinks; - bool tlssecure; - - for(nodeInfo &x : nodes) - { - json.Parse(x.proxyStr.data()); - - remark = x.remarks; - hostname = GetMember(json, "Hostname"); - port = std::to_string((unsigned short)stoi(GetMember(json, "Port"))); - method = GetMember(json, "EncryptMethod"); - id = GetMember(json, "UserID"); - aid = GetMember(json, "AlterID"); - transproto = GetMember(json, "TransferProtocol"); - host = GetMember(json, "Host"); - path = GetMember(json, "Path"); - faketype = GetMember(json, "FakeType"); - tlssecure = GetMember(json, "TLSSecure") == "true"; - - switch(x.linkType) - { - case SPEEDTEST_MESSAGE_FOUNDVMESS: - proxyStr = "vmess://" + base64_encode(vmessLinkConstruct(remark, hostname, port, faketype, id, aid, transproto, path, host, tlssecure ? "tls" : "")); - break; - default: - continue; - } - allLinks += proxyStr + "\n"; - } - - return base64_encode(allLinks); -} - -std::string netchToTrojan(std::vector &nodes, extra_settings &ext) -{ - rapidjson::Document json; - std::string server, port, psk, remark; - std::string proxyStr, allLinks; - - for(nodeInfo &x : nodes) - { - json.Parse(x.proxyStr.data()); - - remark = x.remarks; - server = GetMember(json, "Hostname"); - port = std::to_string((unsigned short)stoi(GetMember(json, "Port"))); - psk = GetMember(json, "Password"); - - switch(x.linkType) - { - case SPEEDTEST_MESSAGE_FOUNDTROJAN: - proxyStr = "trojan://" + psk + "@" + server + ":" + port + "#" + UrlEncode(remark); - break; - default: - continue; - } - allLinks += proxyStr + "\n"; - } - - return base64_encode(allLinks); -} - std::string netchToQuan(std::vector &nodes, std::string &base_conf, std::vector &ruleset_content_array, string_array &extra_proxy_group, extra_settings &ext) { INIReader ini; diff --git a/src/subexport.h b/src/subexport.h index b2b4428..d7dc4c3 100644 --- a/src/subexport.h +++ b/src/subexport.h @@ -61,11 +61,8 @@ std::string netchToSurge(std::vector &nodes, std::string &base_conf, s std::string netchToMellow(std::vector &nodes, std::string &base_conf, std::vector &ruleset_content_array, string_array &extra_proxy_group, extra_settings &ext); void netchToMellow(std::vector &nodes, INIReader &ini, std::vector &ruleset_content_array, string_array &extra_proxy_group, extra_settings &ext); std::string netchToLoon(std::vector &nodes, std::string &base_conf, std::vector &ruleset_content_array, string_array &extra_proxy_group, extra_settings &ext); -std::string netchToSS(std::vector &nodes, extra_settings &ext); std::string netchToSSSub(std::string &base_conf, std::vector &nodes, extra_settings &ext); -std::string netchToSSR(std::vector &nodes, extra_settings &ext); -std::string netchToVMess(std::vector &nodes, extra_settings &ext); -std::string netchToTrojan(std::vector &nodes, extra_settings &ext); +std::string netchToSingle(std::vector &nodes, int types, extra_settings &ext); std::string netchToQuanX(std::vector &nodes, std::string &base_conf, std::vector &ruleset_content_array, string_array &extra_proxy_group, extra_settings &ext); void netchToQuanX(std::vector &nodes, INIReader &ini, std::vector &ruleset_content_array, string_array &extra_proxy_group, extra_settings &ext); std::string netchToQuan(std::vector &nodes, std::string &base_conf, std::vector &ruleset_content_array, string_array &extra_proxy_group, extra_settings &ext); diff --git a/src/templates.cpp b/src/templates.cpp index 3afd95b..f6f69df 100644 --- a/src/templates.cpp +++ b/src/templates.cpp @@ -190,7 +190,7 @@ const std::string clash_script_template = R"(def main(ctx, md): {% if not rule.keyword == "" %}{% include "keyword_template" %}{% endif %} {% endfor %} - geoips = { {{ geoips }} } +{% if exists("geoips") %} geoips = { {{ geoips }} } ip = md.dst_ip if ip == "": ip = ctx.resolve_ip(md.host) @@ -199,7 +199,7 @@ const std::string clash_script_template = R"(def main(ctx, md): return "{{ match_group }}" for key in geoips: if ctx.geoip(ip) == key: - return geoips[key] + return geoips[key]{% endif %} return "{{ match_group }}")"; const std::string clash_script_group_template = R"({% if rule.has_domain == "false" and rule.has_ipcidr == "false" %} if ctx.rule_providers["{{ rule.name }}_classical"].match(md): @@ -414,7 +414,9 @@ int renderClashScript(YAML::Node &base_rule, std::vector &rules } if(script) { - parse_json_pointer(data, "geoips", geoips.erase(geoips.size() - 1)); + if(geoips.size()) + parse_json_pointer(data, "geoips", geoips.erase(geoips.size() - 1)); + parse_json_pointer(data, "match_group", match_group); inja::Environment env; diff --git a/src/webget.cpp b/src/webget.cpp index d45049d..4dfc084 100644 --- a/src/webget.cpp +++ b/src/webget.cpp @@ -45,6 +45,14 @@ static int writer(char *data, size_t size, size_t nmemb, std::string *writerData return size * nmemb; } +static int dummy_writer(char *data, size_t size, size_t nmemb, void *writerData) +{ + /// dummy writer, do not save anything + (void)data; + (void)writerData; + return size * nmemb; +} + static int size_checker(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow) { if(dltotal > 1048576.0) @@ -52,7 +60,7 @@ static int size_checker(void *clientp, double dltotal, double dlnow, double ulto return 0; } -static inline void curl_set_common_options(CURL *curl_handle, const char *url) +static inline void curl_set_common_options(CURL *curl_handle, const char *url, long max_file_size = 1048576L) { curl_easy_setopt(curl_handle, CURLOPT_URL, url); curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, global_log_level == LOG_LEVEL_VERBOSE ? 1L : 0L); @@ -64,42 +72,63 @@ static inline void curl_set_common_options(CURL *curl_handle, const char *url) curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 0L); curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, 15L); curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, user_agent_str.data()); - curl_easy_setopt(curl_handle, CURLOPT_MAXFILESIZE, 1048576L); + if(max_file_size) + curl_easy_setopt(curl_handle, CURLOPT_MAXFILESIZE, max_file_size); curl_easy_setopt(curl_handle, CURLOPT_PROGRESSFUNCTION, size_checker); } -static std::string curlGet(const std::string &url, const std::string &proxy, std::string &response_headers, CURLcode &return_code) +//static std::string curlGet(const std::string &url, const std::string &proxy, std::string &response_headers, CURLcode &return_code, const string_map &request_headers) +static int curlGet(const FetchArgument argument, FetchResult &result) { CURL *curl_handle; - std::string data, new_url = url; + std::string *data = result.content, new_url = argument.url; struct curl_slist *list = NULL; + defer(curl_slist_free_all(list);) long retVal = 0; curl_init(); curl_handle = curl_easy_init(); - if(proxy.size()) + if(argument.proxy.size()) { - if(startsWith(proxy, "cors:")) + if(startsWith(argument.proxy, "cors:")) { list = curl_slist_append(list, "X-Requested-With: subconverter " VERSION); - curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list); - new_url = proxy.substr(5) + url; + new_url = argument.proxy.substr(5) + argument.url; } else - curl_easy_setopt(curl_handle, CURLOPT_PROXY, proxy.data()); + curl_easy_setopt(curl_handle, CURLOPT_PROXY, argument.proxy.data()); } curl_set_common_options(curl_handle, new_url.data()); - curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, writer); - curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, &data); - curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, writer); - curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, &response_headers); + + if(argument.request_headers) + { + for(auto &x : *argument.request_headers) + list = curl_slist_append(list, (x.first + ": " + x.second).data()); + } + if(list) + curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list); + + if(result.content) + { + curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, writer); + curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, result.content); + } + else + curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, dummy_writer); + if(result.response_headers) + { + curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, writer); + curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, result.response_headers); + } + else + curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, dummy_writer); unsigned int fail_count = 0, max_fails = 1; while(true) { - return_code = curl_easy_perform(curl_handle); - if(return_code == CURLE_OK || max_fails >= fail_count) + *result.status_code = curl_easy_perform(curl_handle); + if(*result.status_code == CURLE_OK || max_fails >= fail_count) break; else fail_count++; @@ -108,11 +137,14 @@ static std::string curlGet(const std::string &url, const std::string &proxy, std curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &retVal); curl_easy_cleanup(curl_handle); - if(return_code != CURLE_OK || retVal != 200) - data.clear(); - data.shrink_to_fit(); + if(data) + { + if(*result.status_code != CURLE_OK || retVal != 200) + data->clear(); + data->shrink_to_fit(); + } - return data; + return *result.status_code; } // data:[][;base64], @@ -139,10 +171,14 @@ std::string buildSocks5ProxyString(const std::string &addr, int port, const std: return proxystr; } -std::string webGet(const std::string &url, const std::string &proxy, std::string &response_headers, unsigned int cache_ttl) +std::string webGet(const std::string &url, const std::string &proxy, unsigned int cache_ttl, std::string *response_headers, string_map *request_headers) { + int return_code = 0; std::string content; - CURLcode return_code; + + FetchArgument argument {url, proxy, request_headers, cache_ttl}; + FetchResult fetch_res {&return_code, &content, response_headers}; + if (startsWith(url, "data:")) return dataGet(url); // cache system @@ -159,19 +195,22 @@ std::string webGet(const std::string &url, const std::string &proxy, std::string { writeLog(0, "CACHE HIT: '" + url + "', using local cache."); guarded_mutex guard(cache_rw_lock); - response_headers = fileGet(path_header, true); + if(response_headers) + *response_headers = fileGet(path_header, true); return fileGet(path, true); } writeLog(0, "CACHE MISS: '" + url + "', TTL timeout, creating new cache."); // out of TTL } else writeLog(0, "CACHE NOT EXIST: '" + url + "', creating new cache."); - content = curlGet(url, proxy, response_headers, return_code); // try to fetch data + //content = curlGet(url, proxy, response_headers, return_code); // try to fetch data + curlGet(argument, fetch_res); if(return_code == CURLE_OK) // success, save new cache { guarded_mutex guard(cache_rw_lock); fileWrite(path, content, true); - fileWrite(path_header, response_headers, true); + if(response_headers) + fileWrite(path_header, *response_headers, true); } else { @@ -180,26 +219,17 @@ std::string webGet(const std::string &url, const std::string &proxy, std::string writeLog(0, "Fetch failed. Serving cached content."); // cache exist, serving cache guarded_mutex guard(cache_rw_lock); content = fileGet(path, true); - response_headers = fileGet(path_header, true); + if(response_headers) + *response_headers = fileGet(path_header, true); } else writeLog(0, "Fetch failed. No local cache available."); // cache not exist or not allow to serve cache, serving nothing } return content; } - return curlGet(url, proxy, response_headers, return_code); -} - -std::string webGet(const std::string &url, const std::string &proxy) -{ - std::string dummy; - return webGet(url, proxy, dummy); -} - -std::string webGet(const std::string &url, const std::string &proxy, unsigned int cache_ttl) -{ - std::string dummy; - return webGet(url, proxy, dummy, cache_ttl); + //return curlGet(url, proxy, response_headers, return_code); + content = curlGet(argument, fetch_res); + return content; } int curlPost(const std::string &url, const std::string &data, const std::string &proxy, const string_array &request_headers, std::string *retData) @@ -215,7 +245,7 @@ int curlPost(const std::string &url, const std::string &data, const std::string for(const std::string &x : request_headers) list = curl_slist_append(list, x.data()); - curl_set_common_options(curl_handle, url.data()); + curl_set_common_options(curl_handle, url.data(), 0L); curl_easy_setopt(curl_handle, CURLOPT_POST, 1L); curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, data.data()); curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE, data.size()); @@ -259,7 +289,7 @@ int curlPatch(const std::string &url, const std::string &data, const std::string for(const std::string &x : request_headers) list = curl_slist_append(list, x.data()); - curl_set_common_options(curl_handle, url.data()); + curl_set_common_options(curl_handle, url.data(), 0L); curl_easy_setopt(curl_handle, CURLOPT_CUSTOMREQUEST, "PATCH"); curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, data.data()); curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE, data.size()); diff --git a/src/webget.h b/src/webget.h index de64796..c1cd60e 100644 --- a/src/webget.h +++ b/src/webget.h @@ -2,12 +2,26 @@ #define WEBGET_H_INCLUDED #include +#include #include "misc.h" -std::string webGet(const std::string &url, const std::string &proxy = ""); -std::string webGet(const std::string &url, const std::string &proxy, unsigned int cache_ttl); -std::string webGet(const std::string &url, const std::string &proxy, std::string &response_headers, unsigned int cache_ttl = 0); +struct FetchArgument +{ + const std::string url; + const std::string proxy; + string_map *request_headers = NULL; + const unsigned int cache_ttl = 0; +}; + +struct FetchResult +{ + int *status_code; + std::string *content = NULL; + std::string *response_headers = NULL; +}; + +std::string webGet(const std::string &url, const std::string &proxy = "", unsigned int cache_ttl = 0, std::string *response_headers = NULL, string_map *request_headers = NULL); int webPost(const std::string &url, const std::string &data, const std::string &proxy, const string_array &request_headers, std::string *retData); int webPatch(const std::string &url, const std::string &data, const std::string &proxy, const string_array &request_headers, std::string *retData); std::string buildSocks5ProxyString(const std::string &addr, int port, const std::string &username, const std::string &password); diff --git a/src/webserver_libevent.cpp b/src/webserver_libevent.cpp index b65a105..509d8c3 100644 --- a/src/webserver_libevent.cpp +++ b/src/webserver_libevent.cpp @@ -32,6 +32,8 @@ struct responseRoute std::vector responses; string_map redirect_map; +const char *request_header_whitelist[] = {"user-agent"}; + static inline void buffer_cleanup(struct evbuffer *eb) { //evbuffer_free(eb); @@ -40,7 +42,6 @@ static inline void buffer_cleanup(struct evbuffer *eb) #endif // MALLOC_TRIM } -//static inline int process_request(const char *method_str, std::string uri, std::string &postdata, std::string &content_type, std::string &return_data, int *status_code, std::map &extra_headers) static inline int process_request(Request &request, Response &response, std::string &return_data) { writeLog(0, "handle_cmd: " + request.method + " handle_uri: " + request.url, LOG_LEVEL_VERBOSE); @@ -123,9 +124,17 @@ void OnReq(evhttp_request *req, void *args) struct evkeyval* kv = req->input_headers->tqh_first; while (kv) { - request.headers.emplace(kv->key, kv->value); - kv = kv->next.tqe_next; + for(auto &x : request_header_whitelist) + { + if(strcmp(kv->key, x) == 0) + request.headers.emplace(kv->key, kv->value); + kv = kv->next.tqe_next; + } } + if(user_agent) + request.headers.emplace("X-User-Agent", user_agent); + request.headers.emplace("X-Client-IP", client_ip); + retVal = process_request(request, response, return_data); content_type = response.content_type;