diff --git a/src/interfaces.cpp b/src/interfaces.cpp index 039b418..bd0947a 100644 --- a/src/interfaces.cpp +++ b/src/interfaces.cpp @@ -330,7 +330,7 @@ int importItems(string_array &target, bool scope_limit = true) { string_array result; std::stringstream ss; - std::string path, content, strLine, dummy; + std::string path, content, strLine; unsigned int itemCount = 0; for(std::string &x : target) { @@ -347,7 +347,7 @@ int importItems(string_array &target, bool scope_limit = true) if(fileExist(path)) content = fileGet(path, scope_limit); else if(isLink(path)) - content = webGet(path, proxy, dummy, cache_config); + content = webGet(path, proxy, cache_config); else writeLog(0, "File not found or not a valid URL: " + path, LOG_LEVEL_ERROR); if(!content.size()) @@ -1286,7 +1286,7 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS) std::string target = getUrlArg(argument, "target"); switch(hash_(target)) { - case "clash"_hash: case "clashr"_hash: case "surge"_hash: case "quan"_hash: case "quanx"_hash: case "loon"_hash: case "surfboard"_hash: case "mellow"_hash: case "ss"_hash: case "ssd"_hash: case "ssr"_hash: case "sssub"_hash: case "v2ray"_hash: case "trojan"_hash: + case "clash"_hash: case "clashr"_hash: case "surge"_hash: case "quan"_hash: case "quanx"_hash: case "loon"_hash: case "surfboard"_hash: case "mellow"_hash: case "ss"_hash: case "ssd"_hash: case "ssr"_hash: case "sssub"_hash: case "v2ray"_hash: case "trojan"_hash: case "mixed"_hash: break; default: *status_code = 400; @@ -1526,7 +1526,7 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS) { x = regTrim(x); writeLog(0, "Fetching node data from url '" + x + "'.", LOG_LEVEL_INFO); - if(addNodes(x, insert_nodes, groupID, proxy, exclude_remarks, include_remarks, stream_temp, time_temp, subInfo, authorized) == -1) + if(addNodes(x, insert_nodes, groupID, proxy, exclude_remarks, include_remarks, stream_temp, time_temp, subInfo, authorized, request.headers) == -1) { *status_code = 400; return std::string("The following link doesn't contain any valid node info: " + x); @@ -1541,7 +1541,7 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS) x = regTrim(x); //std::cerr<<"Fetching node data from url '"< nodes; - std::string base_content, url = argument.size() <= 5 ? "" : argument.substr(5), dummy; + std::string base_content, url = argument.size() <= 5 ? "" : argument.substr(5); const std::string proxygroup_name = clash_use_new_field_name ? "proxy-groups" : "Proxy Group", rule_name = clash_use_new_field_name ? "rules" : "Rule"; ini.store_any_line = true; @@ -1962,7 +1953,7 @@ std::string surgeConfToClash(RESPONSE_CALLBACK_ARGS) { //std::cerr<<"Fetching node data from url '"< class __defer_struct final {private: T fn; bool __cancelle //#define defer(x) std::unique_ptr DO_CONCAT(__defer_deleter_,__LINE__) (nullptr, [&](...){x}); #define defer(x) __defer_struct DO_CONCAT(__defer_deleter,__COUNTER__) ([&](...){x;}); +#define getbit(x,n) (((int)x < 1) ? 0 : ((x >> (n - 1)) & 1)) +#define setbit(x,n,v) x ^= (-v ^ x) & (1UL << (n - 1)) + typedef std::string::size_type string_size; typedef std::vector string_array; typedef std::map string_map; diff --git a/src/nodemanip.cpp b/src/nodemanip.cpp index e29fec4..1cc1544 100644 --- a/src/nodemanip.cpp +++ b/src/nodemanip.cpp @@ -22,7 +22,7 @@ void copyNodes(std::vector &source, std::vector &dest) } } -int addNodes(std::string link, std::vector &allNodes, int groupID, std::string proxy, string_array &exclude_remarks, string_array &include_remarks, string_array &stream_rules, string_array &time_rules, std::string &subInfo, bool authorized) +int addNodes(std::string link, std::vector &allNodes, int groupID, std::string proxy, string_array &exclude_remarks, string_array &include_remarks, string_array &stream_rules, string_array &time_rules, std::string &subInfo, bool authorized, string_map &request_headers) { int linkType = -1; std::vector nodes; @@ -85,7 +85,7 @@ int addNodes(std::string link, std::vector &allNodes, int groupID, std writeLog(LOG_TYPE_INFO, "Downloading subscription data..."); if(startsWith(link, "surge:///install-config")) //surge config link link = UrlDecode(getUrlArg(link, "url")); - strSub = webGet(link, proxy, extra_headers, cache_subscription); + strSub = webGet(link, proxy, cache_subscription, &extra_headers, &request_headers); /* if(strSub.size() == 0) { diff --git a/src/nodemanip.h b/src/nodemanip.h index 668ce04..c114bb2 100644 --- a/src/nodemanip.h +++ b/src/nodemanip.h @@ -6,6 +6,6 @@ #include "nodeinfo.h" -int addNodes(std::string link, std::vector &allNodes, int groupID, std::string proxy, string_array &exclude_remarks, string_array &include_remarks, string_array &stream_rules, string_array &time_rules, std::string &subInfo, bool authorized); +int addNodes(std::string link, std::vector &allNodes, int groupID, std::string proxy, string_array &exclude_remarks, string_array &include_remarks, string_array &stream_rules, string_array &time_rules, std::string &subInfo, bool authorized, string_map &request_headers); #endif // NODEMANIP_H_INCLUDED diff --git a/src/subexport.cpp b/src/subexport.cpp index e66f42d..55a505f 100644 --- a/src/subexport.cpp +++ b/src/subexport.cpp @@ -651,7 +651,7 @@ void rulesetToClash(YAML::Node &base_rule, std::vector &ruleset total_rules++; continue; } - convertRuleset(retrieved_rules, x.rule_type); + retrieved_rules = convertRuleset(retrieved_rules, x.rule_type); char delimiter = getLineBreak(retrieved_rules); strStrm.clear(); @@ -743,7 +743,7 @@ std::string rulesetToClashStr(YAML::Node &base_rule, std::vector &ruleset_ continue; } - convertRuleset(retrieved_rules, x.rule_type); + retrieved_rules = convertRuleset(retrieved_rules, x.rule_type); char delimiter = getLineBreak(retrieved_rules); strStrm.clear(); @@ -1735,13 +1735,16 @@ std::string netchToSurge(std::vector &nodes, std::string &base_conf, s return ini.ToString(); } -std::string netchToSS(std::vector &nodes, extra_settings &ext) +std::string netchToSingle(std::vector &nodes, int types, extra_settings &ext) { + /// types: SS=1 SSR=2 VMess=4 Trojan=8 rapidjson::Document json; std::string remark, hostname, port, password, method; std::string plugin, pluginopts; std::string protocol, protoparam, obfs, obfsparam; + std::string id, aid, transproto, faketype, host, path, quicsecure, quicsecret; std::string proxyStr, allLinks; + bool tlssecure, ss = getbit(types, 1), ssr = getbit(types, 2), vmess = getbit(types, 3), trojan = getbit(types, 4); for(nodeInfo &x : nodes) { @@ -1762,16 +1765,55 @@ std::string netchToSS(std::vector &nodes, extra_settings &ext) switch(x.linkType) { case SPEEDTEST_MESSAGE_FOUNDSS: - proxyStr = "ss://" + urlsafe_base64_encode(method + ":" + password) + "@" + hostname + ":" + port; - if(plugin.size() && pluginopts.size()) + if(ss) { - proxyStr += "/?plugin=" + UrlEncode(plugin + ";" + pluginopts); + proxyStr = "ss://" + urlsafe_base64_encode(method + ":" + password) + "@" + hostname + ":" + port; + if(plugin.size() && pluginopts.size()) + { + proxyStr += "/?plugin=" + UrlEncode(plugin + ";" + pluginopts); + } + proxyStr += "#" + UrlEncode(remark); } - proxyStr += "#" + UrlEncode(remark); + else if(ssr) + { + if(std::count(ssr_ciphers.begin(), ssr_ciphers.end(), method) > 0 && !GetMember(json, "Plugin").size() && !GetMember(json, "Plugin").size()) + proxyStr = "ssr://" + urlsafe_base64_encode(hostname + ":" + port + ":origin:" + method + ":plain:" + urlsafe_base64_encode(password) \ + + "/?group=" + urlsafe_base64_encode(x.group) + "&remarks=" + urlsafe_base64_encode(remark)); + } + else + continue; break; case SPEEDTEST_MESSAGE_FOUNDSSR: - if(std::count(ss_ciphers.begin(), ss_ciphers.end(), method) > 0 && protocol == "origin" && obfs == "plain") - proxyStr = "ss://" + urlsafe_base64_encode(method + ":" + password) + "@" + hostname + ":" + port + "#" + UrlEncode(remark); + if(ssr) + { + proxyStr = "ssr://" + urlsafe_base64_encode(hostname + ":" + port + ":" + protocol + ":" + method + ":" + obfs + ":" + urlsafe_base64_encode(password) \ + + "/?group=" + urlsafe_base64_encode(x.group) + "&remarks=" + urlsafe_base64_encode(remark) \ + + "&obfsparam=" + urlsafe_base64_encode(obfsparam) + "&protoparam=" + urlsafe_base64_encode(protoparam)); + } + else if(ss) + { + if(std::count(ss_ciphers.begin(), ss_ciphers.end(), method) > 0 && protocol == "origin" && obfs == "plain") + proxyStr = "ss://" + urlsafe_base64_encode(method + ":" + password) + "@" + hostname + ":" + port + "#" + UrlEncode(remark); + } + else + continue; + break; + case SPEEDTEST_MESSAGE_FOUNDVMESS: + if(!vmess) + continue; + id = GetMember(json, "UserID"); + aid = GetMember(json, "AlterID"); + transproto = GetMember(json, "TransferProtocol"); + host = GetMember(json, "Host"); + path = GetMember(json, "Path"); + faketype = GetMember(json, "FakeType"); + tlssecure = GetMember(json, "TLSSecure") == "true"; + proxyStr = "vmess://" + base64_encode(vmessLinkConstruct(remark, hostname, port, faketype, id, aid, transproto, path, host, tlssecure ? "tls" : "")); + break; + case SPEEDTEST_MESSAGE_FOUNDTROJAN: + if(!trojan) + continue; + proxyStr = "trojan://" + password + "@" + hostname + ":" + port + "#" + UrlEncode(remark); break; default: continue; @@ -1857,115 +1899,6 @@ std::string netchToSSSub(std::string &base_conf, std::vector &nodes, e return output_content; } -std::string netchToSSR(std::vector &nodes, extra_settings &ext) -{ - rapidjson::Document json; - std::string remark, hostname, port, password, method; - std::string protocol, protoparam, obfs, obfsparam; - std::string proxyStr, allLinks; - - for(nodeInfo &x : nodes) - { - json.Parse(x.proxyStr.data()); - - remark = x.remarks; - hostname = GetMember(json, "Hostname"); - port = std::to_string((unsigned short)stoi(GetMember(json, "Port"))); - password = GetMember(json, "Password"); - method = GetMember(json, "EncryptMethod"); - protocol = GetMember(json, "Protocol"); - protoparam = GetMember(json, "ProtocolParam"); - obfs = GetMember(json, "OBFS"); - obfsparam = GetMember(json, "OBFSParam"); - - switch(x.linkType) - { - case SPEEDTEST_MESSAGE_FOUNDSSR: - proxyStr = "ssr://" + urlsafe_base64_encode(hostname + ":" + port + ":" + protocol + ":" + method + ":" + obfs + ":" + urlsafe_base64_encode(password) \ - + "/?group=" + urlsafe_base64_encode(x.group) + "&remarks=" + urlsafe_base64_encode(remark) \ - + "&obfsparam=" + urlsafe_base64_encode(obfsparam) + "&protoparam=" + urlsafe_base64_encode(protoparam)); - break; - case SPEEDTEST_MESSAGE_FOUNDSS: - if(std::count(ssr_ciphers.begin(), ssr_ciphers.end(), method) > 0 && !GetMember(json, "Plugin").size() && !GetMember(json, "Plugin").size()) - proxyStr = "ssr://" + urlsafe_base64_encode(hostname + ":" + port + ":origin:" + method + ":plain:" + urlsafe_base64_encode(password) \ - + "/?group=" + urlsafe_base64_encode(x.group) + "&remarks=" + urlsafe_base64_encode(remark)); - break; - default: - continue; - } - allLinks += proxyStr + "\n"; - } - - return base64_encode(allLinks); -} - -std::string netchToVMess(std::vector &nodes, extra_settings &ext) -{ - rapidjson::Document json; - std::string remark, hostname, port, method; - std::string id, aid, transproto, faketype, host, path, quicsecure, quicsecret; - std::string proxyStr, allLinks; - bool tlssecure; - - for(nodeInfo &x : nodes) - { - json.Parse(x.proxyStr.data()); - - remark = x.remarks; - hostname = GetMember(json, "Hostname"); - port = std::to_string((unsigned short)stoi(GetMember(json, "Port"))); - method = GetMember(json, "EncryptMethod"); - id = GetMember(json, "UserID"); - aid = GetMember(json, "AlterID"); - transproto = GetMember(json, "TransferProtocol"); - host = GetMember(json, "Host"); - path = GetMember(json, "Path"); - faketype = GetMember(json, "FakeType"); - tlssecure = GetMember(json, "TLSSecure") == "true"; - - switch(x.linkType) - { - case SPEEDTEST_MESSAGE_FOUNDVMESS: - proxyStr = "vmess://" + base64_encode(vmessLinkConstruct(remark, hostname, port, faketype, id, aid, transproto, path, host, tlssecure ? "tls" : "")); - break; - default: - continue; - } - allLinks += proxyStr + "\n"; - } - - return base64_encode(allLinks); -} - -std::string netchToTrojan(std::vector &nodes, extra_settings &ext) -{ - rapidjson::Document json; - std::string server, port, psk, remark; - std::string proxyStr, allLinks; - - for(nodeInfo &x : nodes) - { - json.Parse(x.proxyStr.data()); - - remark = x.remarks; - server = GetMember(json, "Hostname"); - port = std::to_string((unsigned short)stoi(GetMember(json, "Port"))); - psk = GetMember(json, "Password"); - - switch(x.linkType) - { - case SPEEDTEST_MESSAGE_FOUNDTROJAN: - proxyStr = "trojan://" + psk + "@" + server + ":" + port + "#" + UrlEncode(remark); - break; - default: - continue; - } - allLinks += proxyStr + "\n"; - } - - return base64_encode(allLinks); -} - std::string netchToQuan(std::vector &nodes, std::string &base_conf, std::vector &ruleset_content_array, string_array &extra_proxy_group, extra_settings &ext) { INIReader ini; diff --git a/src/subexport.h b/src/subexport.h index b2b4428..d7dc4c3 100644 --- a/src/subexport.h +++ b/src/subexport.h @@ -61,11 +61,8 @@ std::string netchToSurge(std::vector &nodes, std::string &base_conf, s std::string netchToMellow(std::vector &nodes, std::string &base_conf, std::vector &ruleset_content_array, string_array &extra_proxy_group, extra_settings &ext); void netchToMellow(std::vector &nodes, INIReader &ini, std::vector &ruleset_content_array, string_array &extra_proxy_group, extra_settings &ext); std::string netchToLoon(std::vector &nodes, std::string &base_conf, std::vector &ruleset_content_array, string_array &extra_proxy_group, extra_settings &ext); -std::string netchToSS(std::vector &nodes, extra_settings &ext); std::string netchToSSSub(std::string &base_conf, std::vector &nodes, extra_settings &ext); -std::string netchToSSR(std::vector &nodes, extra_settings &ext); -std::string netchToVMess(std::vector &nodes, extra_settings &ext); -std::string netchToTrojan(std::vector &nodes, extra_settings &ext); +std::string netchToSingle(std::vector &nodes, int types, extra_settings &ext); std::string netchToQuanX(std::vector &nodes, std::string &base_conf, std::vector &ruleset_content_array, string_array &extra_proxy_group, extra_settings &ext); void netchToQuanX(std::vector &nodes, INIReader &ini, std::vector &ruleset_content_array, string_array &extra_proxy_group, extra_settings &ext); std::string netchToQuan(std::vector &nodes, std::string &base_conf, std::vector &ruleset_content_array, string_array &extra_proxy_group, extra_settings &ext); diff --git a/src/templates.cpp b/src/templates.cpp index 3afd95b..f6f69df 100644 --- a/src/templates.cpp +++ b/src/templates.cpp @@ -190,7 +190,7 @@ const std::string clash_script_template = R"(def main(ctx, md): {% if not rule.keyword == "" %}{% include "keyword_template" %}{% endif %} {% endfor %} - geoips = { {{ geoips }} } +{% if exists("geoips") %} geoips = { {{ geoips }} } ip = md.dst_ip if ip == "": ip = ctx.resolve_ip(md.host) @@ -199,7 +199,7 @@ const std::string clash_script_template = R"(def main(ctx, md): return "{{ match_group }}" for key in geoips: if ctx.geoip(ip) == key: - return geoips[key] + return geoips[key]{% endif %} return "{{ match_group }}")"; const std::string clash_script_group_template = R"({% if rule.has_domain == "false" and rule.has_ipcidr == "false" %} if ctx.rule_providers["{{ rule.name }}_classical"].match(md): @@ -414,7 +414,9 @@ int renderClashScript(YAML::Node &base_rule, std::vector &rules } if(script) { - parse_json_pointer(data, "geoips", geoips.erase(geoips.size() - 1)); + if(geoips.size()) + parse_json_pointer(data, "geoips", geoips.erase(geoips.size() - 1)); + parse_json_pointer(data, "match_group", match_group); inja::Environment env; diff --git a/src/webget.cpp b/src/webget.cpp index d45049d..4dfc084 100644 --- a/src/webget.cpp +++ b/src/webget.cpp @@ -45,6 +45,14 @@ static int writer(char *data, size_t size, size_t nmemb, std::string *writerData return size * nmemb; } +static int dummy_writer(char *data, size_t size, size_t nmemb, void *writerData) +{ + /// dummy writer, do not save anything + (void)data; + (void)writerData; + return size * nmemb; +} + static int size_checker(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow) { if(dltotal > 1048576.0) @@ -52,7 +60,7 @@ static int size_checker(void *clientp, double dltotal, double dlnow, double ulto return 0; } -static inline void curl_set_common_options(CURL *curl_handle, const char *url) +static inline void curl_set_common_options(CURL *curl_handle, const char *url, long max_file_size = 1048576L) { curl_easy_setopt(curl_handle, CURLOPT_URL, url); curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, global_log_level == LOG_LEVEL_VERBOSE ? 1L : 0L); @@ -64,42 +72,63 @@ static inline void curl_set_common_options(CURL *curl_handle, const char *url) curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 0L); curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, 15L); curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, user_agent_str.data()); - curl_easy_setopt(curl_handle, CURLOPT_MAXFILESIZE, 1048576L); + if(max_file_size) + curl_easy_setopt(curl_handle, CURLOPT_MAXFILESIZE, max_file_size); curl_easy_setopt(curl_handle, CURLOPT_PROGRESSFUNCTION, size_checker); } -static std::string curlGet(const std::string &url, const std::string &proxy, std::string &response_headers, CURLcode &return_code) +//static std::string curlGet(const std::string &url, const std::string &proxy, std::string &response_headers, CURLcode &return_code, const string_map &request_headers) +static int curlGet(const FetchArgument argument, FetchResult &result) { CURL *curl_handle; - std::string data, new_url = url; + std::string *data = result.content, new_url = argument.url; struct curl_slist *list = NULL; + defer(curl_slist_free_all(list);) long retVal = 0; curl_init(); curl_handle = curl_easy_init(); - if(proxy.size()) + if(argument.proxy.size()) { - if(startsWith(proxy, "cors:")) + if(startsWith(argument.proxy, "cors:")) { list = curl_slist_append(list, "X-Requested-With: subconverter " VERSION); - curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list); - new_url = proxy.substr(5) + url; + new_url = argument.proxy.substr(5) + argument.url; } else - curl_easy_setopt(curl_handle, CURLOPT_PROXY, proxy.data()); + curl_easy_setopt(curl_handle, CURLOPT_PROXY, argument.proxy.data()); } curl_set_common_options(curl_handle, new_url.data()); - curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, writer); - curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, &data); - curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, writer); - curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, &response_headers); + + if(argument.request_headers) + { + for(auto &x : *argument.request_headers) + list = curl_slist_append(list, (x.first + ": " + x.second).data()); + } + if(list) + curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list); + + if(result.content) + { + curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, writer); + curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, result.content); + } + else + curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, dummy_writer); + if(result.response_headers) + { + curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, writer); + curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, result.response_headers); + } + else + curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, dummy_writer); unsigned int fail_count = 0, max_fails = 1; while(true) { - return_code = curl_easy_perform(curl_handle); - if(return_code == CURLE_OK || max_fails >= fail_count) + *result.status_code = curl_easy_perform(curl_handle); + if(*result.status_code == CURLE_OK || max_fails >= fail_count) break; else fail_count++; @@ -108,11 +137,14 @@ static std::string curlGet(const std::string &url, const std::string &proxy, std curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &retVal); curl_easy_cleanup(curl_handle); - if(return_code != CURLE_OK || retVal != 200) - data.clear(); - data.shrink_to_fit(); + if(data) + { + if(*result.status_code != CURLE_OK || retVal != 200) + data->clear(); + data->shrink_to_fit(); + } - return data; + return *result.status_code; } // data:[][;base64], @@ -139,10 +171,14 @@ std::string buildSocks5ProxyString(const std::string &addr, int port, const std: return proxystr; } -std::string webGet(const std::string &url, const std::string &proxy, std::string &response_headers, unsigned int cache_ttl) +std::string webGet(const std::string &url, const std::string &proxy, unsigned int cache_ttl, std::string *response_headers, string_map *request_headers) { + int return_code = 0; std::string content; - CURLcode return_code; + + FetchArgument argument {url, proxy, request_headers, cache_ttl}; + FetchResult fetch_res {&return_code, &content, response_headers}; + if (startsWith(url, "data:")) return dataGet(url); // cache system @@ -159,19 +195,22 @@ std::string webGet(const std::string &url, const std::string &proxy, std::string { writeLog(0, "CACHE HIT: '" + url + "', using local cache."); guarded_mutex guard(cache_rw_lock); - response_headers = fileGet(path_header, true); + if(response_headers) + *response_headers = fileGet(path_header, true); return fileGet(path, true); } writeLog(0, "CACHE MISS: '" + url + "', TTL timeout, creating new cache."); // out of TTL } else writeLog(0, "CACHE NOT EXIST: '" + url + "', creating new cache."); - content = curlGet(url, proxy, response_headers, return_code); // try to fetch data + //content = curlGet(url, proxy, response_headers, return_code); // try to fetch data + curlGet(argument, fetch_res); if(return_code == CURLE_OK) // success, save new cache { guarded_mutex guard(cache_rw_lock); fileWrite(path, content, true); - fileWrite(path_header, response_headers, true); + if(response_headers) + fileWrite(path_header, *response_headers, true); } else { @@ -180,26 +219,17 @@ std::string webGet(const std::string &url, const std::string &proxy, std::string writeLog(0, "Fetch failed. Serving cached content."); // cache exist, serving cache guarded_mutex guard(cache_rw_lock); content = fileGet(path, true); - response_headers = fileGet(path_header, true); + if(response_headers) + *response_headers = fileGet(path_header, true); } else writeLog(0, "Fetch failed. No local cache available."); // cache not exist or not allow to serve cache, serving nothing } return content; } - return curlGet(url, proxy, response_headers, return_code); -} - -std::string webGet(const std::string &url, const std::string &proxy) -{ - std::string dummy; - return webGet(url, proxy, dummy); -} - -std::string webGet(const std::string &url, const std::string &proxy, unsigned int cache_ttl) -{ - std::string dummy; - return webGet(url, proxy, dummy, cache_ttl); + //return curlGet(url, proxy, response_headers, return_code); + content = curlGet(argument, fetch_res); + return content; } int curlPost(const std::string &url, const std::string &data, const std::string &proxy, const string_array &request_headers, std::string *retData) @@ -215,7 +245,7 @@ int curlPost(const std::string &url, const std::string &data, const std::string for(const std::string &x : request_headers) list = curl_slist_append(list, x.data()); - curl_set_common_options(curl_handle, url.data()); + curl_set_common_options(curl_handle, url.data(), 0L); curl_easy_setopt(curl_handle, CURLOPT_POST, 1L); curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, data.data()); curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE, data.size()); @@ -259,7 +289,7 @@ int curlPatch(const std::string &url, const std::string &data, const std::string for(const std::string &x : request_headers) list = curl_slist_append(list, x.data()); - curl_set_common_options(curl_handle, url.data()); + curl_set_common_options(curl_handle, url.data(), 0L); curl_easy_setopt(curl_handle, CURLOPT_CUSTOMREQUEST, "PATCH"); curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, data.data()); curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE, data.size()); diff --git a/src/webget.h b/src/webget.h index de64796..c1cd60e 100644 --- a/src/webget.h +++ b/src/webget.h @@ -2,12 +2,26 @@ #define WEBGET_H_INCLUDED #include +#include #include "misc.h" -std::string webGet(const std::string &url, const std::string &proxy = ""); -std::string webGet(const std::string &url, const std::string &proxy, unsigned int cache_ttl); -std::string webGet(const std::string &url, const std::string &proxy, std::string &response_headers, unsigned int cache_ttl = 0); +struct FetchArgument +{ + const std::string url; + const std::string proxy; + string_map *request_headers = NULL; + const unsigned int cache_ttl = 0; +}; + +struct FetchResult +{ + int *status_code; + std::string *content = NULL; + std::string *response_headers = NULL; +}; + +std::string webGet(const std::string &url, const std::string &proxy = "", unsigned int cache_ttl = 0, std::string *response_headers = NULL, string_map *request_headers = NULL); int webPost(const std::string &url, const std::string &data, const std::string &proxy, const string_array &request_headers, std::string *retData); int webPatch(const std::string &url, const std::string &data, const std::string &proxy, const string_array &request_headers, std::string *retData); std::string buildSocks5ProxyString(const std::string &addr, int port, const std::string &username, const std::string &password); diff --git a/src/webserver_libevent.cpp b/src/webserver_libevent.cpp index b65a105..509d8c3 100644 --- a/src/webserver_libevent.cpp +++ b/src/webserver_libevent.cpp @@ -32,6 +32,8 @@ struct responseRoute std::vector responses; string_map redirect_map; +const char *request_header_whitelist[] = {"user-agent"}; + static inline void buffer_cleanup(struct evbuffer *eb) { //evbuffer_free(eb); @@ -40,7 +42,6 @@ static inline void buffer_cleanup(struct evbuffer *eb) #endif // MALLOC_TRIM } -//static inline int process_request(const char *method_str, std::string uri, std::string &postdata, std::string &content_type, std::string &return_data, int *status_code, std::map &extra_headers) static inline int process_request(Request &request, Response &response, std::string &return_data) { writeLog(0, "handle_cmd: " + request.method + " handle_uri: " + request.url, LOG_LEVEL_VERBOSE); @@ -123,9 +124,17 @@ void OnReq(evhttp_request *req, void *args) struct evkeyval* kv = req->input_headers->tqh_first; while (kv) { - request.headers.emplace(kv->key, kv->value); - kv = kv->next.tqe_next; + for(auto &x : request_header_whitelist) + { + if(strcmp(kv->key, x) == 0) + request.headers.emplace(kv->key, kv->value); + kv = kv->next.tqe_next; + } } + if(user_agent) + request.headers.emplace("X-User-Agent", user_agent); + request.headers.emplace("X-Client-IP", client_ip); + retVal = process_request(request, response, return_data); content_type = response.content_type;