From 5d2bae1b3238e4304229b115f1e8ea75d2fa8d40 Mon Sep 17 00:00:00 2001 From: Tindy X <49061470+tindy2013@users.noreply.github.com> Date: Sat, 21 Mar 2020 15:25:21 +0800 Subject: [PATCH] Enhancements Fix compatibility with some non-standard Surge configurations. Refactor Clash Rule generation to speed up the process. Optimize codes. --- src/interfaces.cpp | 157 ++++++++++++++---------------------------- src/multithread.cpp | 5 ++ src/multithread.h | 1 + src/speedtestutil.cpp | 4 +- src/subexport.cpp | 72 +++++++++++++++++++ 5 files changed, 131 insertions(+), 108 deletions(-) diff --git a/src/interfaces.cpp b/src/interfaces.cpp index eabdc5c..7971cdc 100644 --- a/src/interfaces.cpp +++ b/src/interfaces.cpp @@ -93,11 +93,7 @@ std::string getRuleset(RESPONSE_CALLBACK_ARGS) } std::string proxy = parseProxy(proxy_ruleset); - - if(fileExist(url)) - output_content = fileGet(url, true); - else - output_content = webGet(url, proxy, dummy, cache_ruleset); + output_content = fetchFile(url, proxy, cache_ruleset); if(!output_content.size()) { @@ -146,7 +142,7 @@ std::string getRuleset(RESPONSE_CALLBACK_ARGS) return output_content; } -int importItems(string_array &target) +int importItems(string_array &target, bool scope_limit = true) { string_array result; std::stringstream ss; @@ -165,7 +161,7 @@ int importItems(string_array &target) std::string proxy = parseProxy(proxy_config); if(fileExist(path)) - content = fileGet(path, api_mode); + content = fileGet(path, scope_limit); else content = webGet(path, proxy, dummy, cache_config); if(!content.size()) @@ -194,7 +190,7 @@ int importItems(string_array &target) return 0; } -void readRegexMatch(YAML::Node node, std::string delimiter, string_array &dest) +void readRegexMatch(YAML::Node node, std::string delimiter, string_array &dest, bool scope_limit = true) { YAML::Node object; std::string url, match, rep, strLine; @@ -217,10 +213,10 @@ void readRegexMatch(YAML::Node node, std::string delimiter, string_array &dest) continue; dest.emplace_back(strLine); } - importItems(dest); + importItems(dest, scope_limit); } -void readEmoji(YAML::Node node, string_array &dest) +void readEmoji(YAML::Node node, string_array &dest, bool scope_limit = true) { YAML::Node object; std::string url, match, rep, strLine; @@ -243,10 +239,10 @@ void readEmoji(YAML::Node node, string_array &dest) continue; dest.emplace_back(strLine); } - importItems(dest); + importItems(dest, scope_limit); } -void readGroup(YAML::Node node, string_array &dest) +void readGroup(YAML::Node node, string_array &dest, bool scope_limit = true) { std::string strLine, name, type, url, interval; string_array tempArray; @@ -289,10 +285,10 @@ void readGroup(YAML::Node node, string_array &dest) }); dest.emplace_back(strLine); } - importItems(dest); + importItems(dest, scope_limit); } -void readRuleset(YAML::Node node, string_array &dest) +void readRuleset(YAML::Node node, string_array &dest, bool scope_limit = true) { std::string strLine, name, url, group; YAML::Node object; @@ -320,7 +316,7 @@ void readRuleset(YAML::Node node, string_array &dest) continue; dest.emplace_back(strLine); } - importItems(dest); + importItems(dest, scope_limit); } void refreshRulesets(string_array &ruleset_list, std::vector &rca) @@ -368,7 +364,7 @@ void refreshRulesets(string_array &ruleset_list, std::vector &r else continue; */ - rc = {rule_group, rule_url, fetchFileAsync(rule_url, proxy, cache_ruleset)}; + rc = {rule_group, rule_url, fetchFileAsync(rule_url, proxy, cache_ruleset, async_fetch_ruleset)}; } rca.emplace_back(rc); /* @@ -437,13 +433,13 @@ void readYAMLConf(YAML::Node &node) section = node["userinfo"]; if(section["stream_rule"].IsSequence()) { - readRegexMatch(section["stream_rule"], "|", tempArray); + readRegexMatch(section["stream_rule"], "|", tempArray, false); safe_set_streams(tempArray); eraseElements(tempArray); } if(section["time_rule"].IsSequence()) { - readRegexMatch(section["time_rule"], "|", tempArray); + readRegexMatch(section["time_rule"], "|", tempArray, false); safe_set_times(tempArray); eraseElements(tempArray); } @@ -463,7 +459,7 @@ void readYAMLConf(YAML::Node &node) if(section["rename_node"].IsSequence()) { - readRegexMatch(section["rename_node"], "@", tempArray); + readRegexMatch(section["rename_node"], "@", tempArray, false); safe_set_renames(tempArray); eraseElements(tempArray); } @@ -488,7 +484,7 @@ void readYAMLConf(YAML::Node &node) section["remove_old_emoji"] >> remove_old_emoji; if(section["rules"].IsSequence()) { - readEmoji(section["rules"], tempArray); + readEmoji(section["rules"], tempArray, false); safe_set_emojis(tempArray); eraseElements(tempArray); } @@ -509,11 +505,11 @@ void readYAMLConf(YAML::Node &node) section["update_ruleset_on_request"] >> update_ruleset_on_request; } if(section["surge_ruleset"].IsSequence()) - readRuleset(section["surge_ruleset"], rulesets); + readRuleset(section["surge_ruleset"], rulesets, false); } if(node["proxy_group"].IsDefined() && node["proxy_group"]["custom_proxy_group"].IsDefined()) - readGroup(node["proxy_group"]["custom_proxy_group"], clash_extra_group); + readGroup(node["proxy_group"]["custom_proxy_group"], clash_extra_group, false); if(node["server"].IsDefined()) { @@ -671,7 +667,7 @@ void readConf() if(ini.ItemPrefixExist("rename_node")) { ini.GetAll("rename_node", tempArray); - importItems(tempArray); + importItems(tempArray, false); safe_set_renames(tempArray); eraseElements(tempArray); } @@ -683,14 +679,14 @@ void readConf() if(ini.ItemPrefixExist("stream_rule")) { ini.GetAll("stream_rule", tempArray); - importItems(tempArray); + importItems(tempArray, false); safe_set_streams(tempArray); eraseElements(tempArray); } if(ini.ItemPrefixExist("time_rule")) { ini.GetAll("time_rule", tempArray); - importItems(tempArray); + importItems(tempArray, false); safe_set_times(tempArray); eraseElements(tempArray); } @@ -716,7 +712,7 @@ void readConf() if(ini.ItemPrefixExist("rule")) { ini.GetAll("rule", tempArray); - importItems(tempArray); + importItems(tempArray, false); safe_set_emojis(tempArray); eraseElements(tempArray); } @@ -732,7 +728,7 @@ void readConf() if(ini.ItemPrefixExist("surge_ruleset")) { ini.GetAll("surge_ruleset", rulesets); - importItems(rulesets); + importItems(rulesets, false); } } else @@ -745,7 +741,7 @@ void readConf() if(ini.ItemPrefixExist("custom_proxy_group")) { ini.GetAll("custom_proxy_group", clash_extra_group); - importItems(clash_extra_group); + importItems(clash_extra_group, false); } ini.EnterSection("server"); @@ -849,13 +845,13 @@ int loadExternalYAML(YAML::Node &node, ExternalConfig &ext) section["overwrite_original_rules"] >> ext.overwrite_original_rules; if(section["custom_proxy_group"].size()) - readGroup(section["custom_proxy_group"], ext.custom_proxy_group); + readGroup(section["custom_proxy_group"], ext.custom_proxy_group, api_mode); if(section["surge_ruleset"].size()) - readRuleset(section["surge_ruleset"], ext.surge_ruleset); + readRuleset(section["surge_ruleset"], ext.surge_ruleset, api_mode); if(section["rename_node"].size()) - readRegexMatch(section["rename_node"], "@", ext.rename); + readRegexMatch(section["rename_node"], "@", ext.rename, api_mode); section["include_remarks"] >> ext.include; section["exclude_remarks"] >> ext.exclude; @@ -867,11 +863,7 @@ int loadExternalConfig(std::string &path, ExternalConfig &ext) { std::string base_content, dummy; std::string proxy = parseProxy(proxy_config); - - if(fileExist(path)) - base_content = fileGet(path, api_mode); - else - base_content = webGet(path, proxy, dummy, cache_config); + base_content = fetchFile(path, proxy, cache_config); try { @@ -898,12 +890,12 @@ int loadExternalConfig(std::string &path, ExternalConfig &ext) if(ini.ItemPrefixExist("custom_proxy_group")) { ini.GetAll("custom_proxy_group", ext.custom_proxy_group); - importItems(ext.custom_proxy_group); + importItems(ext.custom_proxy_group, api_mode); } if(ini.ItemPrefixExist("surge_ruleset")) { ini.GetAll("surge_ruleset", ext.surge_ruleset); - importItems(ext.surge_ruleset); + importItems(ext.surge_ruleset, api_mode); } if(ini.ItemExist("clash_rule_base")) @@ -929,12 +921,12 @@ int loadExternalConfig(std::string &path, ExternalConfig &ext) if(ini.ItemPrefixExist("rename")) { ini.GetAll("rename", ext.rename); - importItems(ext.rename); + importItems(ext.rename, api_mode); } if(ini.ItemPrefixExist("emoji")) { ini.GetAll("emoji", ext.emoji); - importItems(ext.emoji); + importItems(ext.emoji, api_mode); } if(ini.ItemPrefixExist("include_remarks")) ini.GetAll("include_remarks", ext.include); @@ -948,13 +940,11 @@ void generateBase() { if(!enable_base_gen) return; - std::string base_content; + std::string base_content, proxy = parseProxy(proxy_config); //std::cerr<<"Generating base content for Clash/R...\n"; writeLog(0, "Generating base content for Clash/R...", LOG_LEVEL_INFO); - if(fileExist(clash_rule_base)) - base_content = fileGet(clash_rule_base); - else - base_content = webGet(clash_rule_base, getSystemProxy()); + base_content = fetchFile(clash_rule_base, proxy, cache_config); + try { clash_base = YAML::Load(base_content); @@ -968,10 +958,7 @@ void generateBase() // mellow base generate removed for now /* std::cerr<<"Generating base content for Mellow...\n"; - if(fileExist(mellow_rule_base)) - base_content = fileGet(mellow_rule_base); - else - base_content = webGet(mellow_rule_base, getSystemProxy()); + base_content = fetchFile(mellow_rule_base, proxy, cache_config); mellow_base.keep_empty_section = true; mellow_base.store_any_line = true; if(mellow_base.Parse(base_content) != INIREADER_EXCEPTION_NONE) @@ -1204,6 +1191,7 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS) //std::cerr<<"Generate target: "; int surge_ver; + proxy = parseProxy(proxy_config); switch(hash_(target)) { case "clash"_hash: case "clashr"_hash: @@ -1217,11 +1205,7 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS) } else if(ruleset_updated || update_ruleset_on_request || ext_clash_base != clash_rule_base || !enable_base_gen) { - if(fileExist(ext_clash_base)) - base_content = fileGet(ext_clash_base); - else - base_content = webGet(ext_clash_base, getSystemProxy(), dummy, cache_config); - + base_content = fetchFile(ext_clash_base, proxy, cache_config); output_content = netchToClash(nodes, base_content, rca, extra_group, target == "clashr", ext); } else @@ -1245,12 +1229,9 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS) } else { - if(fileExist(ext_surge_base)) - base_content = fileGet(ext_surge_base); - else - base_content = webGet(ext_surge_base, getSystemProxy(), dummy, cache_config); - + base_content = fetchFile(ext_surge_base, proxy, cache_config); output_content = netchToSurge(nodes, base_content, rca, extra_group, surge_ver, ext); + if(upload == "true") uploadGist("surge" + version, upload_path, output_content, true); @@ -1262,11 +1243,8 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS) case "surfboard"_hash: //std::cerr<<"Surfboard"< headers; int ret_code = 200; for(std::string &x : sections) @@ -1929,14 +1879,7 @@ int simpleGenerator() if(ini.GetBool("direct") == true) { std::string url = ini.Get("url"); - if(fileExist(url)) - content = fileGet(url, false); - else - { - //check for proxy settings - std::string proxy = parseProxy(proxy_subscription); - content = webGet(url, proxy); - } + content = fetchFile(url, proxy, cache_subscription); if(content.empty()) { //std::cerr<<"Artifact '"< fetchFileAsync(const std::string &path, const st retVal.wait(); return retVal; } + +std::string fetchFile(const std::string &path, const std::string &proxy, int cache_ttl) +{ + return fetchFileAsync(path, proxy, cache_ttl, false).get(); +} diff --git a/src/multithread.h b/src/multithread.h index 4550824..afe27de 100644 --- a/src/multithread.h +++ b/src/multithread.h @@ -22,5 +22,6 @@ void safe_set_renames(string_array &data); void safe_set_streams(string_array &data); void safe_set_times(string_array &data); std::shared_future fetchFileAsync(const std::string &path, const std::string &proxy, int cache_ttl, bool async = false); +std::string fetchFile(const std::string &path, const std::string &proxy, int cache_ttl); #endif // MULTITHREAD_H_INCLUDED diff --git a/src/speedtestutil.cpp b/src/speedtestutil.cpp index fd49d30..c7eaf3b 100644 --- a/src/speedtestutil.cpp +++ b/src/speedtestutil.cpp @@ -1226,7 +1226,9 @@ bool explodeSurge(std::string surge, const std::string &custom_port, int local_p for(auto &y : headers) { header = split(trim(y), ":"); - if(regMatch(header[0], "(?i)host")) + if(header.size() != 2) + continue; + else if(regMatch(header[0], "(?i)host")) host = trim_quote(header[1]); else if(regMatch(header[0], "(?i)edge")) edge = trim_quote(header[1]); diff --git a/src/subexport.cpp b/src/subexport.cpp index 6e404dc..7489977 100644 --- a/src/subexport.cpp +++ b/src/subexport.cpp @@ -25,6 +25,13 @@ extern string_array ss_ciphers, ssr_ciphers; string_array clashr_protocols = {"auth_aes128_md5", "auth_aes128_sha1"}; string_array clashr_obfs = {"plain", "http_simple", "http_post", "tls1.2_ticket_auth"}; +template T safe_as (const YAML::Node& node) +{ + if(node.IsDefined() && !node.IsNull()) + return node.as(); + return T(); +}; + std::string hostnameToIPAddr(const std::string &host) { int retVal; @@ -463,6 +470,62 @@ void rulesetToClash(YAML::Node &base_rule, std::vector &ruleset base_rule[field_name] = Rules; } +std::string rulesetToClashStr(YAML::Node &base_rule, std::vector &ruleset_content_array, bool overwrite_original_rules, bool new_field_name) +{ + string_array allRules, vArray; + std::string rule_group, retrived_rules, strLine; + std::stringstream strStrm; + const std::string field_name = new_field_name ? "rules" : "Rule"; + std::string output_content = "\n" + field_name + ":\n"; + + if(!overwrite_original_rules && base_rule[field_name].IsDefined()) + { + for(size_t i = 0; i < base_rule[field_name].size(); i++) + output_content += " - " + safe_as(base_rule[field_name][i]) + "\n"; + } + base_rule.remove(field_name); + + for(ruleset_content &x : ruleset_content_array) + { + rule_group = x.rule_group; + retrived_rules = x.rule_content.get(); + if(retrived_rules.find("[]") == 0) + { + strLine = retrived_rules.substr(2); + if(strLine.find("FINAL") == 0) + strLine.replace(0, 5, "MATCH"); + strLine += "," + rule_group; + if(std::count(strLine.begin(), strLine.end(), ',') > 2) + strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2"); + output_content += " - " + strLine + "\n"; + continue; + } + char delimiter = count(retrived_rules.begin(), retrived_rules.end(), '\n') < 1 ? '\r' : '\n'; + + strStrm.clear(); + strStrm<= 2 && strLine[0] == '/' && strLine[1] == '/')) //empty lines and comments are ignored + continue; + if(strLine.find("USER-AGENT") == 0 || strLine.find("URL-REGEX") == 0 || strLine.find("PROCESS-NAME") == 0 || strLine.find("AND") == 0 || strLine.find("OR") == 0) //remove unsupported types + continue; + strLine += "," + rule_group; + if(std::count(strLine.begin(), strLine.end(), ',') > 2) + strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2"); + output_content += " - " + strLine + "\n"; + } + } + return output_content; +} + void rulesetToSurge(INIReader &base_rule, std::vector &ruleset_content_array, int surge_ver, bool overwrite_original_rules, std::string remote_path_prefix) { string_array allRules; @@ -960,10 +1023,19 @@ std::string netchToClash(std::vector &nodes, std::string &base_conf, s if(ext.nodelist) return YAML::Dump(yamlnode); + /* if(ext.enable_rule_generator) rulesetToClash(yamlnode, ruleset_content_array, ext.overwrite_original_rules, ext.clash_new_field_name); return YAML::Dump(yamlnode); + */ + if(!ext.enable_rule_generator) + return YAML::Dump(yamlnode); + + std::string output_content = rulesetToClashStr(yamlnode, ruleset_content_array, ext.overwrite_original_rules, ext.clash_new_field_name); + output_content.insert(0, YAML::Dump(yamlnode)); + + return output_content; } std::string netchToSurge(std::vector &nodes, std::string &base_conf, std::vector &ruleset_content_array, string_array &extra_proxy_group, int surge_ver, extra_settings &ext)