From 64b5d3bdaa88452407b1829358e4c29256dc3937 Mon Sep 17 00:00:00 2001 From: Tindy X <49061470+tindy2013@users.noreply.github.com> Date: Mon, 8 Jun 2020 19:56:06 +0800 Subject: [PATCH] Enhancements Fix exporting useless info in Clash HTTP/SOCKS 5 nodes. Add support for exporting Clash classical rule-provider. Optimize codes. Update build script. --- .github/workflows/docker.yml | 6 +- src/interfaces.cpp | 104 ++++++++++++++++++++--------------- src/subexport.cpp | 77 ++++++++++++++++---------- src/subexport.h | 1 + src/templates.cpp | 64 ++++++++++++++------- src/templates.h | 2 +- 6 files changed, 159 insertions(+), 95 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0fccbc6..fa757ab 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -30,7 +30,7 @@ jobs: - name: Docker buildx image and push on master branch if: github.ref == 'refs/heads/master' run: | - docker buildx build --output "type=image,push=true" --platform=linux/amd64,linux/arm/v7,linux/arm64 --tag tindy2013/subconverter:latest scripts/ + docker buildx build --output "type=image,push=true" --platform=linux/amd64,linux/arm/v7,linux/arm64,linux/386 --tag tindy2013/subconverter:latest scripts/ - name: Replace tag without `v` if: startsWith(github.ref, 'refs/tags/') @@ -44,5 +44,5 @@ jobs: - name: Docker buildx image and push on release if: startsWith(github.ref, 'refs/tags/') run: | - docker buildx build --output "type=image,push=true" --platform=linux/amd64,linux/arm/v7,linux/arm64 --tag tindy2013/subconverter:${{steps.version.outputs.result}} scripts/ - docker buildx build --output "type=image,push=true" --platform=linux/amd64,linux/arm/v7,linux/arm64 --tag tindy2013/subconverter:latest scripts/ + docker buildx build --output "type=image,push=true" --platform=linux/amd64,linux/arm/v7,linux/arm64,linux/386 --tag tindy2013/subconverter:${{steps.version.outputs.result}} scripts/ + docker buildx build --output "type=image,push=true" --platform=linux/amd64,linux/arm/v7,linux/arm64,linux/386 --tag tindy2013/subconverter:latest scripts/ diff --git a/src/interfaces.cpp b/src/interfaces.cpp index 9810152..efba64e 100644 --- a/src/interfaces.cpp +++ b/src/interfaces.cpp @@ -103,17 +103,18 @@ std::string parseProxy(const std::string &source) } #define basic_types "DOMAIN", "DOMAIN-SUFFIX", "DOMAIN-KEYWORD", "IP-CIDR", "SRC-IP-CIDR", "GEOIP", "MATCH", "FINAL" +const string_array clash_rule_type = {basic_types, "IP-CIDR6", "SRC-PORT", "DST-PORT"}; const string_array surge_rule_type = {basic_types, "IP-CIDR6", "USER-AGENT", "URL-REGEX", "AND", "OR", "NOT", "PROCESS-NAME", "IN-PORT", "DEST-PORT", "SRC-IP"}; const string_array quanx_rule_type = {basic_types, "USER-AGENT", "HOST", "HOST-SUFFIX", "HOST-KEYWORD"}; std::string getRuleset(RESPONSE_CALLBACK_ARGS) { - /// type: 1 for Surge, 2 for Quantumult X, 3 for Clash domain rule-provider, 4 for Clash ipcidr rule-provider, 5 for Surge DOMAIN-SET + /// type: 1 for Surge, 2 for Quantumult X, 3 for Clash domain rule-provider, 4 for Clash ipcidr rule-provider, 5 for Surge DOMAIN-SET, 6 for Clash classical ruleset std::string url = urlsafe_base64_decode(getUrlArg(argument, "url")), type = getUrlArg(argument, "type"), group = urlsafe_base64_decode(getUrlArg(argument, "group")); std::string output_content, dummy; int type_int = to_int(type, 0); - if(!url.size() || !type.size() || (type_int == 2 && !group.size()) || (type_int < 1 && type_int > 5)) + if(!url.size() || !type.size() || (type_int == 2 && !group.size()) || (type_int < 1 && type_int > 6)) { *status_code = 400; return "Invalid request!"; @@ -136,11 +137,31 @@ std::string getRuleset(RESPONSE_CALLBACK_ARGS) ss << output_content; char delimiter = count(output_content.begin(), output_content.end(), '\n') < 1 ? '\r' : '\n'; - std::string::size_type lineSize; + std::string::size_type lineSize, posb, pose; + auto filterLine = [&]() + { + posb = 0; + pose = strLine.find(','); + if(pose == strLine.npos) + return 1; + posb = pose + 1; + pose = strLine.find(',', posb); + if(pose == strLine.npos) + { + pose = strLine.size(); + if(strLine[pose - 1] == '\r') + pose--; + } + else + pose -= posb + 1; + return 0; + }; + lineSize = output_content.size(); output_content.clear(); + output_content.reserve(lineSize); - if(type_int == 3 || type_int == 4) + if(type_int == 3 || type_int == 4 || type_int == 6) output_content = "payload:\n"; while(getline(ss, strLine, delimiter)) @@ -158,51 +179,35 @@ std::string getRuleset(RESPONSE_CALLBACK_ARGS) case 3: if(!startsWith(strLine, "DOMAIN-SUFFIX,") && !startsWith(strLine, "DOMAIN,")) continue; - vArray = split(strLine, ","); - if(vArray.size() < 2) + if(filterLine()) continue; - vArray[1] = regTrim(vArray[1]); - switch(hash_(vArray[0])) - { - case "DOMAIN-SUFFIX"_hash: - strLine = " - '+." + vArray[1] + "'"; - break; - case "DOMAIN"_hash: - strLine = " - '" + vArray[1] + "'"; - break; - //case "DOMAIN_KEYWORD"_hash: - //strLine = " - '." + vArray[1] + ".*'"; - //break; - } - output_content += strLine + "\n"; + output_content += " - '"; + if(strLine[posb - 2] == 'X') + output_content += "+."; + output_content += strLine.substr(posb, pose); + output_content += "'\n"; continue; case 4: if(!startsWith(strLine, "IP-CIDR,") && !startsWith(strLine, "IP-CIDR6,")) continue; - vArray = split(strLine, ","); - if(vArray.size() < 2) + if(filterLine()) continue; - output_content += " - '" + vArray[1] + "'\n"; + output_content += " - '"; + output_content += strLine.substr(posb, pose); + output_content += "'\n"; continue; case 5: if(!startsWith(strLine, "DOMAIN-SUFFIX,") && !startsWith(strLine, "DOMAIN,")) continue; - vArray = split(strLine, ","); - if(vArray.size() < 2) + if(filterLine()) continue; - vArray[1] = regTrim(vArray[1]); - switch(hash_(vArray[0])) - { - case "DOMAIN-SUFFIX"_hash: - case "DOMAIN"_hash: - strLine = vArray[1]; - break; - //case "DOMAIN_KEYWORD"_hash: - //strLine = " - '." + vArray[1] + ".*'"; - //break; - } - output_content += strLine + "\n"; + output_content += strLine.substr(posb, pose); + output_content += '\n'; continue; + case 6: + if(!std::any_of(clash_rule_type.begin(), clash_rule_type.end(), [&strLine](std::string type){return startsWith(strLine, type);})) + continue; + output_content += " - "; } lineSize = strLine.size(); @@ -225,13 +230,25 @@ std::string getRuleset(RESPONSE_CALLBACK_ARGS) strLine = regReplace(strLine, rule_match_regex, "$1$3"); } } - output_content += strLine + "\n"; + output_content += strLine; + output_content += '\n'; } - if(type_int == 3 && output_content == "payload:\n") - output_content += " - '--placeholder--'"; - if(type_int == 4 && output_content == "payload:\n") - output_content += " - '0.0.0.0/32'"; + if(output_content == "payload:\n") + { + switch(type_int) + { + case 3: + output_content += " - '--placeholder--'"; + break; + case 4: + output_content += " - '0.0.0.0/32'"; + break; + case 6: + output_content += " - 'DOMAIN,--placeholder--'"; + break; + } + } return output_content; } @@ -1185,7 +1202,7 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS) tribool sort_flag = getUrlArg(argument, "sort"), use_sort_script = getUrlArg(argument, "sort_script"); tribool clash_new_field = getUrlArg(argument, "new_name"), clash_script = getUrlArg(argument, "script"), add_insert = getUrlArg(argument, "insert"); tribool scv = getUrlArg(argument, "scv"), fdn = getUrlArg(argument, "fdn"), expand = getUrlArg(argument, "expand"), append_sub_userinfo = getUrlArg(argument, "append_info"); - tribool prepend_insert = getUrlArg(argument, "prepend"); + tribool prepend_insert = getUrlArg(argument, "prepend"), classical = getUrlArg(argument, "classic"); std::string base_content, output_content; string_array extra_group, extra_ruleset, include_remarks = def_include_remarks, exclude_remarks = def_exclude_remarks; @@ -1274,6 +1291,7 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS) ext.filter_deprecated = fdn.get(filter_deprecated); ext.clash_new_field_name = clash_new_field.get(clash_use_new_field_name); ext.clash_script = clash_script.get(); + ext.clash_classical_ruleset = classical.get(); if(!expand) ext.clash_new_field_name = true; else diff --git a/src/subexport.cpp b/src/subexport.cpp index e28ae33..1ad7038 100644 --- a/src/subexport.cpp +++ b/src/subexport.cpp @@ -1010,31 +1010,42 @@ void preprocessNodes(std::vector &nodes, extra_settings &ext) bool failed = true; if(ext.sort_script.size()) { - duk_context *ctx = duktape_init(); - if(ctx) + try { - defer(duk_destroy_heap(ctx);) - if(duktape_peval(ctx, ext.sort_script) == 0) + duk_context *ctx = duktape_init(); + if(ctx) { - auto comparer = [&](const nodeInfo &a, const nodeInfo &b) + defer(duk_destroy_heap(ctx);) + if(duktape_peval(ctx, ext.sort_script) == 0) { - duk_get_global_string(ctx, "compare"); - /// push 2 nodeinfo - duktape_push_nodeinfo(ctx, a); - duktape_push_nodeinfo(ctx, b); - /// call function - duk_pcall(ctx, 2); - return duktape_get_res_int(ctx); - }; - std::sort(nodes.begin(), nodes.end(), comparer); - failed = false; - } - else - { - writeLog(0, "Error when trying to parse script:\n" + duktape_get_err_stack(ctx), LOG_LEVEL_ERROR); - duk_pop(ctx); /// pop err + auto comparer = [&](const nodeInfo &a, const nodeInfo &b) + { + if(a.linkType < 1 || a.linkType > 5) + return 1; + if(b.linkType < 1 || b.linkType > 5) + return 0; + duk_get_global_string(ctx, "compare"); + /// push 2 nodeinfo + duktape_push_nodeinfo(ctx, a); + duktape_push_nodeinfo(ctx, b); + /// call function + duk_pcall(ctx, 2); + return duktape_get_res_int(ctx); + }; + std::sort(nodes.begin(), nodes.end(), comparer); + failed = false; + } + else + { + writeLog(0, "Error when trying to parse script:\n" + duktape_get_err_stack(ctx), LOG_LEVEL_ERROR); + duk_pop(ctx); /// pop err + } } } + catch (std::exception&) + { + //failed + } } if(failed) std::sort(nodes.begin(), nodes.end(), [](const nodeInfo &a, const nodeInfo &b) { @@ -1184,19 +1195,27 @@ void netchToClash(std::vector &nodes, YAML::Node &yamlnode, string_arr break; case SPEEDTEST_MESSAGE_FOUNDSOCKS: singleproxy["type"] = "socks5"; - singleproxy["username"] = username; - singleproxy["password"] = password; - if(std::all_of(password.begin(), password.end(), ::isdigit) && !password.empty()) - singleproxy["password"].SetTag("str"); + if(!username.empty()) + singleproxy["username"] = username; + if(!password.empty()) + { + singleproxy["password"] = password; + if(std::all_of(password.begin(), password.end(), ::isdigit)) + singleproxy["password"].SetTag("str"); + } if(scv) singleproxy["skip-cert-verify"] = true; break; case SPEEDTEST_MESSAGE_FOUNDHTTP: singleproxy["type"] = "http"; - singleproxy["username"] = username; - singleproxy["password"] = password; - if(std::all_of(password.begin(), password.end(), ::isdigit) && !password.empty()) - singleproxy["password"].SetTag("str"); + if(!username.empty()) + singleproxy["username"] = username; + if(!password.empty()) + { + singleproxy["password"] = password; + if(std::all_of(password.begin(), password.end(), ::isdigit)) + singleproxy["password"].SetTag("str"); + } singleproxy["tls"] = type == "HTTPS"; if(scv) singleproxy["skip-cert-verify"] = true; @@ -1348,7 +1367,7 @@ std::string netchToClash(std::vector &nodes, std::string &base_conf, s { if(yamlnode["mode"].IsDefined()) yamlnode["mode"] = ext.clash_script ? "Script" : "Rule"; - renderClashScript(yamlnode, ruleset_content_array, ext.managed_config_prefix, ext.clash_script, ext.overwrite_original_rules); + renderClashScript(yamlnode, ruleset_content_array, ext.managed_config_prefix, ext.clash_script, ext.overwrite_original_rules, ext.clash_classical_ruleset); return YAML::Dump(yamlnode); } diff --git a/src/subexport.h b/src/subexport.h index fda689d..b23b0e5 100644 --- a/src/subexport.h +++ b/src/subexport.h @@ -36,6 +36,7 @@ struct extra_settings tribool udp = false; tribool tfo = false; tribool skip_cert_verify = false; + bool clash_classical_ruleset = false; std::string sort_script; }; diff --git a/src/templates.cpp b/src/templates.cpp index 057e093..da8f41e 100644 --- a/src/templates.cpp +++ b/src/templates.cpp @@ -11,6 +11,13 @@ extern std::string managed_config_prefix; +template T safe_as (const YAML::Node& node) +{ + if(node.IsDefined() && !node.IsNull()) + return node.as(); + return T(); +}; + static inline void parse_json_pointer(nlohmann::json &json, const std::string &path, const std::string &value) { std::string pointer = "/" + replace_all_distinct(path, ".", "/"); @@ -177,12 +184,14 @@ const std::string clash_script_template = R"(def main(ctx, md): return geoips[key] return "{{ match_group }}")"; -const std::string clash_script_group_template = R"({% if rule.has_domain == "true" %} if ctx.rule_providers["{{ rule.name }}_domain"].match(md): +const std::string clash_script_group_template = R"({% if rule.has_domain == "false" and rule.has_ipcidr == "false" %} if ctx.rule_providers["{{ rule.name }}_classical"].match(md): + ctx.log('[Script] matched {{ rule.group }} rule') + return "{{ rule.group }}"{% else %}{% if rule.has_domain == "true" %} if ctx.rule_providers["{{ rule.name }}_domain"].match(md): ctx.log('[Script] matched {{ rule.group }} DOMAIN rule') return "{{ rule.group }}"{% endif %} {% if rule.has_ipcidr == "true" %} if ctx.rule_providers["{{ rule.name }}_ipcidr"].match(md): ctx.log('[Script] matched {{ rule.group }} IP rule') - return "{{ rule.group }}"{% endif %})"; + return "{{ rule.group }}"{% endif %}{% endif %})"; const std::string clash_script_keyword_template = R"( keywords = [{{ rule.keyword }}] for keyword in keywords: @@ -190,7 +199,7 @@ const std::string clash_script_keyword_template = R"( keywords = [{{ rule.keywo ctx.log('[Script] matched {{ rule.group }} DOMAIN-KEYWORD rule') return "{{ rule.group }}")"; -int renderClashScript(YAML::Node &base_rule, std::vector &ruleset_content_array, std::string remote_path_prefix, bool script, bool overwrite_original_rules) +int renderClashScript(YAML::Node &base_rule, std::vector &ruleset_content_array, std::string remote_path_prefix, bool script, bool overwrite_original_rules, bool clash_classical_ruleset) { nlohmann::json data; std::string match_group, geoips, retrieved_rules; @@ -199,10 +208,10 @@ int renderClashScript(YAML::Node &base_rule, std::vector &rules string_array vArray, groups; string_map keywords, urls, names; std::map has_domain, has_ipcidr; - YAML::Node rules; + string_array rules; if(!overwrite_original_rules && base_rule["rules"].IsDefined()) - rules = base_rule["rules"]; + rules = safe_as(base_rule["rules"]); for(ruleset_content &x : ruleset_content_array) { @@ -229,7 +238,7 @@ int renderClashScript(YAML::Node &base_rule, std::vector &rules strLine += "," + rule_group; if(std::count(strLine.begin(), strLine.end(), ',') > 2) strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2"); - rules.push_back(strLine); + rules.emplace_back(strLine); continue; } else @@ -241,6 +250,13 @@ int renderClashScript(YAML::Node &base_rule, std::vector &rules rule_name = std::to_string(hash_(rule_path)); names[rule_name] = rule_group; urls[rule_name] = rule_path; + if(clash_classical_ruleset) + { + groups.emplace_back(rule_name); + if(!script) + rules.emplace_back("RULE-SET," + rule_group + "_" + rule_name + "_classical," + rule_group); + continue; + } } else continue; @@ -286,7 +302,7 @@ int renderClashScript(YAML::Node &base_rule, std::vector &rules strLine += "," + rule_group; if(std::count(strLine.begin(), strLine.end(), ',') > 2) strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2"); - rules.push_back(strLine); + rules.emplace_back(strLine); } } else if(startsWith(strLine, "DOMAIN,") || startsWith(strLine, "DOMAIN-SUFFIX,")) @@ -295,9 +311,9 @@ int renderClashScript(YAML::Node &base_rule, std::vector &rules has_ipcidr[rule_name] = true; } if(has_domain[rule_name] && !script) - rules.push_back("RULE-SET," + rule_group + "_" + rule_name + "_domain," + rule_group); + rules.emplace_back("RULE-SET," + rule_group + "_" + rule_name + "_domain," + rule_group); if(has_ipcidr[rule_name] && !script) - rules.push_back("RULE-SET," + rule_group + "_" + rule_name + "_ipcidr," + rule_group); + rules.emplace_back("RULE-SET," + rule_group + "_" + rule_name + "_ipcidr," + rule_group); if(std::find(groups.begin(), groups.end(), rule_name) == groups.end()) groups.emplace_back(rule_name); } @@ -308,19 +324,29 @@ int renderClashScript(YAML::Node &base_rule, std::vector &rules std::string json_path = "rules." + std::to_string(index) + "."; std::string url = urls[x], keyword = keywords[x], name = names[x]; bool group_has_domain = has_domain[x], group_has_ipcidr = has_ipcidr[x]; - if(group_has_domain) + if(clash_classical_ruleset) { - base_rule["rule-providers"][name + "_" + x + "_domain"]["type"] = "http"; - base_rule["rule-providers"][name + "_" + x + "_domain"]["behavior"] = "domain"; - base_rule["rule-providers"][name + "_" + x + "_domain"]["url"] = remote_path_prefix + "/getruleset?type=3&url=" + urlsafe_base64_encode(url); - base_rule["rule-providers"][name + "_" + x + "_domain"]["path"] = "./rule-providers_" + x + "_domain.yaml"; + base_rule["rule-providers"][name + "_" + x + "_classical"]["type"] = "http"; + base_rule["rule-providers"][name + "_" + x + "_classical"]["behavior"] = "classical"; + base_rule["rule-providers"][name + "_" + x + "_classical"]["url"] = remote_path_prefix + "/getruleset?type=6&url=" + urlsafe_base64_encode(url); + base_rule["rule-providers"][name + "_" + x + "_classical"]["path"] = "./providers/rule-provider_" + x + "_classical.yaml"; } - if(group_has_ipcidr) + else { - base_rule["rule-providers"][name + "_" + x + "_ipcidr"]["type"] = "http"; - base_rule["rule-providers"][name + "_" + x + "_ipcidr"]["behavior"] = "ipcidr"; - base_rule["rule-providers"][name + "_" + x + "_ipcidr"]["url"] = remote_path_prefix + "/getruleset?type=4&url=" + urlsafe_base64_encode(url); - base_rule["rule-providers"][name + "_" + x + "_ipcidr"]["path"] = "./rule-providers_" + x + "_ipcidr.yaml"; + if(group_has_domain) + { + base_rule["rule-providers"][name + "_" + x + "_domain"]["type"] = "http"; + base_rule["rule-providers"][name + "_" + x + "_domain"]["behavior"] = "domain"; + base_rule["rule-providers"][name + "_" + x + "_domain"]["url"] = remote_path_prefix + "/getruleset?type=3&url=" + urlsafe_base64_encode(url); + base_rule["rule-providers"][name + "_" + x + "_domain"]["path"] = "./providers/rule-provider_" + x + "_domain.yaml"; + } + if(group_has_ipcidr) + { + base_rule["rule-providers"][name + "_" + x + "_ipcidr"]["type"] = "http"; + base_rule["rule-providers"][name + "_" + x + "_ipcidr"]["behavior"] = "ipcidr"; + base_rule["rule-providers"][name + "_" + x + "_ipcidr"]["url"] = remote_path_prefix + "/getruleset?type=4&url=" + urlsafe_base64_encode(url); + base_rule["rule-providers"][name + "_" + x + "_ipcidr"]["path"] = "./providers/rule-provider_" + x + "_ipcidr.yaml"; + } } if(script) { diff --git a/src/templates.h b/src/templates.h index 9188f4c..c1d3e9c 100644 --- a/src/templates.h +++ b/src/templates.h @@ -16,6 +16,6 @@ struct template_args }; int render_template(const std::string &content, const template_args &vars, std::string &output, const std::string &include_scope = "template"); -int renderClashScript(YAML::Node &base_rule, std::vector &ruleset_content_array, std::string remote_path_prefix, bool script, bool overwrite_original_rules); +int renderClashScript(YAML::Node &base_rule, std::vector &ruleset_content_array, std::string remote_path_prefix, bool script, bool overwrite_original_rules, bool clash_classic_ruleset); #endif // TEMPLATES_H_INCLUDED