taixingyiji/subconverter
1
0
Fork 0
mirror of https://github.com/asdlokj1qpi233/subconverter.git synced 2025-10-28 04:12:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Enhancements

Browse Source 
Add no-resolve to generated Clash rule-set if exists (#679).
Do not set CURLOPT_USERAGENT if User-Agent header is provided.
Print curl verbose logs with standard format.
Change thread id in logs to thread name.
Optimize generation of rule contents.
This commit is contained in:
Tindy X
2023-12-05 18:37:14 +08:00
parent adaef88501
commit a0e25158f5
5 changed files with 130 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
src/generator/config/ruleconvert.cpp
View File

@@ -95,6 +95,32 @@ std::string convertRuleset(const std::string &content, int type)
}
}
static std::string transformRuleToCommon(string_view_array &temp, const std::string &input, const std::string &group, bool no_resolve_only = false)
{
temp.clear();
std::string strLine;
split(temp, input, ',');
if(temp.size() < 2)
{
strLine = temp[0];
strLine += ",";
strLine += group;
}
else
{
strLine = temp[0];
strLine += ",";
strLine += temp[1];
strLine += ",";
strLine += group;
if(temp.size() > 2 && (!no_resolve_only || temp[2] == "no-resolve"))
{
strLine += ",";
strLine += temp[2];
}
}
return strLine;
}
void rulesetToClash(YAML::Node &base_rule, std::vector<RulesetContent> &ruleset_content_array, bool overwrite_original_rules, bool new_field_name)
{
@@ -108,6 +134,7 @@ void rulesetToClash(YAML::Node &base_rule, std::vector<RulesetContent> &ruleset_
if(!overwrite_original_rules && base_rule[field_name].IsDefined())
rules = base_rule[field_name];
std::vector<std::string_view> temp(4);
for(RulesetContent &x : ruleset_content_array)
{
if(global.maxAllowedRules && total_rules > global.maxAllowedRules)
@@ -124,10 +151,8 @@ void rulesetToClash(YAML::Node &base_rule, std::vector<RulesetContent> &ruleset_
strLine = retrieved_rules.substr(2);
if(startsWith(strLine, "FINAL"))
strLine.replace(0, 5, "MATCH");
strLine += "," + rule_group;
if(count_least(strLine, ',', 3))
strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2");
allRules.emplace_back(std::move(strLine));
strLine = transformRuleToCommon(temp, strLine, rule_group);
allRules.emplace_back(strLine);
total_rules++;
continue;
}
@@ -152,11 +177,8 @@ void rulesetToClash(YAML::Node &base_rule, std::vector<RulesetContent> &ruleset_
strLine.erase(strLine.find("//"));
strLine = trimWhitespace(strLine);
}
strLine += "," + rule_group;
if(count_least(strLine, ',', 3))
strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2");
allRules.emplace_back(std::move(strLine));
//rules.push_back(strLine);
strLine = transformRuleToCommon(temp, strLine, rule_group);
allRules.emplace_back(strLine);
}
}
@@ -183,6 +205,7 @@ std::string rulesetToClashStr(YAML::Node &base_rule, std::vector<RulesetContent>
}
base_rule.remove(field_name);
string_view_array temp(4);
for(RulesetContent &x : ruleset_content_array)
{
if(global.maxAllowedRules && total_rules > global.maxAllowedRules)
@@ -199,9 +222,7 @@ std::string rulesetToClashStr(YAML::Node &base_rule, std::vector<RulesetContent>
strLine = retrieved_rules.substr(2);
if(startsWith(strLine, "FINAL"))
strLine.replace(0, 5, "MATCH");
strLine += "," + rule_group;
if(count_least(strLine, ',', 3))
strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2");
strLine = transformRuleToCommon(temp, strLine, rule_group);
output_content += " - " + strLine + "\n";
total_rules++;
continue;
@@ -227,9 +248,7 @@ std::string rulesetToClashStr(YAML::Node &base_rule, std::vector<RulesetContent>
strLine.erase(strLine.find("//"));
strLine = trimWhitespace(strLine);
}
strLine += "," + rule_group;
if(count_least(strLine, ',', 3))
strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2");
strLine = transformRuleToCommon(temp, strLine, rule_group);
output_content += " - " + strLine + "\n";
total_rules++;
}
@@ -277,6 +296,7 @@ void rulesetToSurge(INIReader &base_rule, std::vector<RulesetContent> &ruleset_c
const std::string rule_match_regex = "^(.*?,.*?)(,.*)(,.*)$";
string_view_array temp(4);
for(RulesetContent &x : ruleset_content_array)
{
if(global.maxAllowedRules && total_rules > global.maxAllowedRules)
@@ -289,21 +309,17 @@ void rulesetToSurge(INIReader &base_rule, std::vector<RulesetContent> &ruleset_c
strLine = x.rule_content.get().substr(2);
if(strLine == "MATCH")
strLine = "FINAL";
strLine += "," + rule_group;
if(surge_ver == -1 || surge_ver == -2)
{
if(count_least(strLine, ',', 3) && regReplace(strLine, rule_match_regex, "$2") == ",no-resolve")
strLine = regReplace(strLine, rule_match_regex, "$1$3$2");
else
strLine = regReplace(strLine, rule_match_regex, "$1$3");
strLine = transformRuleToCommon(temp, strLine, rule_group, true);
}
else
{
if(!startsWith(strLine, "AND") && !startsWith(strLine, "OR") && !startsWith(strLine, "NOT") && count_least(strLine, ',', 3))
strLine = regReplace(strLine, rule_match_regex, "$1$3$2");
if(!startsWith(strLine, "AND") && !startsWith(strLine, "OR") && !startsWith(strLine, "NOT"))
strLine = transformRuleToCommon(temp, strLine, rule_group);
}
strLine = replaceAllDistinct(strLine, ",,", ",");
allRules.emplace_back(std::move(strLine));
allRules.emplace_back(strLine);
total_rules++;
continue;
}
@@ -436,15 +452,12 @@ void rulesetToSurge(INIReader &base_rule, std::vector<RulesetContent> &ruleset_c
{
if(startsWith(strLine, "IP-CIDR6"))
strLine.replace(0, 8, "IP6-CIDR");
if(count_least(strLine, ',', 3) && regReplace(strLine, rule_match_regex, "$2") == ",no-resolve")
strLine = regReplace(strLine, rule_match_regex, "$1$3$2");
else
strLine = regReplace(strLine, rule_match_regex, "$1$3");
strLine = transformRuleToCommon(temp, strLine, rule_group, true);
}
else
{
if(!startsWith(strLine, "AND") && !startsWith(strLine, "OR") && !startsWith(strLine, "NOT") && count_least(strLine, ',', 3))
strLine = regReplace(strLine, rule_match_regex, "$1$3$2");
if(!startsWith(strLine, "AND") && !startsWith(strLine, "OR") && !startsWith(strLine, "NOT"))
strLine = transformRuleToCommon(temp, strLine, rule_group);
}
allRules.emplace_back(std::move(strLine));
total_rules++;

34
src/generator/template/templates.cpp
View File

@@ -378,7 +378,7 @@ int renderClashScript(YAML::Node &base_rule, std::vector<RulesetContent> &rulese
}
if(!script)
rules.emplace_back("RULE-SET," + rule_name + "," + rule_group);
groups.emplace_back(std::move(rule_name));
groups.emplace_back(rule_name);
continue;
}
if(!remote_path_prefix.empty())
@@ -398,7 +398,7 @@ int renderClashScript(YAML::Node &base_rule, std::vector<RulesetContent> &rulese
{
if(!script)
rules.emplace_back("RULE-SET," + rule_name + "," + rule_group);
groups.emplace_back(std::move(rule_name));
groups.emplace_back(rule_name);
continue;
}
}
@@ -419,6 +419,7 @@ int renderClashScript(YAML::Node &base_rule, std::vector<RulesetContent> &rulese
strStrm.clear();
strStrm<<retrieved_rules;
std::string::size_type lineSize;
bool has_no_resolve = false;
while(getline(strStrm, strLine, delimiter))
{
lineSize = strLine.size();
@@ -441,23 +442,40 @@ int renderClashScript(YAML::Node &base_rule, std::vector<RulesetContent> &rulese
}
else
{
strLine += "," + rule_group;
if(count_least(strLine, ',', 3))
strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2");
rules.emplace_back(std::move(strLine));
vArray = split(strLine, ",");
if(vArray.size() < 2)
{
strLine = vArray[0] + "," + rule_group;
}
else
{
strLine = vArray[0] + "," + vArray[1] + "," + rule_group;
if(vArray.size() > 2)
strLine += "," + vArray[2];
}
rules.emplace_back(strLine);
}
}
else if(!has_domain[rule_name] && (startsWith(strLine, "DOMAIN,") || startsWith(strLine, "DOMAIN-SUFFIX,")))
has_domain[rule_name] = true;
else if(!has_ipcidr[rule_name] && (startsWith(strLine, "IP-CIDR,") || startsWith(strLine, "IP-CIDR6,")))
{
has_ipcidr[rule_name] = true;
if(strLine.find(",no-resolve") != std::string::npos)
has_no_resolve = true;
}
}
if(has_domain[rule_name] && !script)
rules.emplace_back("RULE-SET," + rule_name + "_domain," + rule_group);
if(has_ipcidr[rule_name] && !script)
rules.emplace_back("RULE-SET," + rule_name + "_ipcidr," + rule_group);
{
if(has_no_resolve)
rules.emplace_back("RULE-SET," + rule_name + "_ipcidr," + rule_group + ",no-resolve");
else
rules.emplace_back("RULE-SET," + rule_name + "_ipcidr," + rule_group);
}
if(std::find(groups.begin(), groups.end(), rule_name) == groups.end())
groups.emplace_back(std::move(rule_name));
groups.emplace_back(rule_name);
}
}
for(std::string &x : groups)

58
src/handler/webget.cpp
View File

@@ -94,7 +94,7 @@ public:
RWLock cache_rw_lock;
//std::string user_agent_str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36";
static std::string user_agent_str = "subconverter/" VERSION " cURL/" LIBCURL_VERSION;
static auto user_agent_str = "subconverter/" VERSION " cURL/" LIBCURL_VERSION;
struct curl_progress_data
{
@@ -113,7 +113,7 @@ static inline void curl_init()
static int writer(char *data, size_t size, size_t nmemb, std::string *writerData)
{
if(writerData == NULL)
if(writerData == nullptr)
return 0;
writerData->append(data, size*nmemb);
@@ -133,7 +133,7 @@ static int size_checker(void *clientp, curl_off_t dltotal, curl_off_t dlnow, cur
{
if(clientp)
{
curl_progress_data *data = reinterpret_cast<curl_progress_data*>(clientp);
auto *data = reinterpret_cast<curl_progress_data*>(clientp);
if(data->size_limit)
{
if(dlnow > data->size_limit)
@@ -143,10 +143,37 @@ static int size_checker(void *clientp, curl_off_t dltotal, curl_off_t dlnow, cur
return 0;
}
static int logger(CURL *handle, curl_infotype type, char *data, size_t size, void *userptr)
{
(void)handle;
(void)userptr;
std::string prefix;
switch(type)
{
case CURLINFO_TEXT:
prefix = "CURL_INFO";
break;
case CURLINFO_HEADER_IN:
case CURLINFO_HEADER_OUT:
prefix = "CURL_HEADER";
break;
case CURLINFO_DATA_IN:
case CURLINFO_DATA_OUT:
default:
return 0;
}
std::string content(data, size);
if(content.back() == '\n')
content.pop_back();
writeLog(0, prefix + ": " + content, LOG_LEVEL_VERBOSE);
return 0;
}
static inline void curl_set_common_options(CURL *curl_handle, const char *url, curl_progress_data *data)
{
curl_easy_setopt(curl_handle, CURLOPT_URL, url);
curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, global.logLevel == LOG_LEVEL_VERBOSE ? 1L : 0L);
curl_easy_setopt(curl_handle, CURLOPT_DEBUGFUNCTION, logger);
curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 0L);
curl_easy_setopt(curl_handle, CURLOPT_NOSIGNAL, 1L);
curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1L);
@@ -154,7 +181,6 @@ static inline void curl_set_common_options(CURL *curl_handle, const char *url, c
curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 0L);
curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, 15L);
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, user_agent_str.data());
curl_easy_setopt(curl_handle, CURLOPT_COOKIEFILE, "");
if(data)
{
@@ -170,18 +196,18 @@ static int curlGet(const FetchArgument &argument, FetchResult &result)
{
CURL *curl_handle;
std::string *data = result.content, new_url = argument.url;
struct curl_slist *list = NULL;
defer(curl_slist_free_all(list);)
curl_slist *header_list = nullptr;
defer(curl_slist_free_all(header_list);)
long retVal = 0;
curl_init();
curl_handle = curl_easy_init();
if(argument.proxy.size())
if(!argument.proxy.empty())
{
if(startsWith(argument.proxy, "cors:"))
{
list = curl_slist_append(list, "X-Requested-With: subconverter " VERSION);
header_list = curl_slist_append(header_list, "X-Requested-With: subconverter " VERSION);
new_url = argument.proxy.substr(5) + argument.url;
}
else
@@ -190,16 +216,20 @@ static int curlGet(const FetchArgument &argument, FetchResult &result)
curl_progress_data limit;
limit.size_limit = global.maxAllowedDownloadSize;
curl_set_common_options(curl_handle, new_url.data(), &limit);
list = curl_slist_append(list, "Content-Type: application/json;charset=utf-8");
header_list = curl_slist_append(header_list, "Content-Type: application/json;charset=utf-8");
if(argument.request_headers)
{
for(auto &x : *argument.request_headers)
list = curl_slist_append(list, (x.first + ": " + x.second).data());
{
header_list = curl_slist_append(header_list, (x.first + ": " + x.second).data());
}
if(!argument.request_headers->contains("User-Agent"))
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, user_agent_str);
}
list = curl_slist_append(list, "SubConverter-Request: 1");
list = curl_slist_append(list, "SubConverter-Version: " VERSION);
if(list)
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list);
header_list = curl_slist_append(header_list, "SubConverter-Request: 1");
header_list = curl_slist_append(header_list, "SubConverter-Version: " VERSION);
if(header_list)
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, header_list);
if(result.content)
{

23
src/utils/logger.cpp
View File

@@ -13,10 +13,10 @@ std::string getTime(int type)
time_t lt;
char tmpbuf[32], cMillis[7];
std::string format;
timeval tv;
gettimeofday(&tv, NULL);
timeval tv = {};
gettimeofday(&tv, nullptr);
snprintf(cMillis, 7, "%.6ld", (long)tv.tv_usec);
lt = time(NULL);
lt = time(nullptr);
struct tm *local = localtime(&lt);
switch(type)
{
@@ -27,19 +27,30 @@ std::string getTime(int type)
format = "%Y/%m/%d %a %H:%M:%S." + std::string(cMillis);
break;
case 3:
default:
format = "%Y-%m-%d %H:%M:%S";
break;
}
strftime(tmpbuf, 32, format.data(), local);
return std::string(tmpbuf);
return {tmpbuf};
}
static std::string get_thread_name()
{
static std::atomic_int counter = 0;
thread_local static std::string name = "Thread-" + std::to_string(counter++);
return name;
}
std::mutex log_mutex;
void writeLog(int type, const std::string &content, int level)
{
if(level > global.logLevel)
return;
std::lock_guard<std::mutex> lock(log_mutex);
const char *levels[] = {"[FATL]", "[ERRO]", "[WARN]", "[INFO]", "[DEBG]", "[VERB]"};
std::cerr<<getTime(2)<<" ["<<getpid()<<" "<<std::this_thread::get_id()<<"]"<<levels[level % 6];
std::cerr<<getTime(2)<<" ["<<getpid()<<" "<<get_thread_name()<<"]"<<levels[level % 6];
std::cerr<<" "<<content<<"\n";
}
@@ -53,7 +64,7 @@ std::string demangle(const char* name)
{
int status = -4;
std::unique_ptr<char, void(*)(void*)> res {
abi::__cxa_demangle(name, NULL, NULL, &status),
abi::__cxa_demangle(name, nullptr, nullptr, &status),
std::free
};
return (status == 0) ? res.get() : name;

1
src/utils/string.h
View File

@@ -10,6 +10,7 @@
using string = std::string;
using string_size = std::string::size_type;
using string_array = std::vector<std::string>;
using string_view_array = std::vector<std::string_view>;
using string_map = std::map<std::string, std::string>;
using string_multimap = std::multimap<std::string, std::string>;
using string_pair_array = std::vector<std::pair<std::string, std::string>>;