Enhancements

Add no-resolve to generated Clash rule-set if exists (#679).
Do not set CURLOPT_USERAGENT if User-Agent header is provided.
Print curl verbose logs with standard format.
Change thread id in logs to thread name.
Optimize generation of rule contents.

src/generator/config/ruleconvert.cpp

@@ -95,6 +95,32 @@ std::string convertRuleset(const std::string &content, int type)
     }
 }
 
+static std::string transformRuleToCommon(string_view_array &temp, const std::string &input, const std::string &group, bool no_resolve_only = false)
+{
+    temp.clear();
+    std::string strLine;
+    split(temp, input, ',');
+    if(temp.size() < 2)
+    {
+        strLine = temp[0];
+        strLine += ",";
+        strLine += group;
+    }
+    else
+    {
+        strLine = temp[0];
+        strLine += ",";
+        strLine += temp[1];
+        strLine += ",";
+        strLine += group;
+        if(temp.size() > 2 && (!no_resolve_only || temp[2] == "no-resolve"))
+        {
+            strLine += ",";
+            strLine += temp[2];
+        }
+    }
+    return strLine;
+}
 
 void rulesetToClash(YAML::Node &base_rule, std::vector<RulesetContent> &ruleset_content_array, bool overwrite_original_rules, bool new_field_name)
 {
@@ -108,6 +134,7 @@ void rulesetToClash(YAML::Node &base_rule, std::vector<RulesetContent> &ruleset_
     if(!overwrite_original_rules && base_rule[field_name].IsDefined())
         rules = base_rule[field_name];
 
+    std::vector<std::string_view> temp(4);
     for(RulesetContent &x : ruleset_content_array)
     {
         if(global.maxAllowedRules && total_rules > global.maxAllowedRules)
@@ -124,10 +151,8 @@ void rulesetToClash(YAML::Node &base_rule, std::vector<RulesetContent> &ruleset_
             strLine = retrieved_rules.substr(2);
             if(startsWith(strLine, "FINAL"))
                 strLine.replace(0, 5, "MATCH");
-            strLine += "," + rule_group;
+            strLine = transformRuleToCommon(temp, strLine, rule_group);
-            if(count_least(strLine, ',', 3))
+            allRules.emplace_back(strLine);
-                strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2");
-            allRules.emplace_back(std::move(strLine));
             total_rules++;
             continue;
         }
@@ -152,11 +177,8 @@ void rulesetToClash(YAML::Node &base_rule, std::vector<RulesetContent> &ruleset_
                 strLine.erase(strLine.find("//"));
                 strLine = trimWhitespace(strLine);
             }
-            strLine += "," + rule_group;
+            strLine = transformRuleToCommon(temp, strLine, rule_group);
-            if(count_least(strLine, ',', 3))
+            allRules.emplace_back(strLine);
-                strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2");
-            allRules.emplace_back(std::move(strLine));
-            //rules.push_back(strLine);
         }
     }
 
@@ -183,6 +205,7 @@ std::string rulesetToClashStr(YAML::Node &base_rule, std::vector<RulesetContent>
     }
     base_rule.remove(field_name);
 
+    string_view_array temp(4);
     for(RulesetContent &x : ruleset_content_array)
     {
         if(global.maxAllowedRules && total_rules > global.maxAllowedRules)
@@ -199,9 +222,7 @@ std::string rulesetToClashStr(YAML::Node &base_rule, std::vector<RulesetContent>
             strLine = retrieved_rules.substr(2);
             if(startsWith(strLine, "FINAL"))
                 strLine.replace(0, 5, "MATCH");
-            strLine += "," + rule_group;
+            strLine = transformRuleToCommon(temp, strLine, rule_group);
-            if(count_least(strLine, ',', 3))
-                strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2");
             output_content += "  - " + strLine + "\n";
             total_rules++;
             continue;
@@ -227,9 +248,7 @@ std::string rulesetToClashStr(YAML::Node &base_rule, std::vector<RulesetContent>
                 strLine.erase(strLine.find("//"));
                 strLine = trimWhitespace(strLine);
             }
-            strLine += "," + rule_group;
+            strLine = transformRuleToCommon(temp, strLine, rule_group);
-            if(count_least(strLine, ',', 3))
-                strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2");
             output_content += "  - " + strLine + "\n";
             total_rules++;
         }
@@ -277,6 +296,7 @@ void rulesetToSurge(INIReader &base_rule, std::vector<RulesetContent> &ruleset_c
 
     const std::string rule_match_regex = "^(.*?,.*?)(,.*)(,.*)$";
 
+    string_view_array temp(4);
     for(RulesetContent &x : ruleset_content_array)
     {
         if(global.maxAllowedRules && total_rules > global.maxAllowedRules)
@@ -289,21 +309,17 @@ void rulesetToSurge(INIReader &base_rule, std::vector<RulesetContent> &ruleset_c
             strLine = x.rule_content.get().substr(2);
             if(strLine == "MATCH")
                 strLine = "FINAL";
-            strLine += "," + rule_group;
             if(surge_ver == -1 || surge_ver == -2)
             {
-                if(count_least(strLine, ',', 3) && regReplace(strLine, rule_match_regex, "$2") == ",no-resolve")
+                strLine = transformRuleToCommon(temp, strLine, rule_group, true);
-                    strLine = regReplace(strLine, rule_match_regex, "$1$3$2");
-                else
-                    strLine = regReplace(strLine, rule_match_regex, "$1$3");
             }
             else
             {
-                if(!startsWith(strLine, "AND") && !startsWith(strLine, "OR") && !startsWith(strLine, "NOT") && count_least(strLine, ',', 3))
+                if(!startsWith(strLine, "AND") && !startsWith(strLine, "OR") && !startsWith(strLine, "NOT"))
-                    strLine = regReplace(strLine, rule_match_regex, "$1$3$2");
+                    strLine = transformRuleToCommon(temp, strLine, rule_group);
             }
             strLine = replaceAllDistinct(strLine, ",,", ",");
-            allRules.emplace_back(std::move(strLine));
+            allRules.emplace_back(strLine);
             total_rules++;
             continue;
         }
@@ -436,15 +452,12 @@ void rulesetToSurge(INIReader &base_rule, std::vector<RulesetContent> &ruleset_c
                 {
                     if(startsWith(strLine, "IP-CIDR6"))
                         strLine.replace(0, 8, "IP6-CIDR");
-                    if(count_least(strLine, ',', 3) && regReplace(strLine, rule_match_regex, "$2") == ",no-resolve")
+                    strLine = transformRuleToCommon(temp, strLine, rule_group, true);
-                        strLine = regReplace(strLine, rule_match_regex, "$1$3$2");
-                    else
-                        strLine = regReplace(strLine, rule_match_regex, "$1$3");
                 }
                 else
                 {
-                    if(!startsWith(strLine, "AND") && !startsWith(strLine, "OR") && !startsWith(strLine, "NOT") && count_least(strLine, ',', 3))
+                    if(!startsWith(strLine, "AND") && !startsWith(strLine, "OR") && !startsWith(strLine, "NOT"))
-                        strLine = regReplace(strLine, rule_match_regex, "$1$3$2");
+                        strLine = transformRuleToCommon(temp, strLine, rule_group);
                 }
                 allRules.emplace_back(std::move(strLine));
                 total_rules++;

src/generator/template/templates.cpp

@@ -378,7 +378,7 @@ int renderClashScript(YAML::Node &base_rule, std::vector<RulesetContent> &rulese
                 }
                 if(!script)
                     rules.emplace_back("RULE-SET," + rule_name + "," + rule_group);
-                groups.emplace_back(std::move(rule_name));
+                groups.emplace_back(rule_name);
                 continue;
             }
             if(!remote_path_prefix.empty())
@@ -398,7 +398,7 @@ int renderClashScript(YAML::Node &base_rule, std::vector<RulesetContent> &rulese
                     {
                         if(!script)
                             rules.emplace_back("RULE-SET," + rule_name + "," + rule_group);
-                        groups.emplace_back(std::move(rule_name));
+                        groups.emplace_back(rule_name);
                         continue;
                     }
                 }
@@ -419,6 +419,7 @@ int renderClashScript(YAML::Node &base_rule, std::vector<RulesetContent> &rulese
             strStrm.clear();
             strStrm<<retrieved_rules;
             std::string::size_type lineSize;
+            bool has_no_resolve = false;
             while(getline(strStrm, strLine, delimiter))
             {
                 lineSize = strLine.size();
@@ -441,23 +442,40 @@ int renderClashScript(YAML::Node &base_rule, std::vector<RulesetContent> &rulese
                     }
                     else
                     {
-                        strLine += "," + rule_group;
+                        vArray = split(strLine, ",");
-                        if(count_least(strLine, ',', 3))
+                        if(vArray.size() < 2)
-                            strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2");
+                        {
-                        rules.emplace_back(std::move(strLine));
+                            strLine = vArray[0] + "," + rule_group;
+                        }
+                        else
+                        {
+                            strLine = vArray[0] + "," + vArray[1] + "," + rule_group;
+                            if(vArray.size() > 2)
+                                strLine += "," + vArray[2];
+                        }
+                        rules.emplace_back(strLine);
                     }
                 }
                 else if(!has_domain[rule_name] && (startsWith(strLine, "DOMAIN,") || startsWith(strLine, "DOMAIN-SUFFIX,")))
                     has_domain[rule_name] = true;
                 else if(!has_ipcidr[rule_name] && (startsWith(strLine, "IP-CIDR,") || startsWith(strLine, "IP-CIDR6,")))
+                {
                     has_ipcidr[rule_name] = true;
+                    if(strLine.find(",no-resolve") != std::string::npos)
+                        has_no_resolve = true;
+                }
             }
             if(has_domain[rule_name] && !script)
                 rules.emplace_back("RULE-SET," + rule_name + "_domain," + rule_group);
             if(has_ipcidr[rule_name] && !script)
-                rules.emplace_back("RULE-SET," + rule_name + "_ipcidr," + rule_group);
+            {
+                if(has_no_resolve)
+                    rules.emplace_back("RULE-SET," + rule_name + "_ipcidr," + rule_group + ",no-resolve");
+                else
+                    rules.emplace_back("RULE-SET," + rule_name + "_ipcidr," + rule_group);
+            }
             if(std::find(groups.begin(), groups.end(), rule_name) == groups.end())
-                groups.emplace_back(std::move(rule_name));
+                groups.emplace_back(rule_name);
         }
     }
     for(std::string &x : groups)

src/handler/webget.cpp

@@ -94,7 +94,7 @@ public:
 RWLock cache_rw_lock;
 
 //std::string user_agent_str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36";
-static std::string user_agent_str = "subconverter/" VERSION " cURL/" LIBCURL_VERSION;
+static auto user_agent_str = "subconverter/" VERSION " cURL/" LIBCURL_VERSION;
 
 struct curl_progress_data
 {
@@ -113,7 +113,7 @@ static inline void curl_init()
 
 static int writer(char *data, size_t size, size_t nmemb, std::string *writerData)
 {
-    if(writerData == NULL)
+    if(writerData == nullptr)
         return 0;
 
     writerData->append(data, size*nmemb);
@@ -133,7 +133,7 @@ static int size_checker(void *clientp, curl_off_t dltotal, curl_off_t dlnow, cur
 {
     if(clientp)
     {
-        curl_progress_data *data = reinterpret_cast<curl_progress_data*>(clientp);
+        auto *data = reinterpret_cast<curl_progress_data*>(clientp);
         if(data->size_limit)
         {
             if(dlnow > data->size_limit)
@@ -143,10 +143,37 @@ static int size_checker(void *clientp, curl_off_t dltotal, curl_off_t dlnow, cur
     return 0;
 }
 
+static int logger(CURL *handle, curl_infotype type, char *data, size_t size, void *userptr)
+{
+    (void)handle;
+    (void)userptr;
+    std::string prefix;
+    switch(type)
+    {
+    case CURLINFO_TEXT:
+        prefix = "CURL_INFO";
+        break;
+    case CURLINFO_HEADER_IN:
+    case CURLINFO_HEADER_OUT:
+        prefix = "CURL_HEADER";
+        break;
+    case CURLINFO_DATA_IN:
+    case CURLINFO_DATA_OUT:
+    default:
+        return 0;
+    }
+    std::string content(data, size);
+    if(content.back() == '\n')
+        content.pop_back();
+    writeLog(0, prefix + ": " + content, LOG_LEVEL_VERBOSE);
+    return 0;
+}
+
 static inline void curl_set_common_options(CURL *curl_handle, const char *url, curl_progress_data *data)
 {
     curl_easy_setopt(curl_handle, CURLOPT_URL, url);
     curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, global.logLevel == LOG_LEVEL_VERBOSE ? 1L : 0L);
+    curl_easy_setopt(curl_handle, CURLOPT_DEBUGFUNCTION, logger);
     curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 0L);
     curl_easy_setopt(curl_handle, CURLOPT_NOSIGNAL, 1L);
     curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1L);
@@ -154,7 +181,6 @@ static inline void curl_set_common_options(CURL *curl_handle, const char *url, c
     curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 0L);
     curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 0L);
     curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, 15L);
-    curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, user_agent_str.data());
     curl_easy_setopt(curl_handle, CURLOPT_COOKIEFILE, "");
     if(data)
     {
@@ -170,18 +196,18 @@ static int curlGet(const FetchArgument &argument, FetchResult &result)
 {
     CURL *curl_handle;
     std::string *data = result.content, new_url = argument.url;
-    struct curl_slist *list = NULL;
+    curl_slist *header_list = nullptr;
-    defer(curl_slist_free_all(list);)
+    defer(curl_slist_free_all(header_list);)
     long retVal = 0;
 
     curl_init();
 
     curl_handle = curl_easy_init();
-    if(argument.proxy.size())
+    if(!argument.proxy.empty())
     {
         if(startsWith(argument.proxy, "cors:"))
         {
-            list = curl_slist_append(list, "X-Requested-With: subconverter " VERSION);
+            header_list = curl_slist_append(header_list, "X-Requested-With: subconverter " VERSION);
             new_url = argument.proxy.substr(5) + argument.url;
         }
         else
@@ -190,16 +216,20 @@ static int curlGet(const FetchArgument &argument, FetchResult &result)
     curl_progress_data limit;
     limit.size_limit = global.maxAllowedDownloadSize;
     curl_set_common_options(curl_handle, new_url.data(), &limit);
-    list = curl_slist_append(list, "Content-Type: application/json;charset=utf-8");
+    header_list = curl_slist_append(header_list, "Content-Type: application/json;charset=utf-8");
     if(argument.request_headers)
     {
         for(auto &x : *argument.request_headers)
-            list = curl_slist_append(list, (x.first + ": " + x.second).data());
+        {
+            header_list = curl_slist_append(header_list, (x.first + ": " + x.second).data());
+        }
+        if(!argument.request_headers->contains("User-Agent"))
+            curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, user_agent_str);
     }
-    list = curl_slist_append(list, "SubConverter-Request: 1");
+    header_list = curl_slist_append(header_list, "SubConverter-Request: 1");
-    list = curl_slist_append(list, "SubConverter-Version: " VERSION);
+    header_list = curl_slist_append(header_list, "SubConverter-Version: " VERSION);
-    if(list)
+    if(header_list)
-        curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list);
+        curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, header_list);
 
     if(result.content)
     {

src/utils/logger.cpp

@@ -13,10 +13,10 @@ std::string getTime(int type)
     time_t lt;
     char tmpbuf[32], cMillis[7];
     std::string format;
-    timeval tv;
+    timeval tv = {};
-    gettimeofday(&tv, NULL);
+    gettimeofday(&tv, nullptr);
     snprintf(cMillis, 7, "%.6ld", (long)tv.tv_usec);
-    lt = time(NULL);
+    lt = time(nullptr);
     struct tm *local = localtime(&lt);
     switch(type)
     {
@@ -27,19 +27,30 @@ std::string getTime(int type)
         format = "%Y/%m/%d %a %H:%M:%S." + std::string(cMillis);
         break;
     case 3:
+    default:
         format = "%Y-%m-%d %H:%M:%S";
         break;
     }
     strftime(tmpbuf, 32, format.data(), local);
-    return std::string(tmpbuf);
+    return {tmpbuf};
 }
 
+static std::string get_thread_name()
+{
+    static std::atomic_int counter = 0;
+    thread_local static std::string name = "Thread-" + std::to_string(counter++);
+    return name;
+}
+
+std::mutex log_mutex;
+
 void writeLog(int type, const std::string &content, int level)
 {
     if(level > global.logLevel)
         return;
+    std::lock_guard<std::mutex> lock(log_mutex);
     const char *levels[] = {"[FATL]", "[ERRO]", "[WARN]", "[INFO]", "[DEBG]", "[VERB]"};
-    std::cerr<<getTime(2)<<" ["<<getpid()<<" "<<std::this_thread::get_id()<<"]"<<levels[level % 6];
+    std::cerr<<getTime(2)<<" ["<<getpid()<<" "<<get_thread_name()<<"]"<<levels[level % 6];
     std::cerr<<" "<<content<<"\n";
 }
 
@@ -53,7 +64,7 @@ std::string demangle(const char* name)
 {
     int status = -4;
     std::unique_ptr<char, void(*)(void*)> res {
-        abi::__cxa_demangle(name, NULL, NULL, &status),
+        abi::__cxa_demangle(name, nullptr, nullptr, &status),
         std::free
     };
     return (status == 0) ? res.get() : name;

src/utils/string.h

@@ -10,6 +10,7 @@
 using string = std::string;
 using string_size = std::string::size_type;
 using string_array = std::vector<std::string>;
+using string_view_array = std::vector<std::string_view>;
 using string_map = std::map<std::string, std::string>;
 using string_multimap = std::multimap<std::string, std::string>;
 using string_pair_array = std::vector<std::pair<std::string, std::string>>;