Enhancements

Fix trying to fetch a ruleset when adding it as a RULE-SET rule to configuration files.
Add URL Decode to username and password field in Telegram-link-style HTTP/Socks 5 links.
Add more function callbacks to template renderer.
Add basic Jinja2Cpp template library support.
INIReader: Add keeping comments and blank lines in direct-save sections.

src/ini_reader.h

@@ -240,7 +240,7 @@ public:
                 strLine.erase(lineSize - 1);
                 lineSize--;
             }
-            if(!lineSize || strLine[0] == ';' || strLine[0] == '#' || (lineSize >= 2 && strLine[0] == '/' && strLine[1] == '/')) //empty lines and comments are ignored
+            if((!lineSize || strLine[0] == ';' || strLine[0] == '#' || (lineSize >= 2 && strLine[0] == '/' && strLine[1] == '/')) && !inDirectSaveSection) //empty lines and comments are ignored
                 continue;
             if(strLine[0] == '[' && strLine[lineSize - 1] == ']') //is a section title
             {

src/interfaces.cpp

@@ -3,6 +3,7 @@
 #include <mutex>
 #include <numeric>
 
+#include <inja.hpp>
 #include <yaml-cpp/yaml.h>
 
 #include "misc.h"
@@ -342,10 +343,11 @@ void refreshRulesets(string_array &ruleset_list, std::vector<ruleset_content> &r
         rule_group = trim(vArray[0]);
         rule_url = trim(vArray[1]);
         */
-        if(x.find(",") == x.npos)
+        string_size pos = x.find(",");
+        if(pos == x.npos)
             continue;
-        rule_group = trim(x.substr(0, x.find(",")));
-        rule_url = trim(x.substr(x.find(",") + 1));
+        rule_group = trim(x.substr(0, pos));
+        rule_url = trim(x.substr(pos + 1));
         if(rule_url.find("[]") == 0)
         {
             writeLog(0, "Adding rule '" + rule_url.substr(2) + "," + rule_group + "'.", LOG_LEVEL_INFO);
@@ -1880,6 +1882,42 @@ std::string getRewriteRemote(RESPONSE_CALLBACK_ARGS)
     return output_content;
 }
 
+std::string parseHostname(inja::Arguments &args)
+{
+    std::string data = args.at(0)->get<std::string>();
+    string_array urls = split(data, ",");
+    if(!urls.size())
+        return std::string();
+
+    std::string input_content, output_content, proxy = parseProxy(proxy_config);
+    for(std::string &x : urls)
+    {
+        input_content = webGet(x, proxy, cache_config);
+        output_content += regReplace(input_content, "(?:[\\s\\S]*?)^(?i:hostname\\s*?=\\s*?)(.*?)\\s$(?:[\\s\\S]*)", "$1") + ",";
+    }
+    string_array vArray = split(output_content, ",");
+    std::set<std::string> hostnames;
+    for(std::string &x : vArray)
+        hostnames.emplace(trim(x));
+    output_content = std::accumulate(hostnames.begin(), hostnames.end(), std::string(), [](std::string a, std::string b)
+    {
+        return std::move(a) + "," + std::move(b);
+    });
+    return output_content;
+}
+
+std::string template_webGet(inja::Arguments &args)
+{
+    std::string data = args.at(0)->get<std::string>(), proxy = parseProxy(proxy_config);
+    return webGet(data, proxy, cache_config);
+}
+
+std::string jinja2_webGet(const std::string &url)
+{
+    std::string proxy = parseProxy(proxy_config);
+    return webGet(url, proxy, cache_config);
+}
+
 static inline std::string intToStream(unsigned long long stream)
 {
     char chrs[16] = {}, units[6] = {' ', 'K', 'M', 'G', 'T', 'P'};

src/interfaces.h

@@ -3,6 +3,7 @@
 
 #include <string>
 #include <map>
+#include <inja.hpp>
 
 #include "subexport.h"
 #include "webserver.h"
@@ -21,4 +22,8 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS);
 std::string simpleToClashR(RESPONSE_CALLBACK_ARGS);
 std::string surgeConfToClash(RESPONSE_CALLBACK_ARGS);
 
+std::string template_webGet(inja::Arguments &args);
+std::string jinja2_webGet(const std::string &url);
+std::string parseHostname(inja::Arguments &args);
+
 #endif // INTERFACES_H_INCLUDED

src/speedtestutil.cpp

@@ -634,8 +634,8 @@ void explodeSocks(std::string link, const std::string &custom_port, nodeInfo &no
     {
         server = getUrlArg(link, "server");
         port = getUrlArg(link, "port");
-        username = getUrlArg(link, "user");
-        password = getUrlArg(link, "pass");
+        username = UrlDecode(getUrlArg(link, "user"));
+        password = UrlDecode(getUrlArg(link, "pass"));
         remarks = UrlDecode(getUrlArg(link, "remarks"));
         group = UrlDecode(getUrlArg(link, "group"));
     }
@@ -659,8 +659,8 @@ void explodeHTTP(std::string link, const std::string &custom_port, nodeInfo &nod
     std::string group, remarks, server, port, username, password;
     server = getUrlArg(link, "server");
     port = getUrlArg(link, "port");
-    username = getUrlArg(link, "user");
-    password = getUrlArg(link, "pass");
+    username = UrlDecode(getUrlArg(link, "user"));
+    password = UrlDecode(getUrlArg(link, "pass"));
     remarks = UrlDecode(getUrlArg(link, "remarks"));
     group = UrlDecode(getUrlArg(link, "group"));
 

src/subexport.cpp

@@ -565,15 +565,10 @@ void rulesetToSurge(INIReader &base_rule, std::vector<ruleset_content> &ruleset_
     for(ruleset_content &x : ruleset_content_array)
     {
         rule_group = x.rule_group;
-        retrived_rules = x.rule_content.get();
-        if(retrived_rules.empty())
+        rule_path = x.rule_path;
+        if(rule_path.empty())
         {
-            writeLog(0, "Failed to fetch ruleset or ruleset is empty: '" + x.rule_path + "'!", LOG_LEVEL_WARNING);
-            continue;
-        }
-        if(retrived_rules.find("[]") == 0)
-        {
-            strLine = retrived_rules.substr(2);
+            strLine = x.rule_content.get().substr(2);
             if(strLine == "MATCH")
                 strLine = "FINAL";
             strLine += "," + rule_group;
@@ -595,7 +590,6 @@ void rulesetToSurge(INIReader &base_rule, std::vector<ruleset_content> &ruleset_
         }
         else
         {
-            rule_path = x.rule_path;
             if(!fileExist(rule_path))
             {
                 if(surge_ver > 2)
@@ -640,6 +634,12 @@ void rulesetToSurge(INIReader &base_rule, std::vector<ruleset_content> &ruleset_
                     continue;
                 }
             }
+            retrived_rules = x.rule_content.get();
+            if(retrived_rules.empty())
+            {
+                writeLog(0, "Failed to fetch ruleset or ruleset is empty: '" + x.rule_path + "'!", LOG_LEVEL_WARNING);
+                continue;
+            }
 
             char delimiter = count(retrived_rules.begin(), retrived_rules.end(), '\n') < 1 ? '\r' : '\n';
 
@@ -1081,6 +1081,8 @@ std::string netchToSurge(std::vector<nodeInfo> &nodes, std::string &base_conf, s
 
     ini.store_any_line = true;
     // filter out sections that requires direct-save
+    ini.AddDirectSaveSection("General");
+    ini.AddDirectSaveSection("Replica");
     ini.AddDirectSaveSection("Rule");
     ini.AddDirectSaveSection("MITM");
     ini.AddDirectSaveSection("Script");

src/template_jinja2.cpp

@@ -0,0 +1,47 @@
+#include <string>
+
+#include <jinja2cpp/user_callable.h>
+#include <jinja2cpp/binding/nlohmann_json.h>
+#include <jinja2cpp/template.h>
+#include <nlohmann/json.hpp>
+
+#include "interfaces.h"
+#include "templates.h"
+#include "webget.h"
+#include "misc.h"
+
+static inline void parse_json_pointer(nlohmann::json &json, const std::string &path, const std::string &value)
+{
+    std::string pointer = "/" + replace_all_distinct(path, ".", "/");
+    json[nlohmann::json::json_pointer(pointer)] = value;
+}
+
+int render_template(const std::string &content, const template_args &vars, std::string &output, const std::string &include_scope)
+{
+    jinja2::Template tpl;
+    nlohmann::json data;
+    for(auto &x : vars.global_vars)
+        parse_json_pointer(data["global"], x.first, x.second);
+    for(auto &x : vars.request_params)
+        parse_json_pointer(data["request"], x.first, x.second);
+    for(auto &x : vars.local_vars)
+        parse_json_pointer(data["local"], x.first, x.second);
+    tpl.Load(content);
+    jinja2::ValuesMap valmap = {{"global", jinja2::Reflect(data["global"])}, {"local", jinja2::Reflect(data["local"])}, {"request", jinja2::Reflect(data["request"])}};
+    valmap["fetch"] = jinja2::MakeCallable(jinja2_webGet, jinja2::ArgInfo{"url"});
+    valmap["replace"] = jinja2::MakeCallable([](const std::string &src, const std::string &target, const std::string &rep)
+    {
+        return regReplace(src, target, rep);
+    }, jinja2::ArgInfo{"src"}, jinja2::ArgInfo{"target"}, jinja2::ArgInfo{"rep"});
+    try
+    {
+        output = tpl.RenderAsString(valmap).value();
+        return 0;
+    }
+    catch (std::exception &e)
+    {
+        output = "Template render failed! Reason: " + std::string(e.what());
+        return -1;
+    }
+    return -2;
+}

src/templates.cpp

@@ -4,6 +4,7 @@
 #include <inja.hpp>
 #include <nlohmann/json.hpp>
 
+#include "interfaces.h"
 #include "templates.h"
 #include "logger.h"
 #include "misc.h"
@@ -49,6 +50,31 @@ int render_template(const std::string &content, const template_args &vars, std::
         std::string data = args.at(0)->get<std::string>();
         return trim(data);
     });
+    m_callbacks.add_callback("replace", 3, [](inja::Arguments &args)
+    {
+        std::string src = args.at(0)->get<std::string>(), target = args.at(1)->get<std::string>(), rep = args.at(2)->get<std::string>();
+        if(target.empty() || src.empty())
+            return src;
+        return regReplace(src, target, rep);
+    });
+    m_callbacks.add_callback("set", 2, [&data](inja::Arguments &args)
+    {
+        std::string key = args.at(0)->get<std::string>(), value = args.at(1)->get<std::string>();
+        parse_json_pointer(data, key, value);
+        return std::string();
+    });
+    m_callbacks.add_callback("join", 2, [](inja::Arguments &args)
+    {
+        std::string str1 = args.at(0)->get<std::string>(), str2 = args.at(1)->get<std::string>();
+        return std::move(str1) + std::move(str2);
+    });
+    m_callbacks.add_callback("join", 3, [](inja::Arguments &args)
+    {
+        std::string str1 = args.at(0)->get<std::string>(), str2 = args.at(1)->get<std::string>(), delim = args.at(2)->get<std::string>();
+        return std::move(str1) + std::move(delim) + std::move(str2);
+    });
+    m_callbacks.add_callback("fetch", 1, template_webGet);
+    m_callbacks.add_callback("parseHostname", 1, parseHostname);
     m_parser_config.include_scope_limit = true;
     m_parser_config.include_scope = include_scope;