Add cache system for configs, rulesets and subscriptions

Add proxy_config option for downloading external config files.
Optimize codes.

base/pref-new.yml

@@ -11,6 +11,7 @@ common:
   mellow_rule_base: base/mellow.conf
   quan_rule_base: base/quan.conf
   quanx_rule_base: base/quanx.conf
+  proxy_config: SYSTEM
   proxy_ruleset: SYSTEM
   proxy_subscription: NONE
   append_proxy_type: false
@@ -76,3 +77,7 @@ advanced:
   print_debug_info: false
   max_pending_connections: 10240
   max_concurrent_threads: 4
+  enable_cache: false
+  cache_subscription: 60
+  cache_config: 300
+  cache_ruleset: 21600

base/pref.ini

@@ -36,8 +36,9 @@ quan_rule_base=base/quan.conf
 ;Quantumult X config base used by the generator, supports local files/URL
 quanx_rule_base=base/quanx.conf
 
-;Proxy used to download rulesets or subscriptions, set to NONE or empty to disable it, set to SYSTEM to use system proxy.
+;Proxy used to download configs, rulesets or subscriptions, set to NONE or empty to disable it, set to SYSTEM to use system proxy.
 ;Accept cURL-supported proxies (http:// https:// socks4a:// socks5://)
+proxy_config=SYSTEM
 proxy_ruleset=SYSTEM
 proxy_subscription=NONE
 
@@ -154,3 +155,7 @@ port=25500
 print_debug_info=false
 max_pending_connections=10240
 max_concurrent_threads=4
+enable_cache=false
+cache_subscription=60
+cache_config=300
+cache_ruleset=21600

src/interfaces.cpp

@@ -35,7 +35,7 @@ std::mutex on_configuring;
 string_array renames, emojis;
 bool add_emoji = false, remove_old_emoji = false, append_proxy_type = false, filter_deprecated = true;
 bool udp_flag = false, tfo_flag = false, scv_flag = false, do_sort = false;
-std::string proxy_ruleset, proxy_subscription;
+std::string proxy_config, proxy_ruleset, proxy_subscription;
 
 std::string clash_rule_base;
 string_array clash_extra_group;
@@ -46,6 +46,9 @@ std::string surge_ssr_path;
 YAML::Node clash_base;
 INIReader surge_base, mellow_base;
 
+//cache system
+int cache_subscription = 60, cache_config = 300, cache_ruleset = 21600;
+
 string_array regex_blacklist = {"(.*)*"};
 
 template <typename T> void operator >> (const YAML::Node& node, T& i)
@@ -57,7 +60,7 @@ template <typename T> void operator >> (const YAML::Node& node, T& i)
 std::string getRuleset(RESPONSE_CALLBACK_ARGS)
 {
     std::string url = urlsafe_base64_decode(getUrlArg(argument, "url")), type = getUrlArg(argument, "type"), group = urlsafe_base64_decode(getUrlArg(argument, "group"));
-    std::string output_content;
+    std::string output_content, dummy;
 
     if(!url.size() || !type.size() || (type == "2" && !group.size()) || (type != "1" && type != "2"))
     {
@@ -65,10 +68,18 @@ std::string getRuleset(RESPONSE_CALLBACK_ARGS)
         return "Invalid request!";
     }
 
+    std::string proxy;
+    if(proxy_ruleset == "SYSTEM")
+        proxy = getSystemProxy();
+    else if(proxy_ruleset == "NONE")
+        proxy = "";
+    else
+        proxy = proxy_ruleset;
+
     if(fileExist(url))
         output_content = fileGet(url, false, true);
     else
-        output_content = webGet(url, "");
+        output_content = webGet(url, proxy, dummy, cache_ruleset);
 
     if(!output_content.size())
     {
@@ -121,7 +132,7 @@ int importItems(string_array &target)
 {
     string_array result;
     std::stringstream ss;
-    std::string path, content, strLine;
+    std::string path, content, strLine, dummy;
     unsigned int itemCount = 0;
     for(std::string &x : target)
     {
@@ -133,10 +144,18 @@ int importItems(string_array &target)
         path = x.substr(x.find(":") + 1);
         writeLog(0, "Trying to import items from " + path);
 
+        std::string proxy;
+        if(proxy_ruleset == "SYSTEM")
+            proxy = getSystemProxy();
+        else if(proxy_ruleset == "NONE")
+            proxy = "";
+        else
+            proxy = proxy_ruleset;
+
         if(fileExist(path))
             content = fileGet(path, false, api_mode);
         else
-            content = webGet(path, "");
+            content = webGet(path, proxy, dummy, cache_config);
         if(!content.size())
             return -1;
 
@@ -293,7 +312,7 @@ void readRuleset(YAML::Node node, string_array &dest)
 void refreshRulesets(string_array &ruleset_list, std::vector<ruleset_content> &rca)
 {
     eraseElements(rca);
-    std::string rule_group, rule_url;
+    std::string rule_group, rule_url, dummy;
     ruleset_content rc;
 
     std::string proxy;
@@ -333,7 +352,7 @@ void refreshRulesets(string_array &ruleset_list, std::vector<ruleset_content> &r
             }
             else if(startsWith(rule_url, "http://") || startsWith(rule_url, "https://") || startsWith(rule_url, "data:"))
             {
-                rc = {rule_group, rule_url, webGet(rule_url, proxy)};
+                rc = {rule_group, rule_url, webGet(rule_url, proxy, dummy, cache_ruleset)};
             }
             else
                 continue;
@@ -385,6 +404,7 @@ void readYAMLConf(YAML::Node &node)
     section["quanx_rule_base"] >> quanx_rule_base;
 
     section["append_proxy_type"] >> append_proxy_type;
+    section["proxy_config"] >> proxy_config;
     section["proxy_ruleset"] >> proxy_ruleset;
     section["proxy_subscription"] >> proxy_subscription;
 
@@ -478,6 +498,17 @@ void readYAMLConf(YAML::Node &node)
         node["advanced"]["print_debug_info"] >> print_debug_info;
         node["advanced"]["max_pending_connections"] >> max_pending_connections;
         node["advanced"]["max_concurrent_threads"] >> max_concurrent_threads;
+        if(node["advanced"]["enable_cache"].IsDefined())
+        {
+            if(node["advanced"]["enable_cache"].as<bool>())
+            {
+                node["advanced"]["cache_subscription"] >> cache_subscription;
+                node["advanced"]["cache_config"] >> cache_config;
+                node["advanced"]["cache_ruleset"] >> cache_ruleset;
+            }
+            else
+                cache_subscription = cache_config = cache_ruleset = 0; //disable cache
+        }
     }
 }
 
@@ -541,6 +572,8 @@ void readConf()
         quanx_rule_base = ini.Get("quanx_rule_base");
     if(ini.ItemExist("append_proxy_type"))
         append_proxy_type = ini.GetBool("append_proxy_type");
+    if(ini.ItemExist("proxy_config"))
+        proxy_config = ini.Get("proxy_config");
     if(ini.ItemExist("proxy_ruleset"))
         proxy_ruleset = ini.Get("proxy_ruleset");
     if(ini.ItemExist("proxy_subscription"))
@@ -655,6 +688,20 @@ void readConf()
         max_pending_connections = ini.GetInt("max_pending_connections");
     if(ini.ItemExist("max_concurrent_threads"))
         max_concurrent_threads = ini.GetInt("max_concurrent_threads");
+    if(ini.ItemExist("enable_cache"))
+    {
+        if(ini.GetBool("enable_cache"))
+        {
+            if(ini.ItemExist("cache_subscription"))
+                cache_subscription = ini.GetInt("cache_subscription");
+            if(ini.ItemExist("cache_config"))
+                cache_config = ini.GetInt("cache_config");
+            if(ini.ItemExist("cache_ruleset"))
+                cache_ruleset = ini.GetInt("cache_ruleset");
+        }
+        else
+            cache_subscription = cache_config = cache_ruleset = 0; //disable cache
+    }
 
     std::cerr<<"Read preference settings completed."<<std::endl;
 }
@@ -708,13 +755,21 @@ int loadExternalYAML(YAML::Node &node, ExternalConfig &ext)
     return 0;
 }
 
-int loadExternalConfig(std::string &path, ExternalConfig &ext, std::string proxy)
+int loadExternalConfig(std::string &path, ExternalConfig &ext)
 {
-    std::string base_content;
+    std::string base_content, dummy;
+    std::string proxy;
+    if(proxy_config == "SYSTEM")
+        proxy = getSystemProxy();
+    else if(proxy_config == "NONE")
+        proxy = "";
+    else
+        proxy = proxy_config;
+
     if(fileExist(path))
         base_content = fileGet(path, false, api_mode);
     else
-        base_content = webGet(path, proxy);
+        base_content = webGet(path, proxy, dummy, cache_config);
 
     try
     {
@@ -828,7 +883,7 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS)
     std::string groups = urlsafe_base64_decode(getUrlArg(argument, "groups")), ruleset = urlsafe_base64_decode(getUrlArg(argument, "ruleset")), config = UrlDecode(getUrlArg(argument, "config"));
     std::vector<ruleset_content> rca;
     extra_settings ext;
-    std::string subInfo;
+    std::string subInfo, dummy;
     bool ruleset_updated = false, authorized = getUrlArg(argument, "token") == access_token;
 
     if(std::find(regex_blacklist.cbegin(), regex_blacklist.cend(), include) != regex_blacklist.cend() || std::find(regex_blacklist.cbegin(), regex_blacklist.cend(), exclude) != regex_blacklist.cend())
@@ -902,7 +957,7 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS)
         extra_ruleset = rulesets;
         //then load external configuration
         ExternalConfig extconf;
-        loadExternalConfig(config, extconf, proxy);
+        loadExternalConfig(config, extconf);
         if(extconf.clash_rule_base.size())
             ext_clash_base = extconf.clash_rule_base;
         if(extconf.surge_rule_base.size())
@@ -1045,7 +1100,7 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS)
             if(fileExist(ext_clash_base))
                 base_content = fileGet(ext_clash_base, false);
             else
-                base_content = webGet(ext_clash_base, getSystemProxy());
+                base_content = webGet(ext_clash_base, getSystemProxy(), dummy, cache_config);
 
             output_content = netchToClash(nodes, base_content, rca, extra_group, target == "clashr", ext);
         }
@@ -1073,7 +1128,7 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS)
             if(fileExist(ext_surge_base))
                 base_content = fileGet(ext_surge_base, false);
             else
-                base_content = webGet(ext_surge_base, getSystemProxy());
+                base_content = webGet(ext_surge_base, getSystemProxy(), dummy, cache_config);
 
             output_content = netchToSurge(nodes, base_content, rca, extra_group, surge_ver, ext);
             if(upload == "true")
@@ -1089,7 +1144,7 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS)
         if(fileExist(ext_surfboard_base))
             base_content = fileGet(ext_surfboard_base, false);
         else
-            base_content = webGet(ext_surfboard_base, getSystemProxy());
+            base_content = webGet(ext_surfboard_base, getSystemProxy(), dummy, cache_config);
 
         output_content = netchToSurge(nodes, base_content, rca, extra_group, -3, ext);
         if(upload == "true")
@@ -1106,7 +1161,7 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS)
             if(fileExist(ext_mellow_base))
                 base_content = fileGet(ext_mellow_base, false);
             else
-                base_content = webGet(ext_mellow_base, getSystemProxy());
+                base_content = webGet(ext_mellow_base, getSystemProxy(), dummy, cache_config);
 
             output_content = netchToMellow(nodes, base_content, rca, extra_group, ext);
         }
@@ -1158,7 +1213,7 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS)
             if(fileExist(ext_quan_base))
                 base_content = fileGet(ext_quan_base, false);
             else
-                base_content = webGet(ext_quan_base, getSystemProxy());
+                base_content = webGet(ext_quan_base, getSystemProxy(), dummy, cache_config);
         }
 
         output_content = netchToQuan(nodes, base_content, rca, extra_group, ext);
@@ -1174,7 +1229,7 @@ std::string subconverter(RESPONSE_CALLBACK_ARGS)
             if(fileExist(ext_quanx_base))
                 base_content = fileGet(ext_quanx_base, false);
             else
-                base_content = webGet(ext_quanx_base, getSystemProxy());
+                base_content = webGet(ext_quanx_base, getSystemProxy(), dummy, cache_config);
         }
 
         output_content = netchToQuanX(nodes, base_content, rca, extra_group, ext);
@@ -1276,7 +1331,7 @@ std::string surgeConfToClash(RESPONSE_CALLBACK_ARGS)
     YAML::Node clash = safe_get_clash_base();
     string_array dummy_str_array;
     std::vector<nodeInfo> nodes;
-    std::string base_content, url = argument.size() <= 5 ? "" : argument.substr(5);
+    std::string base_content, url = argument.size() <= 5 ? "" : argument.substr(5), dummy;
     ini.store_any_line = true;
 
     if(!url.size())
@@ -1292,10 +1347,18 @@ std::string surgeConfToClash(RESPONSE_CALLBACK_ARGS)
         return "Please insert your subscription link instead of clicking the default link.";
     }
 
+    std::string proxy;
+    if(proxy_config == "SYSTEM")
+        proxy = getSystemProxy();
+    else if(proxy_config == "NONE")
+        proxy = "";
+    else
+        proxy = proxy_config;
+
     if(fileExist(url))
         base_content = fileGet(url, false);
     else
-        base_content = webGet(url, getSystemProxy());
+        base_content = webGet(url, proxy, dummy, cache_config);
 
     if(ini.Parse(base_content) != INIREADER_EXCEPTION_NONE)
     {
@@ -1312,7 +1375,6 @@ std::string surgeConfToClash(RESPONSE_CALLBACK_ARGS)
         return errmsg;
     }
 
-    std::string proxy;
     if(proxy_subscription == "SYSTEM")
         proxy = getSystemProxy();
     else if(proxy_subscription == "NONE")
@@ -1320,7 +1382,6 @@ std::string surgeConfToClash(RESPONSE_CALLBACK_ARGS)
     else
         proxy = proxy_subscription;
 
-
     std::string subInfo;
     std::cerr<<"Fetching node data from url '"<<url<<"'."<<std::endl;
     if(addNodes(url, nodes, 0, proxy, dummy_str_array, dummy_str_array, dummy_str_array, dummy_str_array, subInfo, false) == -1)
@@ -1412,7 +1473,7 @@ std::string surgeConfToClash(RESPONSE_CALLBACK_ARGS)
             strArray = split(x, ",");
             if(strArray.size() != 3)
                 continue;
-            content = webGet(strArray[1], "");
+            content = webGet(strArray[1], proxy, dummy, cache_ruleset);
             if(!content.size())
                 continue;
 

src/misc.cpp

@@ -756,7 +756,8 @@ std::string fileGetMD5(const std::string &filepath)
 int fileWrite(const std::string &path, const std::string &content, bool overwrite)
 {
     std::fstream outfile;
-    std::ios::openmode mode = overwrite ? std::ios::out : std::ios::app;
+    std::ios_base::openmode mode = overwrite ? std::ios_base::out : std::ios_base::app;
+    mode |= std::ios_base::binary;
     outfile.open(path, mode);
     outfile << content << std::endl;
     outfile.close();

src/nodemanip.cpp

@@ -12,6 +12,7 @@
 std::string override_conf_port;
 int socksport;
 bool ss_libev, ssr_libev;
+extern int cache_subscription;
 
 void copyNodes(std::vector<nodeInfo> &source, std::vector<nodeInfo> &dest)
 {
@@ -48,7 +49,7 @@ int addNodes(std::string link, std::vector<nodeInfo> &allNodes, int groupID, std
         writeLog(LOG_TYPE_INFO, "Downloading subscription data...");
         if(strFind(link, "surge:///install-config")) //surge config link
             link = UrlDecode(getUrlArg(link, "url"));
-        strSub = webGet(link, proxy, extra_headers);
+        strSub = webGet(link, proxy, extra_headers, cache_subscription);
         /*
         if(strSub.size() == 0)
         {

src/subexport.cpp

@@ -970,7 +970,7 @@ std::string netchToSurge(std::vector<nodeInfo> &nodes, std::string &base_conf, s
             if(ext.skip_cert_verify)
                 proxy += ", skip-cert-verify=1";
         }
-        else if(type == "HTTP" || type == "HTTPS")
+        else if(x.linkType == SPEEDTEST_MESSAGE_FOUNDHTTP)
         {
             proxy = "http," + hostname + "," + port + ", " + username + ", " + password;
             proxy += std::string(", tls=") + (type == "HTTPS" ? "true" : "false");
@@ -1982,7 +1982,7 @@ void netchToMellow(std::vector<nodeInfo> &nodes, INIReader &ini, std::vector<rul
         {
             proxy = remark + ", builtin, socks, address=" + hostname + ", port=" + port + ", user=" + username + ", pass=" + password;
         }
-        else if(type == "HTTP")
+        else if(x.linkType == SPEEDTEST_MESSAGE_FOUNDHTTP)
         {
             proxy = remark + ", builtin, http, address=" + hostname + ", port=" + port + ", user=" + username + ", pass=" + password;
         }

src/webget.cpp

@@ -1,10 +1,13 @@
 #include <iostream>
 #include <unistd.h>
+#include <sys/stat.h>
 
 #include <curl/curl.h>
 
 #include "webget.h"
 #include "version.h"
+#include "misc.h"
+#include "logger.h"
 
 extern bool print_debug_info;
 
@@ -21,10 +24,9 @@ static int writer(char *data, size_t size, size_t nmemb, std::string *writerData
     return size * nmemb;
 }
 
-static std::string curlGet(std::string url, std::string proxy, std::string &response_headers)
+static std::string curlGet(std::string url, std::string proxy, std::string &response_headers, CURLcode &return_code)
 {
     CURL *curl_handle;
-    CURLcode res;
     std::string data;
 
     curl_global_init(CURL_GLOBAL_ALL);
@@ -45,10 +47,10 @@ static std::string curlGet(std::string url, std::string proxy, std::string &resp
     if(proxy != "")
         curl_easy_setopt(curl_handle, CURLOPT_PROXY, proxy.data());
 
-    res = curl_easy_perform(curl_handle);
+    return_code = curl_easy_perform(curl_handle);
     curl_easy_cleanup(curl_handle);
 
-    if(res != CURLE_OK)
+    if(return_code != CURLE_OK)
         data.clear();
 
     return data;
@@ -78,11 +80,43 @@ std::string buildSocks5ProxyString(std::string addr, int port, std::string usern
     return proxystr;
 }
 
-std::string webGet(std::string url, std::string proxy, std::string &response_headers)
+std::string webGet(std::string url, std::string proxy, std::string &response_headers, unsigned int cache_ttl)
 {
+    std::string content;
+    CURLcode return_code;
     if (startsWith(url, "data:"))
         return dataGet(url);
-    return curlGet(url, proxy, response_headers);
+    // cache system
+    if(cache_ttl > 0)
+    {
+        mkdir("cache");
+        std::string url_md5 = getMD5(url);
+        std::string path = "cache/" + url_md5;
+        struct _stat result;
+        if(_stat(path.data(), &result) == 0)
+        {
+            time_t mtime = result.st_mtime, now = time(NULL);
+            if(difftime(now, mtime) <= cache_ttl)
+            {
+                writeLog(0, "CACHE HIT: '" + url + "', using local cache.");
+                response_headers = fileGet(path + "_header", true, true);
+                return fileGet(path, true, true);
+            }
+            writeLog(0, "CACHE MISS: '" + url + "', TTL timeout, creating new cache.");
+        }
+        else
+            writeLog(0, "CACHE NOT EXIST: '" + url + "', creating new cache.");
+        content = curlGet(url, proxy, response_headers, return_code);
+        if(return_code == CURLE_OK)
+        {
+            fileWrite(path, content, true);
+            fileWrite(path + "_header", response_headers, true);
+        }
+        else
+            writeLog(0, "Fetched failed. Skip creating new cache.");
+        return content;
+    }
+    return curlGet(url, proxy, response_headers, return_code);
 }
 
 std::string webGet(std::string url, std::string proxy)

src/webget.h

@@ -10,7 +10,7 @@
 #include "misc.h"
 
 std::string webGet(std::string url, std::string proxy);
-std::string webGet(std::string url, std::string proxy, std::string &response_headers);
+std::string webGet(std::string url, std::string proxy, std::string &response_headers, unsigned int cache_ttl = 0);
 int curlPost(std::string url, std::string data, std::string proxy, std::string auth_token, std::string *retData);
 int curlPatch(std::string url, std::string data, std::string proxy, std::string auth_token, std::string *retData);
 std::string buildSocks5ProxyString(std::string addr, int port, std::string username, std::string password);