Enhancements

Fix ruleset types other than Surge Ruleset not correctly expanded to rules.
Fix trying to limit upload file size.
Fix when no GEOIP rule is added, the Clash Script will not be able to generate.
Add "mixed" target to generate subscriptions with all types of standard subscriptions.
Add passing X-User-Agent and X-Client-IP header to subscription downloader.
Optimize codes.

src/webget.cpp

@@ -45,6 +45,14 @@ static int writer(char *data, size_t size, size_t nmemb, std::string *writerData
     return size * nmemb;
 }
 
+static int dummy_writer(char *data, size_t size, size_t nmemb, void *writerData)
+{
+    /// dummy writer, do not save anything
+    (void)data;
+    (void)writerData;
+    return size * nmemb;
+}
+
 static int size_checker(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
 {
     if(dltotal > 1048576.0)
@@ -52,7 +60,7 @@ static int size_checker(void *clientp, double dltotal, double dlnow, double ulto
     return 0;
 }
 
-static inline void curl_set_common_options(CURL *curl_handle, const char *url)
+static inline void curl_set_common_options(CURL *curl_handle, const char *url, long max_file_size = 1048576L)
 {
     curl_easy_setopt(curl_handle, CURLOPT_URL, url);
     curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, global_log_level == LOG_LEVEL_VERBOSE ? 1L : 0L);
@@ -64,42 +72,63 @@ static inline void curl_set_common_options(CURL *curl_handle, const char *url)
     curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 0L);
     curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, 15L);
     curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, user_agent_str.data());
-    curl_easy_setopt(curl_handle, CURLOPT_MAXFILESIZE, 1048576L);
+    if(max_file_size)
+        curl_easy_setopt(curl_handle, CURLOPT_MAXFILESIZE, max_file_size);
     curl_easy_setopt(curl_handle, CURLOPT_PROGRESSFUNCTION, size_checker);
 }
 
-static std::string curlGet(const std::string &url, const std::string &proxy, std::string &response_headers, CURLcode &return_code)
+//static std::string curlGet(const std::string &url, const std::string &proxy, std::string &response_headers, CURLcode &return_code, const string_map &request_headers)
+static int curlGet(const FetchArgument argument, FetchResult &result)
 {
     CURL *curl_handle;
-    std::string data, new_url = url;
+    std::string *data = result.content, new_url = argument.url;
     struct curl_slist *list = NULL;
+    defer(curl_slist_free_all(list);)
     long retVal = 0;
 
     curl_init();
 
     curl_handle = curl_easy_init();
-    if(proxy.size())
+    if(argument.proxy.size())
     {
-        if(startsWith(proxy, "cors:"))
+        if(startsWith(argument.proxy, "cors:"))
         {
             list = curl_slist_append(list, "X-Requested-With: subconverter " VERSION);
-            curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list);
-            new_url = proxy.substr(5) + url;
+            new_url = argument.proxy.substr(5) + argument.url;
         }
         else
-            curl_easy_setopt(curl_handle, CURLOPT_PROXY, proxy.data());
+            curl_easy_setopt(curl_handle, CURLOPT_PROXY, argument.proxy.data());
     }
     curl_set_common_options(curl_handle, new_url.data());
-    curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, writer);
-    curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, &data);
-    curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, writer);
-    curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, &response_headers);
+
+    if(argument.request_headers)
+    {
+        for(auto &x : *argument.request_headers)
+            list = curl_slist_append(list, (x.first + ": " + x.second).data());
+    }
+    if(list)
+        curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list);
+
+    if(result.content)
+    {
+        curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, writer);
+        curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, result.content);
+    }
+    else
+        curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, dummy_writer);
+    if(result.response_headers)
+    {
+        curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, writer);
+        curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, result.response_headers);
+    }
+    else
+        curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, dummy_writer);
 
     unsigned int fail_count = 0, max_fails = 1;
     while(true)
     {
-        return_code = curl_easy_perform(curl_handle);
-        if(return_code == CURLE_OK || max_fails >= fail_count)
+        *result.status_code = curl_easy_perform(curl_handle);
+        if(*result.status_code == CURLE_OK || max_fails >= fail_count)
             break;
         else
             fail_count++;
@@ -108,11 +137,14 @@ static std::string curlGet(const std::string &url, const std::string &proxy, std
     curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &retVal);
     curl_easy_cleanup(curl_handle);
 
-    if(return_code != CURLE_OK || retVal != 200)
-        data.clear();
-    data.shrink_to_fit();
+    if(data)
+    {
+        if(*result.status_code != CURLE_OK || retVal != 200)
+            data->clear();
+        data->shrink_to_fit();
+    }
 
-    return data;
+    return *result.status_code;
 }
 
 // data:[<mediatype>][;base64],<data>
@@ -139,10 +171,14 @@ std::string buildSocks5ProxyString(const std::string &addr, int port, const std:
     return proxystr;
 }
 
-std::string webGet(const std::string &url, const std::string &proxy, std::string &response_headers, unsigned int cache_ttl)
+std::string webGet(const std::string &url, const std::string &proxy, unsigned int cache_ttl, std::string *response_headers, string_map *request_headers)
 {
+    int return_code = 0;
     std::string content;
-    CURLcode return_code;
+
+    FetchArgument argument {url, proxy, request_headers, cache_ttl};
+    FetchResult fetch_res {&return_code, &content, response_headers};
+
     if (startsWith(url, "data:"))
         return dataGet(url);
     // cache system
@@ -159,19 +195,22 @@ std::string webGet(const std::string &url, const std::string &proxy, std::string
             {
                 writeLog(0, "CACHE HIT: '" + url + "', using local cache.");
                 guarded_mutex guard(cache_rw_lock);
-                response_headers = fileGet(path_header, true);
+                if(response_headers)
+                    *response_headers = fileGet(path_header, true);
                 return fileGet(path, true);
             }
             writeLog(0, "CACHE MISS: '" + url + "', TTL timeout, creating new cache."); // out of TTL
         }
         else
             writeLog(0, "CACHE NOT EXIST: '" + url + "', creating new cache.");
-        content = curlGet(url, proxy, response_headers, return_code); // try to fetch data
+        //content = curlGet(url, proxy, response_headers, return_code); // try to fetch data
+        curlGet(argument, fetch_res);
         if(return_code == CURLE_OK) // success, save new cache
         {
             guarded_mutex guard(cache_rw_lock);
             fileWrite(path, content, true);
-            fileWrite(path_header, response_headers, true);
+            if(response_headers)
+                fileWrite(path_header, *response_headers, true);
         }
         else
         {
@@ -180,26 +219,17 @@ std::string webGet(const std::string &url, const std::string &proxy, std::string
                 writeLog(0, "Fetch failed. Serving cached content."); // cache exist, serving cache
                 guarded_mutex guard(cache_rw_lock);
                 content = fileGet(path, true);
-                response_headers = fileGet(path_header, true);
+                if(response_headers)
+                    *response_headers = fileGet(path_header, true);
             }
             else
                 writeLog(0, "Fetch failed. No local cache available."); // cache not exist or not allow to serve cache, serving nothing
         }
         return content;
     }
-    return curlGet(url, proxy, response_headers, return_code);
-}
-
-std::string webGet(const std::string &url, const std::string &proxy)
-{
-    std::string dummy;
-    return webGet(url, proxy, dummy);
-}
-
-std::string webGet(const std::string &url, const std::string &proxy, unsigned int cache_ttl)
-{
-    std::string dummy;
-    return webGet(url, proxy, dummy, cache_ttl);
+    //return curlGet(url, proxy, response_headers, return_code);
+    content = curlGet(argument, fetch_res);
+    return content;
 }
 
 int curlPost(const std::string &url, const std::string &data, const std::string &proxy, const string_array &request_headers, std::string *retData)
@@ -215,7 +245,7 @@ int curlPost(const std::string &url, const std::string &data, const std::string
     for(const std::string &x : request_headers)
         list = curl_slist_append(list, x.data());
 
-    curl_set_common_options(curl_handle, url.data());
+    curl_set_common_options(curl_handle, url.data(), 0L);
     curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
     curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, data.data());
     curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE, data.size());
@@ -259,7 +289,7 @@ int curlPatch(const std::string &url, const std::string &data, const std::string
     for(const std::string &x : request_headers)
         list = curl_slist_append(list, x.data());
 
-    curl_set_common_options(curl_handle, url.data());
+    curl_set_common_options(curl_handle, url.data(), 0L);
     curl_easy_setopt(curl_handle, CURLOPT_CUSTOMREQUEST, "PATCH");
     curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, data.data());
     curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE, data.size());