Enhancements

Fix broken SSID group exported to Quantumult X configurations. (#167) Fix compatibility with some V2Ray-Core configurations. Add support for using CORS proxy in downloading. Optimize codes.
2025-10-27 20:03:01 +00:00 · 2020-05-15 22:47:31 +08:00
parent 2628c1beb3
commit 54747a3268
9 changed files with 183 additions and 228 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -45,7 +45,6 @@ ADD_EXECUTABLE(subconverter
    src/misc.cpp
    src/multithread.cpp
    src/nodemanip.cpp
-    src/rapidjson_extra.cpp
    src/speedtestutil.cpp
    src/subexport.cpp
    src/templates.cpp
--- a/base/pref.ini
+++ b/base/pref.ini
@@ -54,6 +54,8 @@ sssub_rule_base=base/all_base.tpl

 ;Proxy used to download configs, rulesets or subscriptions, set to NONE or empty to disable it, set to SYSTEM to use system proxy.
 ;Accept cURL-supported proxies (http:// https:// socks4a:// socks5://)
+;Additional support for CORS proxy ( https://github.com/Rob--W/cors-anywhere https://github.com/Zibri/cloudflare-cors-anywhere etc.), prefix the address with "cors:" to recognize the address as CORS proxy.
+;Example:  http://127.0.0.1:80  socks5://example.com:1080  cors:https://cors-anywhere.herokuapp.com/
 proxy_config=SYSTEM
 proxy_ruleset=SYSTEM
 proxy_subscription=NONE
--- a/scripts/build.clang.sh
+++ b/scripts/build.clang.sh
@@ -9,7 +9,6 @@ c++ -std=c++17 -Wall -fexceptions -c src/main.cpp -o obj/main.o
 c++ -std=c++17 -Wall -fexceptions -c src/misc.cpp -o obj/misc.o
 c++ -std=c++17 -Wall -fexceptions -c src/multithread.cpp -o obj/multithread.o
 c++ -std=c++17 -Wall -fexceptions -c src/nodemanip.cpp -o obj/nodemanip.o
-c++ -std=c++17 -Wall -fexceptions -c src/rapidjson_extra.cpp -o obj/rapidjson_extra.o
 c++ -std=c++17 -Wall -fexceptions -c src/speedtestutil.cpp -o obj/speedtestutil.o
 c++ -std=c++17 -Wall -fexceptions -c src/subexport.cpp -o obj/subexport.o
 c++ -std=c++17 -Wall -fexceptions -c src/upload.cpp -o obj/upload.o
--- a/src/rapidjson_extra.cpp
+++ b/src/rapidjson_extra.cpp
@@ -1,56 +0,0 @@
-#include <rapidjson/writer.h>
-
-#include "rapidjson_extra.h"
-
-void operator >> (const rapidjson::Value& value, std::string& i)
-{
-    if(value.IsNull())
-        i = std::string();
-    else if(value.IsInt64())
-        i = std::to_string(value.GetInt64());
-    else if(value.IsDouble())
-        i = std::to_string(value.GetDouble());
-    else if(value.IsString())
-        i = std::string(value.GetString());
-    else if(value.IsBool())
-        i = value.GetBool() ? "true" : "false";
-    else
-        i = std::string();
-}
-
-void operator >> (const rapidjson::Value& value, int& i)
-{
-    if(value.IsNull())
-        i = 0;
-    else if(value.IsInt())
-        i = value.GetInt();
-    else if(value.IsString())
-        i = std::stoi(value.GetString());
-    else if(value.IsBool())
-        i = value.GetBool() ? 1 : 0;
-    else
-        i = 0;
-}
-
-std::string GetMember(const rapidjson::Value& value, const std::string &member)
-{
-    std::string retStr;
-    if(value.HasMember(member.data()))
-        value[member.data()] >> retStr;
-    return retStr;
-}
-
-void GetMember(const rapidjson::Value& value, const std::string &member, std::string& target)
-{
-    std::string retStr = GetMember(value, member);
-    if(retStr.size())
-        target.assign(retStr);
-}
-
-std::string SerializeObject(const rapidjson::Value& value)
-{
-    rapidjson::StringBuffer sb;
-    rapidjson::Writer<rapidjson::StringBuffer> writer_json(sb);
-    value.Accept(writer_json);
-    return sb.GetString();
-}
--- a/src/rapidjson_extra.h
+++ b/src/rapidjson_extra.h
@@ -1,13 +1,71 @@
 #ifndef RAPIDJSON_EXTRA_H_INCLUDED
 #define RAPIDJSON_EXTRA_H_INCLUDED

+#include <exception>
+
+template <typename T> void exception_thrower(T e)
+{
+    if(!e)
+        throw std::runtime_error("assert");
+}
+
+#define RAPIDJSON_ASSERT(x) exception_thrower(x)
 #include <rapidjson/document.h>
+#include <rapidjson/writer.h>
 #include <string>

-void operator >> (const rapidjson::Value& value, std::string& i);
-void operator >> (const rapidjson::Value& value, int& i);
-std::string GetMember(const rapidjson::Value& value, const std::string &member);
-void GetMember(const rapidjson::Value& value, const std::string &member, std::string& target);
-std::string SerializeObject(const rapidjson::Value& value);
+static inline void operator >> (const rapidjson::Value& value, std::string& i)
+{
+    if(value.IsNull())
+        i = std::string();
+    else if(value.IsInt64())
+        i = std::to_string(value.GetInt64());
+    else if(value.IsDouble())
+        i = std::to_string(value.GetDouble());
+    else if(value.IsString())
+        i = std::string(value.GetString());
+    else if(value.IsBool())
+        i = value.GetBool() ? "true" : "false";
+    else
+        i = std::string();
+}
+
+static inline void operator >> (const rapidjson::Value& value, int& i)
+{
+    if(value.IsNull())
+        i = 0;
+    else if(value.IsInt())
+        i = value.GetInt();
+    else if(value.IsString())
+        i = std::stoi(value.GetString());
+    else if(value.IsBool())
+        i = value.GetBool() ? 1 : 0;
+    else
+        i = 0;
+}
+
+static inline std::string GetMember(const rapidjson::Value& value, const std::string &member)
+{
+    std::string retStr;
+    if(value.IsObject() && value.HasMember(member.data()))
+        value[member.data()] >> retStr;
+    return retStr;
+}
+
+static inline void GetMember(const rapidjson::Value& value, const std::string &member, std::string& target)
+{
+    std::string retStr = GetMember(value, member);
+    if(retStr.size())
+        target.assign(retStr);
+}
+
+static inline std::string SerializeObject(const rapidjson::Value& value)
+{
+    rapidjson::StringBuffer sb;
+    rapidjson::Writer<rapidjson::StringBuffer> writer_json(sb);
+    value.Accept(writer_json);
+    return sb.GetString();
+}
+

 #endif // RAPIDJSON_EXTRA_H_INCLUDED
--- a/src/speedtestutil.cpp
+++ b/src/speedtestutil.cpp
@@ -5,7 +5,6 @@

 #define PCRE2_CODE_UNIT_WIDTH 8
 #include <pcre2.h>
-#include <rapidjson/document.h>

 #include "misc.h"
 #include "printout.h"
@@ -117,75 +116,84 @@ void explodeVmessConf(std::string content, const std::string &custom_port, bool
    int configType, index = nodes.size();
    std::map<std::string, std::string> subdata;
    std::map<std::string, std::string>::iterator iter;
+    std::string streamset = "streamSettings", tcpset = "tcpSettings", wsset = "wsSettings";
+    regGetMatch(content, "((?i)streamsettings)", 2, NULL, &streamset);
+    regGetMatch(content, "((?i)tcpsettings)", 2, NULL, &tcpset);
+    regGetMatch(content, "((?1)wssettings)", 2, NULL, &wsset);

    json.Parse(content.data());
    if(json.HasParseError())
        return;
-    if(json.HasMember("outbounds")) //single config
+    try
    {
-        if(json["outbounds"].Size() > 0 && json["outbounds"][0].HasMember("settings") && json["outbounds"][0]["settings"].HasMember("vnext") && json["outbounds"][0]["settings"]["vnext"].Size() > 0)
+        if(json.HasMember("outbounds")) //single config
        {
-            nodejson = json["outbounds"][0];
-            add = GetMember(nodejson["settings"]["vnext"][0], "address");
-            port = GetMember(nodejson["settings"]["vnext"][0], "port");
-            if(nodejson["settings"]["vnext"][0]["users"].Size())
+            if(json["outbounds"].Size() > 0 && json["outbounds"][0].HasMember("settings") && json["outbounds"][0]["settings"].HasMember("vnext") && json["outbounds"][0]["settings"]["vnext"].Size() > 0)
            {
-                id = GetMember(nodejson["settings"]["vnext"][0]["users"][0], "id");
-                aid = GetMember(nodejson["settings"]["vnext"][0]["users"][0], "alterId");
-                cipher = GetMember(nodejson["settings"]["vnext"][0]["users"][0], "security");
-            }
-            if(nodejson.HasMember("streamSettings"))
-            {
-                net = GetMember(nodejson["streamSettings"], "network");
-                tls = GetMember(nodejson["streamSettings"], "security");
-                if(net == "ws")
+                nodejson = json["outbounds"][0];
+                add = GetMember(nodejson["settings"]["vnext"][0], "address");
+                port = GetMember(nodejson["settings"]["vnext"][0], "port");
+                if(nodejson["settings"]["vnext"][0]["users"].Size())
                {
-                    if(nodejson["streamSettings"].HasMember("wssettings"))
-                        settings = nodejson["streamSettings"]["wssettings"];
-                    else if(nodejson["streamSettings"].HasMember("wsSettings"))
-                        settings = nodejson["streamSettings"]["wsSettings"];
+                    id = GetMember(nodejson["settings"]["vnext"][0]["users"][0], "id");
+                    aid = GetMember(nodejson["settings"]["vnext"][0]["users"][0], "alterId");
+                    cipher = GetMember(nodejson["settings"]["vnext"][0]["users"][0], "security");
+                }
+                if(nodejson.HasMember(streamset.data()))
+                {
+                    net = GetMember(nodejson[streamset.data()], "network");
+                    tls = GetMember(nodejson[streamset.data()], "security");
+                    if(net == "ws")
+                    {
+                        if(nodejson[streamset.data()].HasMember(wsset.data()))
+                            settings = nodejson[streamset.data()][wsset.data()];
+                        else
+                            settings.RemoveAllMembers();
+                        path = GetMember(settings, "path");
+                        if(settings.HasMember("headers"))
+                        {
+                            host = GetMember(settings["headers"], "Host");
+                            edge = GetMember(settings["headers"], "Edge");
+                        }
+                    }
+                    if(nodejson[streamset.data()].HasMember(tcpset.data()))
+                        settings = nodejson[streamset.data()][tcpset.data()];
                    else
                        settings.RemoveAllMembers();
-                    path = GetMember(settings, "path");
-                    if(settings.HasMember("headers"))
+                    if(settings.IsObject() && settings.HasMember("header"))
                    {
-                        host = GetMember(settings["headers"], "Host");
-                        edge = GetMember(settings["headers"], "Edge");
-                    }
-                }
-                if(nodejson["streamSettings"].HasMember("tcpSettings"))
-                    settings = nodejson["streamSettings"]["tcpSettings"];
-                else if(nodejson["streamSettings"].HasMember("tcpsettings"))
-                    settings = nodejson["streamSettings"]["tcpsettings"];
-                else
-                    settings.RemoveAllMembers();
-                if(settings.HasMember("header"))
-                {
-                    type = GetMember(settings["header"], "type");
-                    if(type == "http")
-                    {
-                        if(settings["header"].HasMember("request"))
+                        type = GetMember(settings["header"], "type");
+                        if(type == "http")
                        {
-                            if(settings["header"]["request"].HasMember("path") && settings["header"]["request"]["path"].Size())
-                                settings["header"]["request"]["path"][0] >> path;
-                            if(settings["header"]["request"].HasMember("headers"))
+                            if(settings["header"].HasMember("request"))
                            {
-                                host = GetMember(settings["header"]["request"]["headers"], "Host");
-                                edge = GetMember(settings["header"]["request"]["headers"], "Edge");
+                                if(settings["header"]["request"].HasMember("path") && settings["header"]["request"]["path"].Size())
+                                    settings["header"]["request"]["path"][0] >> path;
+                                if(settings["header"]["request"].HasMember("headers"))
+                                {
+                                    host = GetMember(settings["header"]["request"]["headers"], "Host");
+                                    edge = GetMember(settings["header"]["request"]["headers"], "Edge");
+                                }
                            }
                        }
                    }
                }
+                node.linkType = SPEEDTEST_MESSAGE_FOUNDVMESS;
+                node.proxyStr = vmessConstruct(add, port, type, id, aid, net, cipher, path, host, edge, tls, udp, tfo, scv);
+                node.group = V2RAY_DEFAULT_GROUP;
+                node.remarks = add + ":" + port;
+                node.server = add;
+                node.port = to_int(port);
+                nodes.push_back(node);
            }
-            node.linkType = SPEEDTEST_MESSAGE_FOUNDVMESS;
-            node.proxyStr = vmessConstruct(add, port, type, id, aid, net, cipher, path, host, edge, tls, udp, tfo, scv);
-            node.group = V2RAY_DEFAULT_GROUP;
-            node.remarks = add + ":" + port;
-            node.server = add;
-            node.port = to_int(port);
-            nodes.push_back(node);
+            return;
        }
+    }
+    catch(std::exception & e)
+    {
+        writeLog(0, "VMessConf parser throws an error. Leaving...", LOG_LEVEL_WARNING);
        return;
+        //ignore
    }
    //read all subscribe remark as group name
    for(unsigned int i = 0; i < json["subItem"].Size(); i++)
--- a/src/subexport.cpp
+++ b/src/subexport.cpp
@@ -1615,40 +1615,34 @@ std::string netchToSS(std::vector<nodeInfo> &nodes, extra_settings &ext)

 std::string netchToSSSub(std::string &base_conf, std::vector<nodeInfo> &nodes, extra_settings &ext)
 {
-    rapidjson::Document json;
-    rapidjson::StringBuffer sb;
-    rapidjson::Writer<rapidjson::StringBuffer> writer(sb);
+    rapidjson::Document json, base;
    std::string remark, hostname, password, method;
    std::string plugin, pluginopts;
    std::string protocol, obfs;
-    std::string route, remote_dns, ipv6, metered, proxy_apps_enabled, bypass, udpdns;
-    string_array android_list;
+    std::string output_content;
    int port;

-    json.Parse(base_conf.data());
-    if(!json.HasParseError())
+    rapidjson::Document::AllocatorType &alloc = json.GetAllocator();
+    json.SetObject();
+    json.AddMember("remarks", "", alloc);
+    json.AddMember("server", "", alloc);
+    json.AddMember("server_port", 0, alloc);
+    json.AddMember("method", "", alloc);
+    json.AddMember("password", "", alloc);
+    json.AddMember("plugin", "", alloc);
+    json.AddMember("plugin_opts", "", alloc);
+
+    base.Parse(base_conf.data());
+    if(!base.HasParseError())
    {
-        route = GetMember(json, "route");
-        remote_dns = GetMember(json, "remote_dns");
-        ipv6 = GetMember(json, "ipv6");
-        metered = GetMember(json, "metered");
-        udpdns = GetMember(json, "udpdns");
-        if(json.HasMember("proxy_apps") && json["proxy_apps"].IsObject())
-        {
-            proxy_apps_enabled = GetMember(json["proxy_apps"], "enabled");
-            bypass = GetMember(json["proxy_apps"], "bypass");
-            if(json["proxy_apps"].HasMember("android_list") && json["proxy_apps"]["android_list"].IsArray())
-            {
-                for(size_t i = 0; i < json["proxy_apps"]["android_list"].Size(); i++)
-                {
-                    if(json["proxy_apps"]["android_list"][i].IsString())
-                        android_list.push_back(json["proxy_apps"]["android_list"][i].GetString());
-                }
-            }
-        }
+        for(auto iter = base.MemberBegin(); iter != base.MemberEnd(); iter++)
+            json.AddMember(iter->name, iter->value, alloc);
    }

-    writer.StartArray();
+    rapidjson::Value jsondata;
+    json.Swap(jsondata);
+
+    output_content = "[";
    for(nodeInfo &x : nodes)
    {
        json.Parse(x.proxyStr.data());
@@ -1676,75 +1670,19 @@ std::string netchToSSSub(std::string &base_conf, std::vector<nodeInfo> &nodes, e
        default:
            continue;
        }
-        writer.StartObject();
-        writer.Key("server");
-        writer.String(hostname.data());
-        writer.Key("server_port");
-        writer.Int(port);
-        writer.Key("method");
-        writer.String(method.data());
-        writer.Key("password");
-        writer.String(password.data());
-        writer.Key("remarks");
-        writer.String(remark.data());
-        writer.Key("plugin");
-        writer.String(plugin.data());
-        writer.Key("plugin_opts");
-        writer.String(pluginopts.data());
-        if(route.size())
-        {
-            writer.Key("route");
-            writer.String(route.data());
-        }
-        if(remote_dns.size())
-        {
-            writer.Key("remote_dns");
-            writer.Bool(remote_dns == "true");
-        }
-        if(ipv6.size())
-        {
-            writer.Key("ipv6");
-            writer.Bool(ipv6 == "true");
-        }
-        if(metered.size())
-        {
-            writer.Key("metered");
-            writer.Bool(metered == "true");
-        }
-        if(udpdns.size())
-        {
-            writer.Key("udpdns");
-            writer.Bool(udpdns == "true");
-        }
-        if(proxy_apps_enabled.size())
-        {
-            bool enabled = proxy_apps_enabled == "true";
-            writer.Key("proxy_apps");
-            writer.StartObject();
-            writer.Key("enabled");
-            writer.Bool(enabled);
-            if(enabled)
-            {
-                if(bypass.size())
-                {
-                    writer.Key("bypass");
-                    writer.Bool(bypass == "true");
-                }
-            }
-            if(android_list.size())
-            {
-                writer.Key("android_list");
-                writer.StartArray();
-                for(const std::string &x : android_list)
-                    writer.String(x.data());
-                writer.EndArray();
-            }
-            writer.EndObject();
-        }
-        writer.EndObject();
+        jsondata["remarks"].SetString(rapidjson::StringRef(remark.c_str(), remark.size()));
+        jsondata["server"].SetString(rapidjson::StringRef(hostname.c_str(), hostname.size()));
+        jsondata["server_port"] = port;
+        jsondata["password"].SetString(rapidjson::StringRef(password.c_str(), password.size()));
+        jsondata["method"].SetString(rapidjson::StringRef(method.c_str(), method.size()));
+        jsondata["plugin"].SetString(rapidjson::StringRef(plugin.c_str(), plugin.size()));
+        jsondata["plugin_opts"].SetString(rapidjson::StringRef(pluginopts.c_str(), pluginopts.size()));
+        output_content += SerializeObject(jsondata) + ",";
    }
-    writer.EndArray();
-    return sb.GetString();
+    if(output_content.size() > 1)
+        output_content.erase(output_content.size() - 1);
+    output_content += "]";
+    return output_content;
 }

 std::string netchToSSR(std::vector<nodeInfo> &nodes, extra_settings &ext)
@@ -2322,27 +2260,27 @@ void netchToQuanX(std::vector<nodeInfo> &nodes, INIReader &ini, std::vector<rule
        case "ssid"_hash:
            if(rules_upper_bound < 4)
                continue;
-            proxies = vArray[0] + ",";
-            proxies += std::accumulate(vArray.begin() + 3, vArray.end(), vArray[2], [](std::string a, std::string b)
-            {
-                return std::move(a) + "," + replace_all_distinct(b, "=", ":");
-            });
-            ini.Set("{NONAME}", vArray[1] + "=" + proxies); //insert order
-            continue;
+            type = "ssid";
+            for(auto iter = vArray.begin() + 2; iter != vArray.end(); iter++)
+                filtered_nodelist.emplace_back(replace_all_distinct(*iter, "=", ":"));
+            break;
        default:
            continue;
        }

        name = vArray[0];

-        for(unsigned int i = 2; i < rules_upper_bound; i++)
-            groupGenerate(vArray[i], nodelist, filtered_nodelist, true);
+        if(hash_(vArray[1]) != "ssid"_hash)
+        {
+            for(unsigned int i = 2; i < rules_upper_bound; i++)
+                groupGenerate(vArray[i], nodelist, filtered_nodelist, true);

-        if(!filtered_nodelist.size())
-            filtered_nodelist.emplace_back("direct");
+            if(!filtered_nodelist.size())
+                filtered_nodelist.emplace_back("direct");

-        if(filtered_nodelist.size() < 2) // force groups with 1 node to be static
-            type = "static";
+            if(filtered_nodelist.size() < 2) // force groups with 1 node to be static
+                type = "static";
+        }

        auto iter = std::find_if(original_groups.begin(), original_groups.end(), [name](const string_multimap::value_type &n)
        {
--- a/src/upload.cpp
+++ b/src/upload.cpp
@@ -1,8 +1,5 @@
 #include <string>

-#include <rapidjson/writer.h>
-#include <rapidjson/document.h>
-
 #include "webget.h"
 #include "ini_reader.h"
 #include "logger.h"
--- a/src/webget.cpp
+++ b/src/webget.cpp
@@ -23,7 +23,7 @@ typedef std::lock_guard<std::mutex> guarded_mutex;
 std::mutex cache_rw_lock;

 //std::string user_agent_str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36";
-std::string user_agent_str = "subconverter/" + std::string(VERSION) + " cURL/" + std::string(LIBCURL_VERSION);
+std::string user_agent_str = "subconverter/" VERSION " cURL/" LIBCURL_VERSION;

 static inline void curl_init()
 {
@@ -70,19 +70,29 @@ static inline void curl_set_common_options(CURL *curl_handle, const char *url)
 static std::string curlGet(const std::string &url, const std::string &proxy, std::string &response_headers, CURLcode &return_code)
 {
    CURL *curl_handle;
-    std::string data;
+    std::string data, new_url = url;
+    struct curl_slist *list = NULL;
    long retVal = 0;

    curl_init();

    curl_handle = curl_easy_init();
-    curl_set_common_options(curl_handle, url.data());
+    if(proxy.size())
+    {
+        if(startsWith(proxy, "cors:"))
+        {
+            list = curl_slist_append(list, "X-Requested-With: subconverter " VERSION);
+            curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list);
+            new_url = proxy.substr(5) + url;
+        }
+        else
+            curl_easy_setopt(curl_handle, CURLOPT_PROXY, proxy.data());
+    }
+    curl_set_common_options(curl_handle, new_url.data());
    curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, writer);
    curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, &data);
    curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, writer);
    curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, &response_headers);
-    if(proxy.size())
-        curl_easy_setopt(curl_handle, CURLOPT_PROXY, proxy.data());

    return_code = curl_easy_perform(curl_handle);
    curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &retVal);