Enhancements

Fix when subscription download timeout before complete, the parser will still try to parse downloaded content.
Change default regex library to PCRE2.
Change Quantumult X "url-test" equivalent to "available".
Optimize codes.
Update build scripts.

CMakeLists.txt

@@ -7,7 +7,7 @@ ENDIF()
 SET(CMAKE_CXX_STANDARD 17)
 ADD_DEFINITIONS(-Wall -Wextra -Wno-unused-parameter -Wno-unused-result)
 
-OPTION(USING_STD_REGEX "Use std::regex from C++ library instead of PCRECPP." OFF)
+OPTION(USING_STD_REGEX "Use std::regex from C++ library instead of PCRE2." OFF)
 OPTION(USING_MALLOC_TRIM "Call malloc_trim after processing request to lower memory usage (Your system must support malloc_trim)." OFF)
 
 INCLUDE(CheckCXXSourceCompiles)
@@ -77,10 +77,13 @@ TARGET_LINK_LIBRARIES(subconverter ${YAML_CPP_LIBRARY})
 IF(USING_STD_REGEX STREQUAL "ON")
     ADD_DEFINITIONS(-DUSE_STD_REGEX)
 ELSE()
-	FIND_PACKAGE(PCRECPP REQUIRED)
+    PKG_CHECK_MODULES(PCRE2 libpcre2-8 REQUIRED)
-	INCLUDE_DIRECTORIES(${PCRECPP_INCLUDE_DIRS})
+    INCLUDE_DIRECTORIES(${PCRE2_INCLUDE_DIRS})
-	TARGET_LINK_LIBRARIES(subconverter ${PCRECPP_LIBRARIES})
+    TARGET_LINK_LIBRARIES(subconverter ${PCRE2_LIBRARIES})
-	ADD_DEFINITIONS(-DPCRE_STATIC)
+    ADD_DEFINITIONS(-DPCRE2_STATIC)
+  
+    FIND_PACKAGE(JPCRE2 REQUIRED)
+    INCLUDE_DIRECTORIES(${JPCRE2_INCLUDE_DIRS})
 ENDIF()
 
 IF(WIN32)

cmake/FindJPCRE2.cmake

@@ -0,0 +1,27 @@
+# - Find JPCRE2
+# Find the JPCRE2 headers.
+#
+#  JPCRE2_INCLUDE_DIRS - where to find jpcre2.h.
+#  JPCRE2_FOUND        - True if JPCRE2 found.
+
+# Look for the header file.
+FIND_PATH(JPCRE2_INCLUDE_DIR NAMES jpcre2.hpp)
+MARK_AS_ADVANCED(JPCRE2_INCLUDE_DIR)
+
+if(JPCRE2_INCLUDE_DIR)
+  set(JPCRE2_FOUND TRUE)
+endif()
+
+IF(JPCRE2_FOUND)
+  SET(JPCRE2_INCLUDE_DIRS ${JPCRE2_INCLUDE_DIR})
+ENDIF(JPCRE2_FOUND)
+
+if(JPCRE2_FOUND)
+  if(NOT JPCRE2_FIND_QUIETLY)
+    message(STATUS "Found JPCRE2 header files in ${JPCRE2_INCLUDE_DIRS}")
+  endif()
+elseif(JPCRE2_FIND_REQUIRED)
+    message(FATAL_ERROR "Could not find JPCRE2")
+else()
+  message(STATUS "Optional package JPCRE2 was not found")
+endif()

scripts/Dockerfile

@@ -3,7 +3,7 @@ MAINTAINER Tindy X <tindy.it@gmail.com>
 
 # build minimized
 RUN apk add git g++ build-base linux-headers cmake && \
-    apk add libressl-dev curl-dev rapidjson-dev libevent-dev pcre-dev yaml-cpp-dev && \
+    apk add libressl-dev curl-dev rapidjson-dev libevent-dev pcre2-dev yaml-cpp-dev && \
     git clone https://github.com/tindy2013/subconverter && \
     cd subconverter && \
     cmake . && \
@@ -12,8 +12,8 @@ RUN apk add git g++ build-base linux-headers cmake && \
     mv base ../ && \
     cd .. && \
     rm -rf subconverter && \
-    apk add libcurl yaml-cpp libevent libpcrecpp && \
+    apk add pcre2 libcurl yaml-cpp libevent libpcrecpp && \
-    apk del git gcc g++ build-base linux-headers cmake libressl-dev curl-dev rapidjson-dev libevent-dev pcre-dev yaml-cpp-dev
+    apk del git gcc g++ build-base linux-headers cmake libressl-dev curl-dev rapidjson-dev libevent-dev pcre2-dev yaml-cpp-dev
 
 # set entry
 CMD /base/subconverter

scripts/build.alpine.release.sh

@@ -3,7 +3,7 @@ mkdir obj
 set -xe
 
 apk add gcc g++ build-base linux-headers cmake make autoconf automake libtool
-apk add openssl-dev openssl-libs-static curl-dev curl-static nghttp2-static zlib-dev rapidjson-dev libevent-dev libevent-static zlib-static pcre-dev bzip2-static
+apk add openssl-dev openssl-libs-static curl-dev curl-static nghttp2-static zlib-dev rapidjson-dev libevent-dev libevent-static zlib-static pcre2-dev bzip2-static 
 
 git clone https://github.com/jbeder/yaml-cpp
 cd yaml-cpp
@@ -11,9 +11,12 @@ cmake -DYAML_CPP_BUILD_TESTS=OFF -DYAML_CPP_BUILD_TOOLS=OFF . > /dev/null
 make install -j4 > /dev/null
 cd ..
 
+curl -LO https://raw.githubusercontent.com/jpcre2/jpcre2/release/src/jpcre2.hpp
+cp jpcre2.hpp /usr/local/include/
+
 cmake .
 make -j4
-g++ -o base/subconverter CMakeFiles/subconverter.dir/src/*.o  -static -lpcrecpp -lpcre -levent -lyaml-cpp -lcurl -lnghttp2 -lssl -lcrypto -lz -lbz2 -ldl -lpthread -O3 -s  
+g++ -o base/subconverter CMakeFiles/subconverter.dir/src/*.o  -static -lpcre2-8 -levent -lyaml-cpp -lcurl -lnghttp2 -lssl -lcrypto -lz -lbz2 -ldl -lpthread -O3 -s  
 
 cd base
 chmod +rx subconverter

scripts/build.clang.sh

@@ -3,16 +3,16 @@ mkdir obj
 
 set -xe
 
-c++ -std=c++17 -D_MACOS -Wall -fexceptions -c src/logger.cpp -o obj/logger.o
+c++ -std=c++17 -Wall -fexceptions -c src/logger.cpp -o obj/logger.o
-c++ -std=c++17 -D_MACOS -Wall -fexceptions -c src/main.cpp -o obj/main.o
+c++ -std=c++17 -Wall -fexceptions -c src/main.cpp -o obj/main.o
-c++ -std=c++17 -D_MACOS -Wall -fexceptions -c src/misc.cpp -o obj/misc.o
+c++ -std=c++17 -Wall -fexceptions -c src/misc.cpp -o obj/misc.o
-c++ -std=c++17 -D_MACOS -Wall -fexceptions -c src/multithread.cpp -o obj/multithread.o
+c++ -std=c++17 -Wall -fexceptions -c src/multithread.cpp -o obj/multithread.o
-c++ -std=c++17 -D_MACOS -Wall -fexceptions -c src/nodemanip.cpp -o obj/nodemanip.o
+c++ -std=c++17 -Wall -fexceptions -c src/nodemanip.cpp -o obj/nodemanip.o
-c++ -std=c++17 -D_MACOS -Wall -fexceptions -c src/rapidjson_extra.cpp -o obj/rapidjson_extra.o
+c++ -std=c++17 -Wall -fexceptions -c src/rapidjson_extra.cpp -o obj/rapidjson_extra.o
-c++ -std=c++17 -D_MACOS -Wall -fexceptions -c src/speedtestutil.cpp -o obj/speedtestutil.o
+c++ -std=c++17 -Wall -fexceptions -c src/speedtestutil.cpp -o obj/speedtestutil.o
-c++ -std=c++17 -D_MACOS -Wall -fexceptions -c src/subexport.cpp -o obj/subexport.o
+c++ -std=c++17 -Wall -fexceptions -c src/subexport.cpp -o obj/subexport.o
-c++ -std=c++17 -D_MACOS -Wall -fexceptions -c src/webget.cpp -o obj/webget.o
+c++ -std=c++17 -Wall -fexceptions -c src/webget.cpp -o obj/webget.o
-c++ -std=c++17 -D_MACOS -Wall -fexceptions -c src/webserver_libevent.cpp -o obj/webserver_libevent.o
+c++ -std=c++17 -Wall -fexceptions -c src/webserver_libevent.cpp -o obj/webserver_libevent.o
-c++ -o subconverter obj/logger.o obj/main.o obj/misc.o obj/multithread.o obj/nodemanip.o obj/rapidjson_extra.o obj/speedtestutil.o obj/subexport.o obj/webget.o obj/webserver_libevent.o -lpcrecpp -lpcre -levent -lpthread -lyaml-cpp -lcurl -lssl -lcrypto -lz -O3 -s 
+c++ -o subconverter obj/*.o -lpcre2-8 -levent -lpthread -lyaml-cpp -lcurl -lssl -lcrypto -lz -O3 -s 
 
 chmod +x subconverter

scripts/build.macos.release.sh

@@ -2,7 +2,7 @@
 mkdir obj
 set -xe
 
-brew reinstall yaml-cpp rapidjson libevent zlib pcre bzip2 pkgconfig
+brew reinstall yaml-cpp rapidjson libevent zlib pcre2 bzip2 pkgconfig
 
 git clone https://github.com/curl/curl
 cd curl
@@ -11,20 +11,22 @@ cd curl
 make -j8 > /dev/null
 cd ..
 
+curl -LO https://raw.githubusercontent.com/jpcre2/jpcre2/release/src/jpcre2.hpp
+cp jpcre2.hpp /usr/local/include/
+
 cp curl/lib/.libs/libcurl.a .
 cp /usr/local/lib/libevent.a .
 cp /usr/local/opt/zlib/lib/libz.a .
 cp /usr/local/opt/openssl@1.1/lib/libssl.a .
 cp /usr/local/opt/openssl@1.1/lib/libcrypto.a .
 cp /usr/local/lib/libyaml-cpp.a .
-cp /usr/local/lib/libpcre.a .
+cp /usr/local/lib/libpcre2-8.a .
-cp /usr/local/lib/libpcrecpp.a .
 cp /usr/local/opt/bzip2/lib/libbz2.a .
 
 export CMAKE_CXX_FLAGS="-I/usr/local/include -I/usr/local/opt/openssl@1.1/include -I/usr/local/opt/curl/include"
-cmake -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl@1.1 -DMACOS=on .
+cmake -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl@1.1 .
 make -j8
-c++ -Xlinker -unexported_symbol -Xlinker "*" -o base/subconverter CMakeFiles/subconverter.dir/src/*.o libpcrecpp.a libpcre.a libevent.a libcurl.a libz.a libssl.a libcrypto.a libyaml-cpp.a libbz2.a -ldl -lpthread -O3
+c++ -Xlinker -unexported_symbol -Xlinker "*" -o base/subconverter CMakeFiles/subconverter.dir/src/*.o libpcre2-8.a libevent.a libcurl.a libz.a libssl.a libcrypto.a libyaml-cpp.a libbz2.a -ldl -lpthread -O3
 
 cd base
 chmod +rx subconverter

scripts/config.termux.sh

@@ -3,7 +3,7 @@ set -xe
 
 apt update
 apt install -y git cmake clang pkg-config
-apt install -y libevent libcurl openssl pcre
+apt install -y libevent libcurl openssl pcre2
 
 git clone https://github.com/jbeder/yaml-cpp
 cd yaml-cpp
@@ -15,3 +15,6 @@ git clone https://github.com/tencent/rapidjson
 cd rapidjson
 cp -r include/* /data/data/com.termux/files/usr/include/
 cd ..
+
+curl -LO https://raw.githubusercontent.com/jpcre2/jpcre2/release/src/jpcre2.hpp
+cp jpcre2.hpp /data/data/com.termux/files/usr/include/

src/misc.cpp

@@ -11,7 +11,8 @@
 #ifdef USE_STD_REGEX
 #include <regex>
 #else
-#include <pcrecpp.h>
+#include <jpcre2.hpp>
+typedef jpcre2::select<char> jp;
 #endif // USE_STD_REGEX
 
 #include <rapidjson/document.h>
@@ -489,7 +490,7 @@ bool regValid(std::string &reg)
 {
     try
     {
-        std::regex r(reg);
+        std::regex r(reg, std::regex::ECMAScript);
         return true;
     }
     catch (std::regex_error &e)
@@ -502,7 +503,13 @@ bool regFind(std::string src, std::string target)
 {
     try
     {
-        std::regex reg(target);
+        std::regex::flag_type flags = std::regex::extended | std::regex::ECMAScript;
+        if(target.find("(?i)") == 0)
+        {
+            target.erase(0, 4);
+            flags |= std::regex::icase;
+        }
+        std::regex reg(target, flags);
         return regex_search(src, reg);
     }
     catch (std::regex_error &e)
@@ -516,7 +523,13 @@ std::string regReplace(std::string src, std::string match, std::string rep)
     std::string result = "";
     try
     {
-        std::regex reg(match);
+        std::regex::flag_type flags = std::regex::extended | std::regex::ECMAScript;
+        if(match.find("(?i)") == 0)
+        {
+            match.erase(0, 4);
+            flags |= std::regex::icase;
+        }
+        std::regex reg(match, flags);
         regex_replace(back_inserter(result), src.begin(), src.end(), reg, rep);
     }
     catch (std::regex_error &e)
@@ -530,7 +543,13 @@ bool regMatch(std::string src, std::string match)
 {
     try
     {
-        std::regex reg(match);
+        std::regex::flag_type flags = std::regex::extended | std::regex::ECMAScript;
+        if(match.find("(?i)") == 0)
+        {
+            match.erase(0, 4);
+            flags |= std::regex::icase;
+        }
+        std::regex reg(match, flags);
         return regex_match(src, reg);
     }
     catch (std::regex_error &e)
@@ -541,62 +560,37 @@ bool regMatch(std::string src, std::string match)
 
 #else
 
-bool regValid(std::string &reg)
+bool regMatch(std::string src, std::string target, bool partial)
-{
-    try
-    {
-        pcrecpp::RE r(reg);
-        return true;
-    }
-    catch (std::exception &e)
 {
+    jp::Regex reg;
+    reg.setPattern(target).addModifier("gm").compile();
+    if(!reg)
         return false;
-    }
+    return reg.replace(src, "$0") == src;
 }
 
 bool regFind(std::string src, std::string target)
 {
-    try
+    jp::Regex reg;
-    {
+    reg.setPattern(target).addModifier("gm").compile();
-        pcrecpp::RE reg(target);
+    if(!reg)
-        return reg.PartialMatch(src);
-    }
-    catch (std::exception &e)
-    {
         return false;
-    }
+    return reg.match(src);
 }
 
-std::string regReplace(std::string src, std::string match, std::string rep)
+std::string regReplace(std::string src, std::string target, std::string rep)
 {
-    std::string result = src;
+    jp::Regex reg;
-    try
+    reg.setPattern(target).addModifier("gm").compile();
-    {
+    if(!reg)
-        if(rep.find("$") != rep.npos)
-            rep = replace_all_distinct(rep, "$", "\\");
-        pcrecpp::RE reg(match);
-        if(reg.GlobalReplace(rep, &result))
-            return result;
-        else
         return src;
-    }
+    return reg.replace(src, rep);
-    catch (std::exception &e)
-    {
-        return src;
-    }
 }
 
-bool regMatch(std::string src, std::string match)
+bool regValid(std::string &target)
 {
-    try
+    jp::Regex reg(target);
-    {
+    return !!reg;
-        pcrecpp::RE reg(match);
-        return reg.FullMatch(src);
-    }
-    catch (std::exception &e)
-    {
-        return false;
-    }
 }
 #endif // USE_STD_REGEX
 

src/misc.h

@@ -40,10 +40,10 @@ bool is_str_utf8(std::string data);
 std::string getFormData(const std::string &raw_data);
 
 void sleep(int interval);
-bool regValid(std::string &reg);
+bool regValid(std::string &target);
 bool regFind(std::string src, std::string target);
 std::string regReplace(std::string src, std::string match, std::string rep);
-bool regMatch(std::string src, std::string match);
+bool regMatch(std::string src, std::string match, bool partial = false);
 std::string speedCalc(double speed);
 std::string getMD5(std::string data);
 bool isIPv4(std::string &address);

src/speedtestutil.cpp

@@ -1593,22 +1593,22 @@ time_t dateStringToTimestamp(std::string date)
 
 bool getSubInfoFromHeader(std::string &header, std::string &result)
 {
-    bool ret = false;
+    std::string pattern = "(?:[\\s\\S]*?)^(?i:Subscription-UserInfo): (.*?)\\s$(?:[\\s\\S]*)", retStr;
-    pcrecpp::RE reg("^(?i:Subscription-UserInfo): (.*?)\\r$", pcrecpp::MULTILINE());
+    if(regFind(header, pattern))
-    try
     {
-        ret = reg.PartialMatch(header, &result);
+        retStr = regReplace(header, pattern, "$1");
-    }
+        if(retStr != header)
-    catch (std::exception &e)
         {
-        //ignore
+            result = retStr;
+            return true;
         }
-    return ret;
+    }
+    return false;
 }
 
 bool getSubInfoFromNodes(std::vector<nodeInfo> &nodes, string_array &stream_rules, string_array &time_rules, std::string &result)
 {
-    std::string remarks, pattern, target, stream_info, time_info;
+    std::string remarks, pattern, target, stream_info, time_info, retStr;
     string_array vArray;
 
     for(nodeInfo &x : nodes)
@@ -1623,13 +1623,14 @@ bool getSubInfoFromNodes(std::vector<nodeInfo> &nodes, string_array &stream_rule
                     continue;
                 pattern = vArray[0];
                 target = vArray[1];
-                pcrecpp::RE reg(pattern, pcrecpp::UTF8());
+                if(regMatch(remarks, pattern))
-                if(reg.FullMatch(remarks))
                 {
-                    if(target.find("$") != target.npos)
+                    retStr = regReplace(remarks, pattern, target);
-                        target = replace_all_distinct(target, "$", "\\");
+                    if(retStr != remarks)
-                    if(reg.GlobalReplace(target, &remarks))
+                    {
-                        stream_info = remarks;
+                        stream_info = retStr;
+                        break;
+                    }
                 }
                 else
                     continue;
@@ -1646,13 +1647,14 @@ bool getSubInfoFromNodes(std::vector<nodeInfo> &nodes, string_array &stream_rule
                     continue;
                 pattern = vArray[0];
                 target = vArray[1];
-                pcrecpp::RE reg(pattern, pcrecpp::UTF8());
+                if(regMatch(remarks, pattern))
-                if(reg.FullMatch(remarks))
                 {
-                    if(target.find("$") != target.npos)
+                    retStr = regReplace(remarks, pattern, target);
-                        target = replace_all_distinct(target, "$", "\\");
+                    if(retStr != remarks)
-                    if(reg.GlobalReplace(target, &remarks))
+                    {
-                        time_info = remarks;
+                        time_info = retStr;
+                        break;
+                    }
                 }
                 else
                     continue;

src/subexport.cpp

@@ -1625,14 +1625,7 @@ void netchToQuanX(std::vector<nodeInfo> &nodes, INIReader &ini, std::vector<rule
             type = "static";
             rules_upper_bound = vArray.size();
         }
-        else if(vArray[1] == "url-test")
+        else if(vArray[1] == "url-test" || vArray[1] == "fallback")
-        {
-            if(vArray.size() < 5)
-                continue;
-            type = "static";
-            rules_upper_bound = vArray.size() - 2;
-        }
-        else if(vArray[1] == "fallback")
         {
             if(vArray.size() < 5)
                 continue;

src/webget.cpp

@@ -23,6 +23,7 @@ static int writer(char *data, size_t size, size_t nmemb, std::string *writerData
 std::string curlGet(std::string url, std::string proxy, std::string &response_headers)
 {
     CURL *curl_handle;
+    CURLcode res;
     std::string data;
 
     curl_global_init(CURL_GLOBAL_ALL);
@@ -43,9 +44,12 @@ std::string curlGet(std::string url, std::string proxy, std::string &response_he
     if(proxy != "")
         curl_easy_setopt(curl_handle, CURLOPT_PROXY, proxy.data());
 
-    curl_easy_perform(curl_handle);
+    res = curl_easy_perform(curl_handle);
     curl_easy_cleanup(curl_handle);
 
+    if(res != CURLE_OK)
+        data.clear();
+
     return data;
 }