Add auto-sync script for rules

Update build scripts.

.github/workflows/build.yml

@@ -15,6 +15,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
     - name: Add commit id into version
       if: ${{ !startsWith(github.ref, 'refs/tags/') }}
       run: SHA=$(git rev-parse --short HEAD) && sed -i 's/\(v[0-9]\.[0-9]\.[0-9]\)/\1-'"$SHA"'/' src/version.h
@@ -40,6 +43,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
     - name: Add commit id into version
       if: ${{ !startsWith(github.ref, 'refs/tags/') }}
       run: SHA=$(git rev-parse --short HEAD) && sed -i 's/\(v[0-9]\.[0-9]\.[0-9]\)/\1-'"$SHA"'/' src/version.h
@@ -65,6 +71,9 @@ jobs:
     runs-on: [self-hosted, linux, ARM64]
     steps:
     - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
     - name: Add commit id into version
       if: ${{ !startsWith(github.ref, 'refs/tags/') }}
       run: SHA=$(git rev-parse --short HEAD) && sed -i 's/\(v[0-9]\.[0-9]\.[0-9]\)/\1-'"$SHA"'/' src/version.h
@@ -90,6 +99,9 @@ jobs:
     runs-on: [self-hosted, linux, ARM64]
     steps:
     - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
     - name: Add commit id into version
       if: ${{ !startsWith(github.ref, 'refs/tags/') }}
       run: SHA=$(git rev-parse --short HEAD) && sed -i 's/\(v[0-9]\.[0-9]\.[0-9]\)/\1-'"$SHA"'/' src/version.h
@@ -115,6 +127,9 @@ jobs:
     runs-on: macos-latest
     steps:
     - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
     - name: Add commit id into version
       if: ${{ !startsWith(github.ref, 'refs/tags/') }}
       run: SHA=$(git rev-parse --short HEAD) && sed -i -e 's/\(v[0-9]\.[0-9]\.[0-9]\)/\1-'"$SHA"'/' src/version.h
@@ -143,6 +158,9 @@ jobs:
         shell: msys2 {0}
     steps:
     - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
     - uses: actions/setup-node@v3
       with:
         node-version: '16'
@@ -180,6 +198,9 @@ jobs:
         shell: msys2 {0}
     steps:
     - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
     - uses: actions/setup-node@v3
       with:
         node-version: '16'

scripts/Dockerfile

@@ -5,7 +5,7 @@ ARG SHA=""
 
 # build minimized
 WORKDIR /
-RUN apk add --no-cache --virtual .build-tools git g++ build-base linux-headers cmake && \
+RUN apk add --no-cache --virtual .build-tools git g++ build-base linux-headers cmake python3 && \
     apk add --no-cache --virtual .build-deps curl-dev rapidjson-dev libevent-dev pcre2-dev yaml-cpp-dev && \
     git clone https://github.com/ftk/quickjspp --depth=1 && \
     cd quickjspp && \
@@ -37,6 +37,9 @@ RUN apk add --no-cache --virtual .build-tools git g++ build-base linux-headers c
     git clone https://github.com/tindy2013/subconverter --depth=1 && \
     cd subconverter && \
     [ -n "$SHA" ] && sed -i 's/\(v[0-9]\.[0-9]\.[0-9]\)/\1-'"$SHA"'/' src/version.h;\
+    python3 -m ensurepip && \
+    python3 -m pip install gitpython && \
+    python3 scripts/update_rules.py -c scripts/rules_config.conf && \
     cmake -DCMAKE_BUILD_TYPE=Release . && \
     make -j $THREADS && \
     mv subconverter /usr/bin && \

scripts/build.alpine.release.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -xe
 
-apk add gcc g++ build-base linux-headers cmake make autoconf automake libtool python2
+apk add gcc g++ build-base linux-headers cmake make autoconf automake libtool python2 python3
 apk add mbedtls-dev mbedtls-static zlib-dev rapidjson-dev libevent-dev libevent-static zlib-static pcre2-dev
 
 git clone https://github.com/curl/curl --depth=1 --branch curl-7_88_1
@@ -46,6 +46,10 @@ make -j2
 rm subconverter
 g++ -o base/subconverter $(find CMakeFiles/subconverter.dir/src/ -name "*.o")  -static -lpcre2-8 -levent -lyaml-cpp -L/usr/lib64 -lcurl -lmbedtls -lmbedcrypto -lmbedx509 -lz -l:quickjs/libquickjs.a -llibcron -O3 -s  
 
+python3 -m ensurepip
+python3 -m pip install gitpython
+python3 scripts/update_rules.py -c scripts/rules_config.conf
+
 cd base
 chmod +rx subconverter
 chmod +r ./*

scripts/build.macos.release.sh

@@ -55,6 +55,10 @@ make -j8
 rm subconverter
 c++ -Xlinker -unexported_symbol -Xlinker "*" -o base/subconverter -framework CoreFoundation -framework Security $(find CMakeFiles/subconverter.dir/src/ -name "*.o") $(find . -name "*.a") -lcurl -O3
 
+python -m ensurepip
+python -m pip install gitpython
+python scripts/update_rules.py -c scripts/rules_config.conf
+
 cd base
 chmod +rx subconverter
 chmod +r ./*

scripts/build.windows.release.sh

@@ -44,6 +44,10 @@ cmake -DCMAKE_INSTALL_PREFIX="$MINGW_PREFIX" -G "Unix Makefiles" -DCMAKE_CXX_STA
 make install -j4
 cd ..
 
+python -m ensurepip
+python -m pip install gitpython
+python scripts/update_rules.py -c scripts/rules_config.conf
+
 rm -f C:/Strawberry/perl/bin/pkg-config C:/Strawberry/perl/bin/pkg-config.bat
 cmake -DCMAKE_BUILD_TYPE=Release -G "Unix Makefiles" .
 make -j4

scripts/rules_config.conf

@@ -0,0 +1,23 @@
+[ACL4SSR]
+name=ACL4SSR
+url=https://github.com/ACL4SSR/ACL4SSR
+checkout=1dc5c92b0c8ceaaecbc66530c309961f53e52c8c
+match=Clash/*.list|Clash/Ruleset/**
+
+[ACL4SSR_config]
+name=ACL4SSR
+url=https://github.com/ACL4SSR/ACL4SSR
+checkout=1dc5c92b0c8ceaaecbc66530c309961f53e52c8c
+match=Clash/config/**
+dest=base/config/
+keep_tree=false
+
+[DivineEngine]
+url=https://github.com/DivineEngine/Profiles
+checkout=f4d75f7d48a3f42129e030bef751d4d22bca02da
+match=Surge/Ruleset/**
+
+[NobyDa]
+url=https://github.com/NobyDa/Script
+checkout=ae4c12f23de8078e02c373c9969b19af28257fcb
+match=Surge/*.list

scripts/update_rules.py

@@ -0,0 +1,66 @@
+import configparser, shutil, glob, os, random, string
+import stat, logging, argparse
+from git import InvalidGitRepositoryError, repo as GitRepo
+
+def del_rw(action, name: str, exc):
+    os.chmod(name, stat.S_IWRITE)
+    os.remove(name)
+
+def open_repo(path: str):
+    if os.path.exists(repo_path) == False: return None
+    try:
+        return GitRepo.Repo(path)
+    except InvalidGitRepositoryError:
+        return None
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-c", "--config", default="rules_config.conf")
+args = parser.parse_args()
+
+config = configparser.ConfigParser()
+config.read(args.config)
+logging.basicConfig(format="%(asctime)s %(message)s", level=logging.DEBUG)
+
+for section in config.sections():
+    repo = config.get(section, "name", fallback=section)
+    url = config.get(section, "url")
+    commit = config.get(section, "checkout")
+    matches = config.get(section, "match").split("|")
+    save_path = config.get(section, "dest", fallback="base/rules/{}".format(repo))
+    keep_tree = config.getboolean(section, "keep_tree", fallback=True)
+
+    logging.info("reading files from url {} with commit {} and matches {}, save to {} keep_tree {}".format(url, commit, matches, save_path, keep_tree))
+
+    # ran_str = ''.join(random.sample(string.ascii_letters + string.digits, 16))
+    repo_path = os.path.join("./tmp/repo/", repo)
+    
+    r = open_repo(repo_path)
+    if r != None:
+        logging.info("repo {} exists, checking out...".format(repo_path))
+        r.git.checkout(commit)
+    else:
+        logging.info("repo {} not exist, cloning...".format(repo_path))
+        shutil.rmtree(repo_path, ignore_errors=True)
+        os.makedirs(repo_path, exist_ok=True)
+        r = GitRepo.Repo.clone_from(url, repo_path)
+        r.git.checkout(commit)
+
+    os.makedirs(save_path, exist_ok=True)
+
+    for pattern in matches:
+        files = glob.glob("{}/{}".format(repo_path, pattern), recursive=True)
+        for file in files:
+            if os.path.isdir(file): continue
+
+            (file_rel_path, file_name) = os.path.split(file.removeprefix(repo_path))
+            if keep_tree: 
+                file_dest_dir = "{}{}".format(save_path, file_rel_path)
+                file_dest_path = "{}/{}".format(file_dest_dir, file_name)
+                os.makedirs(file_dest_dir, exist_ok=True)
+            else:
+                file_dest_path = "{}{}".format(save_path, file_name)
+
+            shutil.copyfile(file, file_dest_path)
+            logging.info("copied {} to {}".format(file, file_dest_path))
+
+shutil.rmtree("./tmp", onerror=del_rw)