mirror of
https://github.com/asdlokj1qpi233/subconverter.git
synced 2026-11-02 01:33:04 +00:00
Complete refactor of project file structure
Fix incorrect handing of default node parameters in SSD subscription. Fix unable to parse some Clash rule-providers (#289). Fix not filtering inline comments in some Surge rulesets. (#285) Add Cron task options. Add clean context option for persisting a JS context across the whole request. Replace JS engine with QuickJS.
This commit is contained in:
88 files changed
+9404
-6356
No files matched your search
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,32 @@
|
||||
#ifndef INTERFACES_H_INCLUDED
|
||||
#define INTERFACES_H_INCLUDED
|
||||
|
||||
#include <string>
|
||||
#include <map>
|
||||
#include <inja.hpp>
|
||||
|
||||
#include "../generator/config/subexport.h"
|
||||
#include "../server/webserver.h"
|
||||
|
||||
void refreshRulesets(string_array &ruleset_list, std::vector<ruleset_content> &rca);
|
||||
void readConf();
|
||||
int simpleGenerator();
|
||||
std::string convertRuleset(const std::string &content, int type);
|
||||
|
||||
std::string getConvertedRuleset(RESPONSE_CALLBACK_ARGS);
|
||||
std::string getScript(RESPONSE_CALLBACK_ARGS);
|
||||
std::string getProfile(RESPONSE_CALLBACK_ARGS);
|
||||
std::string getRuleset(RESPONSE_CALLBACK_ARGS);
|
||||
std::string getRewriteRemote(RESPONSE_CALLBACK_ARGS);
|
||||
|
||||
std::string subconverter(RESPONSE_CALLBACK_ARGS);
|
||||
std::string simpleToClashR(RESPONSE_CALLBACK_ARGS);
|
||||
std::string surgeConfToClash(RESPONSE_CALLBACK_ARGS);
|
||||
|
||||
std::string renderTemplate(RESPONSE_CALLBACK_ARGS);
|
||||
|
||||
std::string template_webGet(inja::Arguments &args);
|
||||
std::string jinja2_webGet(const std::string &url);
|
||||
std::string parseHostname(inja::Arguments &args);
|
||||
|
||||
#endif // INTERFACES_H_INCLUDED
|
||||
@@ -0,0 +1,82 @@
|
||||
#include <future>
|
||||
#include <thread>
|
||||
#include "webget.h"
|
||||
#include "multithread.h"
|
||||
//#include "vfs.h"
|
||||
|
||||
#include "../utils/network.h"
|
||||
|
||||
//safety lock for multi-thread
|
||||
std::mutex on_emoji, on_rename, on_stream, on_time;
|
||||
|
||||
extern string_array gEmojis, gRenames;
|
||||
extern string_array gStreamNodeRules, gTimeNodeRules;
|
||||
|
||||
string_array safe_get_emojis()
|
||||
{
|
||||
guarded_mutex guard(on_emoji);
|
||||
return gEmojis;
|
||||
}
|
||||
|
||||
string_array safe_get_renames()
|
||||
{
|
||||
guarded_mutex guard(on_rename);
|
||||
return gRenames;
|
||||
}
|
||||
|
||||
string_array safe_get_streams()
|
||||
{
|
||||
guarded_mutex guard(on_stream);
|
||||
return gStreamNodeRules;
|
||||
}
|
||||
|
||||
string_array safe_get_times()
|
||||
{
|
||||
guarded_mutex guard(on_time);
|
||||
return gTimeNodeRules;
|
||||
}
|
||||
|
||||
void safe_set_emojis(string_array &data)
|
||||
{
|
||||
guarded_mutex guard(on_emoji);
|
||||
gEmojis.swap(data);
|
||||
}
|
||||
|
||||
void safe_set_renames(string_array &data)
|
||||
{
|
||||
guarded_mutex guard(on_rename);
|
||||
gRenames.swap(data);
|
||||
}
|
||||
|
||||
void safe_set_streams(string_array &data)
|
||||
{
|
||||
guarded_mutex guard(on_stream);
|
||||
gStreamNodeRules.swap(data);
|
||||
}
|
||||
|
||||
void safe_set_times(string_array &data)
|
||||
{
|
||||
guarded_mutex guard(on_time);
|
||||
gTimeNodeRules.swap(data);
|
||||
}
|
||||
|
||||
std::shared_future<std::string> fetchFileAsync(const std::string &path, const std::string &proxy, int cache_ttl, bool async)
|
||||
{
|
||||
std::shared_future<std::string> retVal;
|
||||
/*if(vfs::vfs_exist(path))
|
||||
retVal = std::async(std::launch::async, [path](){return vfs::vfs_get(path);});
|
||||
else */if(fileExist(path, true))
|
||||
retVal = std::async(std::launch::async, [path](){return fileGet(path, true);});
|
||||
else if(isLink(path))
|
||||
retVal = std::async(std::launch::async, [path, proxy, cache_ttl](){return webGet(path, proxy, cache_ttl);});
|
||||
else
|
||||
return std::async(std::launch::async, [](){return std::string();});
|
||||
if(!async)
|
||||
retVal.wait();
|
||||
return retVal;
|
||||
}
|
||||
|
||||
std::string fetchFile(const std::string &path, const std::string &proxy, int cache_ttl)
|
||||
{
|
||||
return fetchFileAsync(path, proxy, cache_ttl, false).get();
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
#ifndef MULTITHREAD_H_INCLUDED
|
||||
#define MULTITHREAD_H_INCLUDED
|
||||
|
||||
#include <mutex>
|
||||
#include <future>
|
||||
|
||||
#include <yaml-cpp/yaml.h>
|
||||
|
||||
#include "../utils/ini_reader/ini_reader.h"
|
||||
#include "../utils/string.h"
|
||||
|
||||
using guarded_mutex = std::lock_guard<std::mutex>;
|
||||
|
||||
string_array safe_get_emojis();
|
||||
string_array safe_get_renames();
|
||||
string_array safe_get_streams();
|
||||
string_array safe_get_times();
|
||||
YAML::Node safe_get_clash_base();
|
||||
INIReader safe_get_mellow_base();
|
||||
void safe_set_emojis(string_array &data);
|
||||
void safe_set_renames(string_array &data);
|
||||
void safe_set_streams(string_array &data);
|
||||
void safe_set_times(string_array &data);
|
||||
std::shared_future<std::string> fetchFileAsync(const std::string &path, const std::string &proxy, int cache_ttl, bool async = false);
|
||||
std::string fetchFile(const std::string &path, const std::string &proxy, int cache_ttl);
|
||||
|
||||
#endif // MULTITHREAD_H_INCLUDED
|
||||
@@ -0,0 +1,117 @@
|
||||
#include <string>
|
||||
|
||||
#include "../utils/ini_reader/ini_reader.h"
|
||||
#include "../utils/logger.h"
|
||||
#include "../utils/rapidjson_extra.h"
|
||||
#include "../utils/system.h"
|
||||
#include "webget.h"
|
||||
|
||||
std::string buildGistData(std::string name, std::string content)
|
||||
{
|
||||
rapidjson::StringBuffer sb;
|
||||
rapidjson::Writer<rapidjson::StringBuffer> writer(sb);
|
||||
writer.StartObject();
|
||||
writer.Key("description");
|
||||
writer.String("subconverter");
|
||||
writer.Key("public");
|
||||
writer.Bool(false);
|
||||
writer.Key("files");
|
||||
writer.StartObject();
|
||||
writer.Key(name.data());
|
||||
writer.StartObject();
|
||||
writer.Key("content");
|
||||
writer.String(content.data());
|
||||
writer.EndObject();
|
||||
writer.EndObject();
|
||||
writer.EndObject();
|
||||
return sb.GetString();
|
||||
}
|
||||
|
||||
int uploadGist(std::string name, std::string path, std::string content, bool writeManageURL)
|
||||
{
|
||||
INIReader ini;
|
||||
rapidjson::Document json;
|
||||
std::string token, id, username, retData, url;
|
||||
int retVal = 0;
|
||||
|
||||
if(!fileExist("gistconf.ini"))
|
||||
{
|
||||
//std::cerr<<"gistconf.ini not found. Skipping...\n";
|
||||
writeLog(0, "gistconf.ini not found. Skipping...", LOG_LEVEL_ERROR);
|
||||
return -1;
|
||||
}
|
||||
|
||||
ini.ParseFile("gistconf.ini");
|
||||
if(ini.EnterSection("common") != 0)
|
||||
{
|
||||
//std::cerr<<"gistconf.ini has incorrect format. Skipping...\n";
|
||||
writeLog(0, "gistconf.ini has incorrect format. Skipping...", LOG_LEVEL_ERROR);
|
||||
return -1;
|
||||
}
|
||||
|
||||
token = ini.Get("token");
|
||||
if(!token.size())
|
||||
{
|
||||
//std::cerr<<"No token is provided. Skipping...\n";
|
||||
writeLog(0, "No token is provided. Skipping...", LOG_LEVEL_ERROR);
|
||||
return -1;
|
||||
}
|
||||
|
||||
id = ini.Get("id");
|
||||
username = ini.Get("username");
|
||||
if(!path.size())
|
||||
{
|
||||
if(ini.ItemExist("path"))
|
||||
path = ini.Get(name, "path");
|
||||
else
|
||||
path = name;
|
||||
}
|
||||
|
||||
if(!id.size())
|
||||
{
|
||||
//std::cerr<<"No gist id is provided. Creating new gist...\n";
|
||||
writeLog(0, "No Gist id is provided. Creating new Gist...", LOG_LEVEL_ERROR);
|
||||
retVal = webPost("https://api.github.com/gists", buildGistData(path, content), getSystemProxy(), {{"Authorization", "token " + token}}, &retData);
|
||||
if(retVal != 201)
|
||||
{
|
||||
//std::cerr<<"Create new Gist failed! Return data:\n"<<retData<<"\n";
|
||||
writeLog(0, "Create new Gist failed!\nReturn code: " + std::to_string(retVal) + "\nReturn data:\n" + retData, LOG_LEVEL_ERROR);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
url = "https://gist.githubusercontent.com/" + username + "/" + id + "/raw/" + path;
|
||||
//std::cerr<<"Gist id provided. Modifying Gist...\n";
|
||||
writeLog(0, "Gist id provided. Modifying Gist...", LOG_LEVEL_INFO);
|
||||
if(writeManageURL)
|
||||
content = "#!MANAGED-CONFIG " + url + "\n" + content;
|
||||
retVal = webPatch("https://api.github.com/gists/" + id, buildGistData(path, content), getSystemProxy(), {{"Authorization", "token " + token}}, &retData);
|
||||
if(retVal != 200)
|
||||
{
|
||||
//std::cerr<<"Modify gist failed! Return data:\n"<<retData<<"\n";
|
||||
writeLog(0, "Modify Gist failed!\nReturn code: " + std::to_string(retVal) + "\nReturn data:\n" + retData, LOG_LEVEL_ERROR);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
json.Parse(retData.data());
|
||||
GetMember(json, "id", id);
|
||||
if(json.HasMember("owner"))
|
||||
GetMember(json["owner"], "login", username);
|
||||
url = "https://gist.githubusercontent.com/" + username + "/" + id + "/raw/" + path;
|
||||
//std::cerr<<"Writing to Gist success!\nGenerator: "<<name<<"\nPath: "<<path<<"\nRaw URL: "<<url<<"\nGist owner: "<<username<<"\n";
|
||||
writeLog(0, "Writing to Gist success!\nGenerator: " + name + "\nPath: " + path + "\nRaw URL: " + url + "\nGist owner: " + username, LOG_LEVEL_INFO);
|
||||
|
||||
ini.EraseSection();
|
||||
ini.Set("token", token);
|
||||
ini.Set("id", id);
|
||||
ini.Set("username", username);
|
||||
|
||||
ini.SetCurrentSection(path);
|
||||
ini.EraseSection();
|
||||
ini.Set("type", name);
|
||||
ini.Set("url", url);
|
||||
|
||||
ini.ToFile("gistconf.ini");
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
#ifndef UPLOAD_H_INCLUDED
|
||||
#define UPLOAD_H_INCLUDED
|
||||
|
||||
#include <string>
|
||||
|
||||
std::string buildGistData(std::string name, std::string content);
|
||||
int uploadGist(std::string name, std::string path, std::string content, bool writeManageURL);
|
||||
|
||||
#endif // UPLOAD_H_INCLUDED
|
||||
@@ -0,0 +1,571 @@
|
||||
#include <iostream>
|
||||
#include <unistd.h>
|
||||
#include <sys/stat.h>
|
||||
//#include <mutex>
|
||||
#include <thread>
|
||||
#include <atomic>
|
||||
|
||||
#include <curl/curl.h>
|
||||
|
||||
#include "../utils/base64/base64.h"
|
||||
#include "../utils/defer.h"
|
||||
#include "../utils/file_extra.h"
|
||||
#include "../utils/logger.h"
|
||||
#include "../utils/urlencode.h"
|
||||
#include "../version.h"
|
||||
#include "webget.h"
|
||||
|
||||
#ifdef _WIN32
|
||||
#ifndef _stat
|
||||
#define _stat stat
|
||||
#endif // _stat
|
||||
#endif // _WIN32
|
||||
|
||||
extern bool gPrintDbgInfo, gServeCacheOnFetchFail;
|
||||
extern int gLogLevel;
|
||||
|
||||
/*
|
||||
using guarded_mutex = std::lock_guard<std::mutex>;
|
||||
std::mutex cache_rw_lock;
|
||||
*/
|
||||
|
||||
class RWLock
|
||||
{
|
||||
#define WRITE_LOCK_STATUS -1
|
||||
#define FREE_STATUS 0
|
||||
private:
|
||||
const std::thread::id NULL_THREAD;
|
||||
const bool WRITE_FIRST;
|
||||
std::thread::id m_write_thread_id;
|
||||
std::atomic_int m_lockCount;
|
||||
std::atomic_uint m_writeWaitCount;
|
||||
public:
|
||||
RWLock(const RWLock&) = delete;
|
||||
RWLock& operator=(const RWLock&) = delete;
|
||||
RWLock(bool writeFirst = true): WRITE_FIRST(writeFirst), m_write_thread_id(), m_lockCount(0), m_writeWaitCount(0) {}
|
||||
virtual ~RWLock() = default;
|
||||
int readLock()
|
||||
{
|
||||
if (std::this_thread::get_id() != m_write_thread_id)
|
||||
{
|
||||
int count;
|
||||
if (WRITE_FIRST)
|
||||
do {
|
||||
while ((count = m_lockCount) == WRITE_LOCK_STATUS || m_writeWaitCount > 0);
|
||||
} while (!m_lockCount.compare_exchange_weak(count, count + 1));
|
||||
else
|
||||
do {
|
||||
while ((count = m_lockCount) == WRITE_LOCK_STATUS);
|
||||
} while (!m_lockCount.compare_exchange_weak(count, count + 1));
|
||||
}
|
||||
return m_lockCount;
|
||||
}
|
||||
int readUnlock()
|
||||
{
|
||||
if (std::this_thread::get_id() != m_write_thread_id)
|
||||
--m_lockCount;
|
||||
return m_lockCount;
|
||||
}
|
||||
int writeLock()
|
||||
{
|
||||
if (std::this_thread::get_id() != m_write_thread_id)
|
||||
{
|
||||
++m_writeWaitCount;
|
||||
for (int zero = FREE_STATUS; !m_lockCount.compare_exchange_weak(zero, WRITE_LOCK_STATUS); zero = FREE_STATUS);
|
||||
--m_writeWaitCount;
|
||||
m_write_thread_id = std::this_thread::get_id();
|
||||
}
|
||||
return m_lockCount;
|
||||
}
|
||||
int writeUnlock()
|
||||
{
|
||||
if (std::this_thread::get_id() != m_write_thread_id)
|
||||
{
|
||||
throw std::runtime_error("writeLock/Unlock mismatch");
|
||||
}
|
||||
if (WRITE_LOCK_STATUS != m_lockCount)
|
||||
{
|
||||
throw std::runtime_error("RWLock internal error");
|
||||
}
|
||||
m_write_thread_id = NULL_THREAD;
|
||||
m_lockCount.store(FREE_STATUS);
|
||||
return m_lockCount;
|
||||
}
|
||||
};
|
||||
|
||||
RWLock cache_rw_lock;
|
||||
|
||||
long gMaxAllowedDownloadSize = 1048576L;
|
||||
|
||||
//std::string user_agent_str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36";
|
||||
std::string user_agent_str = "subconverter/" VERSION " cURL/" LIBCURL_VERSION;
|
||||
|
||||
struct curl_progress_data
|
||||
{
|
||||
long size_limit = 0L;
|
||||
};
|
||||
|
||||
static inline void curl_init()
|
||||
{
|
||||
static bool init = false;
|
||||
if(!init)
|
||||
{
|
||||
curl_global_init(CURL_GLOBAL_ALL);
|
||||
init = true;
|
||||
}
|
||||
}
|
||||
|
||||
static int writer(char *data, size_t size, size_t nmemb, std::string *writerData)
|
||||
{
|
||||
if(writerData == NULL)
|
||||
return 0;
|
||||
|
||||
writerData->append(data, size*nmemb);
|
||||
|
||||
return size * nmemb;
|
||||
}
|
||||
|
||||
static int dummy_writer(char *data, size_t size, size_t nmemb, void *writerData)
|
||||
{
|
||||
/// dummy writer, do not save anything
|
||||
(void)data;
|
||||
(void)writerData;
|
||||
return size * nmemb;
|
||||
}
|
||||
|
||||
static int size_checker(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
|
||||
{
|
||||
if(clientp)
|
||||
{
|
||||
curl_progress_data *data = reinterpret_cast<curl_progress_data*>(clientp);
|
||||
if(data->size_limit)
|
||||
{
|
||||
if(dlnow > data->size_limit)
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void curl_set_common_options(CURL *curl_handle, const char *url, curl_progress_data *data)
|
||||
{
|
||||
curl_easy_setopt(curl_handle, CURLOPT_URL, url);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, gLogLevel == LOG_LEVEL_VERBOSE ? 1L : 0L);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 0L);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_NOSIGNAL, 1L);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_MAXREDIRS, 20L);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 0L);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 0L);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, 15L);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, user_agent_str.data());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_COOKIEFILE, "");
|
||||
if(data)
|
||||
{
|
||||
if(data->size_limit)
|
||||
curl_easy_setopt(curl_handle, CURLOPT_MAXFILESIZE, data->size_limit);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_XFERINFOFUNCTION, size_checker);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_XFERINFODATA, data);
|
||||
}
|
||||
}
|
||||
|
||||
//static std::string curlGet(const std::string &url, const std::string &proxy, std::string &response_headers, CURLcode &return_code, const string_map &request_headers)
|
||||
static int curlGet(const FetchArgument &argument, FetchResult &result)
|
||||
{
|
||||
CURL *curl_handle;
|
||||
std::string *data = result.content, new_url = argument.url;
|
||||
struct curl_slist *list = NULL;
|
||||
defer(curl_slist_free_all(list);)
|
||||
long retVal = 0;
|
||||
|
||||
curl_init();
|
||||
|
||||
curl_handle = curl_easy_init();
|
||||
if(argument.proxy.size())
|
||||
{
|
||||
if(startsWith(argument.proxy, "cors:"))
|
||||
{
|
||||
list = curl_slist_append(list, "X-Requested-With: subconverter " VERSION);
|
||||
new_url = argument.proxy.substr(5) + argument.url;
|
||||
}
|
||||
else
|
||||
curl_easy_setopt(curl_handle, CURLOPT_PROXY, argument.proxy.data());
|
||||
}
|
||||
curl_progress_data limit;
|
||||
limit.size_limit = gMaxAllowedDownloadSize;
|
||||
curl_set_common_options(curl_handle, new_url.data(), &limit);
|
||||
list = curl_slist_append(list, "Content-Type: application/json;charset='utf-8'");
|
||||
if(argument.request_headers)
|
||||
{
|
||||
for(auto &x : *argument.request_headers)
|
||||
list = curl_slist_append(list, (x.first + ": " + x.second).data());
|
||||
}
|
||||
list = curl_slist_append(list, "SubConverter-Request: 1");
|
||||
list = curl_slist_append(list, "SubConverter-Version: " VERSION);
|
||||
if(list)
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list);
|
||||
|
||||
if(result.content)
|
||||
{
|
||||
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, writer);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, result.content);
|
||||
}
|
||||
else
|
||||
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, dummy_writer);
|
||||
if(result.response_headers)
|
||||
{
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, writer);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, result.response_headers);
|
||||
}
|
||||
else
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, dummy_writer);
|
||||
|
||||
if(argument.cookies)
|
||||
{
|
||||
string_array cookies = split(*argument.cookies, "\r\n");
|
||||
for(auto &x : cookies)
|
||||
curl_easy_setopt(curl_handle, CURLOPT_COOKIELIST, x.c_str());
|
||||
}
|
||||
|
||||
switch(argument.method)
|
||||
{
|
||||
case HTTP_POST:
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, argument.post_data.data());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE, argument.post_data.size());
|
||||
break;
|
||||
case HTTP_PATCH:
|
||||
curl_easy_setopt(curl_handle, CURLOPT_CUSTOMREQUEST, "PATCH");
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, argument.post_data.data());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE, argument.post_data.size());
|
||||
break;
|
||||
case HTTP_HEAD:
|
||||
curl_easy_setopt(curl_handle, CURLOPT_NOBODY, 1L);
|
||||
break;
|
||||
case HTTP_GET:
|
||||
break;
|
||||
}
|
||||
|
||||
unsigned int fail_count = 0, max_fails = 1;
|
||||
while(true)
|
||||
{
|
||||
*result.status_code = curl_easy_perform(curl_handle);
|
||||
if(*result.status_code == CURLE_OK || max_fails >= fail_count)
|
||||
break;
|
||||
else
|
||||
fail_count++;
|
||||
}
|
||||
|
||||
curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &retVal);
|
||||
|
||||
if(result.cookies)
|
||||
{
|
||||
curl_slist *cookies = nullptr;
|
||||
curl_easy_getinfo(curl_handle, CURLINFO_COOKIELIST, &cookies);
|
||||
if(cookies)
|
||||
{
|
||||
auto each = cookies;
|
||||
while(each)
|
||||
{
|
||||
result.cookies->append(each->data);
|
||||
*result.cookies += "\r\n";
|
||||
each = each->next;
|
||||
}
|
||||
}
|
||||
curl_slist_free_all(cookies);
|
||||
}
|
||||
|
||||
curl_easy_cleanup(curl_handle);
|
||||
|
||||
if(data)
|
||||
{
|
||||
if(*result.status_code != CURLE_OK || retVal != 200)
|
||||
data->clear();
|
||||
data->shrink_to_fit();
|
||||
}
|
||||
|
||||
return *result.status_code;
|
||||
}
|
||||
|
||||
// data:[<mediatype>][;base64],<data>
|
||||
static std::string dataGet(const std::string &url)
|
||||
{
|
||||
if (!startsWith(url, "data:"))
|
||||
return std::string();
|
||||
std::string::size_type comma = url.find(',');
|
||||
if (comma == std::string::npos || comma == url.size() - 1)
|
||||
return std::string();
|
||||
|
||||
std::string data = urlDecode(url.substr(comma + 1));
|
||||
if (endsWith(url.substr(0, comma), ";base64")) {
|
||||
return urlSafeBase64Decode(data);
|
||||
} else {
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
std::string buildSocks5ProxyString(const std::string &addr, int port, const std::string &username, const std::string &password)
|
||||
{
|
||||
std::string authstr = username.size() && password.size() ? username + ":" + password + "@" : "";
|
||||
std::string proxystr = "socks5://" + authstr + addr + ":" + std::to_string(port);
|
||||
return proxystr;
|
||||
}
|
||||
|
||||
std::string webGet(const std::string &url, const std::string &proxy, unsigned int cache_ttl, std::string *response_headers, string_icase_map *request_headers)
|
||||
{
|
||||
int return_code = 0;
|
||||
std::string content;
|
||||
|
||||
FetchArgument argument {HTTP_GET, url, proxy, "", request_headers, nullptr, cache_ttl};
|
||||
FetchResult fetch_res {&return_code, &content, response_headers, nullptr};
|
||||
|
||||
if (startsWith(url, "data:"))
|
||||
return dataGet(url);
|
||||
// cache system
|
||||
if(cache_ttl > 0)
|
||||
{
|
||||
md("cache");
|
||||
const std::string url_md5 = getMD5(url);
|
||||
const std::string path = "cache/" + url_md5, path_header = path + "_header";
|
||||
struct stat result;
|
||||
if(stat(path.data(), &result) == 0) // cache exist
|
||||
{
|
||||
time_t mtime = result.st_mtime, now = time(NULL); // get cache modified time and current time
|
||||
if(difftime(now, mtime) <= cache_ttl) // within TTL
|
||||
{
|
||||
writeLog(0, "CACHE HIT: '" + url + "', using local cache.");
|
||||
//guarded_mutex guard(cache_rw_lock);
|
||||
cache_rw_lock.readLock();
|
||||
defer(cache_rw_lock.readUnlock();)
|
||||
if(response_headers)
|
||||
*response_headers = fileGet(path_header, true);
|
||||
return fileGet(path, true);
|
||||
}
|
||||
writeLog(0, "CACHE MISS: '" + url + "', TTL timeout, creating new cache."); // out of TTL
|
||||
}
|
||||
else
|
||||
writeLog(0, "CACHE NOT EXIST: '" + url + "', creating new cache.");
|
||||
//content = curlGet(url, proxy, response_headers, return_code); // try to fetch data
|
||||
curlGet(argument, fetch_res);
|
||||
if(return_code == CURLE_OK) // success, save new cache
|
||||
{
|
||||
//guarded_mutex guard(cache_rw_lock);
|
||||
cache_rw_lock.writeLock();
|
||||
defer(cache_rw_lock.writeUnlock();)
|
||||
fileWrite(path, content, true);
|
||||
if(response_headers)
|
||||
fileWrite(path_header, *response_headers, true);
|
||||
}
|
||||
else
|
||||
{
|
||||
if(fileExist(path) && gServeCacheOnFetchFail) // failed, check if cache exist
|
||||
{
|
||||
writeLog(0, "Fetch failed. Serving cached content."); // cache exist, serving cache
|
||||
//guarded_mutex guard(cache_rw_lock);
|
||||
cache_rw_lock.readLock();
|
||||
defer(cache_rw_lock.readUnlock();)
|
||||
content = fileGet(path, true);
|
||||
if(response_headers)
|
||||
*response_headers = fileGet(path_header, true);
|
||||
}
|
||||
else
|
||||
writeLog(0, "Fetch failed. No local cache available."); // cache not exist or not allow to serve cache, serving nothing
|
||||
}
|
||||
return content;
|
||||
}
|
||||
//return curlGet(url, proxy, response_headers, return_code);
|
||||
curlGet(argument, fetch_res);
|
||||
return content;
|
||||
}
|
||||
|
||||
void flushCache()
|
||||
{
|
||||
//guarded_mutex guard(cache_rw_lock);
|
||||
cache_rw_lock.writeLock();
|
||||
defer(cache_rw_lock.writeUnlock();)
|
||||
operateFiles("cache", [](const std::string &file){ remove(("cache/" + file).data()); return 0; });
|
||||
}
|
||||
|
||||
int curlPost(const std::string &url, const std::string &data, const std::string &proxy, const string_array &request_headers, std::string *retData)
|
||||
{
|
||||
CURL *curl_handle;
|
||||
CURLcode res;
|
||||
struct curl_slist *list = NULL;
|
||||
long retVal = 0;
|
||||
|
||||
curl_init();
|
||||
curl_handle = curl_easy_init();
|
||||
list = curl_slist_append(list, "Content-Type: application/json;charset='utf-8'");
|
||||
for(const std::string &x : request_headers)
|
||||
list = curl_slist_append(list, x.data());
|
||||
|
||||
curl_progress_data limit;
|
||||
curl_set_common_options(curl_handle, url.data(), &limit);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, data.data());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE, data.size());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, writer);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, retData);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list);
|
||||
if(proxy.size())
|
||||
curl_easy_setopt(curl_handle, CURLOPT_PROXY, proxy.data());
|
||||
|
||||
res = curl_easy_perform(curl_handle);
|
||||
curl_slist_free_all(list);
|
||||
|
||||
if(res == CURLE_OK)
|
||||
{
|
||||
curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &retVal);
|
||||
}
|
||||
|
||||
curl_easy_cleanup(curl_handle);
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
int webPost(const std::string &url, const std::string &data, const std::string &proxy, const string_icase_map &request_headers, std::string *retData)
|
||||
{
|
||||
//return curlPost(url, data, proxy, request_headers, retData);
|
||||
int return_code = 0;
|
||||
FetchArgument argument {HTTP_POST, url, proxy, "", &request_headers, nullptr, 0};
|
||||
FetchResult fetch_res {&return_code, retData, nullptr, nullptr};
|
||||
return webGet(argument, fetch_res);
|
||||
}
|
||||
|
||||
int curlPatch(const std::string &url, const std::string &data, const std::string &proxy, const string_array &request_headers, std::string *retData)
|
||||
{
|
||||
CURL *curl_handle;
|
||||
CURLcode res;
|
||||
long retVal = 0;
|
||||
struct curl_slist *list = NULL;
|
||||
|
||||
curl_init();
|
||||
|
||||
curl_handle = curl_easy_init();
|
||||
|
||||
list = curl_slist_append(list, "Content-Type: application/json;charset='utf-8'");
|
||||
for(const std::string &x : request_headers)
|
||||
list = curl_slist_append(list, x.data());
|
||||
|
||||
curl_progress_data limit;
|
||||
curl_set_common_options(curl_handle, url.data(), &limit);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_CUSTOMREQUEST, "PATCH");
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, data.data());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE, data.size());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, writer);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, retData);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list);
|
||||
if(proxy.size())
|
||||
curl_easy_setopt(curl_handle, CURLOPT_PROXY, proxy.data());
|
||||
|
||||
res = curl_easy_perform(curl_handle);
|
||||
curl_slist_free_all(list);
|
||||
if(res == CURLE_OK)
|
||||
{
|
||||
res = curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &retVal);
|
||||
}
|
||||
|
||||
curl_easy_cleanup(curl_handle);
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
int webPatch(const std::string &url, const std::string &data, const std::string &proxy, const string_icase_map &request_headers, std::string *retData)
|
||||
{
|
||||
//return curlPatch(url, data, proxy, request_headers, retData);
|
||||
int return_code = 0;
|
||||
FetchArgument argument {HTTP_PATCH, url, proxy, "", &request_headers, nullptr, 0};
|
||||
FetchResult fetch_res {&return_code, retData, nullptr, nullptr};
|
||||
return webGet(argument, fetch_res);
|
||||
}
|
||||
|
||||
int curlHead(const std::string &url, const std::string &proxy, const string_array &request_headers, std::string &response_headers)
|
||||
{
|
||||
CURL *curl_handle;
|
||||
CURLcode res;
|
||||
long retVal = 0;
|
||||
struct curl_slist *list = NULL;
|
||||
|
||||
curl_init();
|
||||
|
||||
curl_handle = curl_easy_init();
|
||||
|
||||
list = curl_slist_append(list, "Content-Type: application/json;charset='utf-8'");
|
||||
for(const std::string &x : request_headers)
|
||||
list = curl_slist_append(list, x.data());
|
||||
|
||||
curl_progress_data limit;
|
||||
curl_set_common_options(curl_handle, url.data(), &limit);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, writer);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, &response_headers);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_NOBODY, 1L);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list);
|
||||
if(proxy.size())
|
||||
curl_easy_setopt(curl_handle, CURLOPT_PROXY, proxy.data());
|
||||
|
||||
res = curl_easy_perform(curl_handle);
|
||||
curl_slist_free_all(list);
|
||||
if(res == CURLE_OK)
|
||||
res = curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &retVal);
|
||||
|
||||
curl_easy_cleanup(curl_handle);
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
int webHead(const std::string &url, const std::string &proxy, const string_icase_map &request_headers, std::string &response_headers)
|
||||
{
|
||||
//return curlHead(url, proxy, request_headers, response_headers);
|
||||
int return_code = 0;
|
||||
FetchArgument argument {HTTP_HEAD, url, proxy, "", &request_headers, nullptr, 0};
|
||||
FetchResult fetch_res {&return_code, nullptr, &response_headers, nullptr};
|
||||
return webGet(argument, fetch_res);
|
||||
}
|
||||
|
||||
string_array headers_map_to_array(const string_map &headers)
|
||||
{
|
||||
string_array result;
|
||||
for(auto &kv : headers)
|
||||
result.push_back(kv.first + ": " + kv.second);
|
||||
return result;
|
||||
}
|
||||
|
||||
int webGet(const FetchArgument& argument, FetchResult &result)
|
||||
{
|
||||
return curlGet(argument, result);
|
||||
/*
|
||||
switch(argument.method)
|
||||
{
|
||||
case HTTP_GET:
|
||||
return curlGet(argument, result);
|
||||
case HTTP_POST:
|
||||
{
|
||||
string_array request_header;
|
||||
if(argument.request_headers != nullptr)
|
||||
request_header = headers_map_to_array(*argument.request_headers);
|
||||
int res = curlPost(argument.url, argument.post_data, argument.proxy, request_header, result.content);
|
||||
*result.status_code = res;
|
||||
return res;
|
||||
}
|
||||
case HTTP_PATCH:
|
||||
{
|
||||
string_array request_header;
|
||||
if(argument.request_headers != nullptr)
|
||||
request_header = headers_map_to_array(*argument.request_headers);
|
||||
int res = curlPatch(argument.url, argument.post_data, argument.proxy, request_header, result.content);
|
||||
*result.status_code = res;
|
||||
return res;
|
||||
}
|
||||
case HTTP_HEAD:
|
||||
{
|
||||
string_array request_header;
|
||||
if(argument.request_headers != nullptr)
|
||||
request_header = headers_map_to_array(*argument.request_headers);
|
||||
int res = curlHead(argument.url, argument.proxy, request_header, *result.response_headers);
|
||||
*result.status_code = res;
|
||||
return res;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
*/
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
#ifndef WEBGET_H_INCLUDED
|
||||
#define WEBGET_H_INCLUDED
|
||||
|
||||
#include <string>
|
||||
#include <map>
|
||||
|
||||
#include "../utils/map_extra.h"
|
||||
#include "../utils/string.h"
|
||||
|
||||
enum http_method
|
||||
{
|
||||
HTTP_GET,
|
||||
HTTP_HEAD,
|
||||
HTTP_POST,
|
||||
HTTP_PATCH
|
||||
};
|
||||
|
||||
struct FetchArgument
|
||||
{
|
||||
const http_method method;
|
||||
const std::string url;
|
||||
const std::string proxy;
|
||||
const std::string post_data;
|
||||
const string_icase_map *request_headers = nullptr;
|
||||
std::string *cookies = nullptr;
|
||||
const unsigned int cache_ttl = 0;
|
||||
};
|
||||
|
||||
struct FetchResult
|
||||
{
|
||||
int *status_code;
|
||||
std::string *content = nullptr;
|
||||
std::string *response_headers = nullptr;
|
||||
std::string *cookies = nullptr;
|
||||
};
|
||||
|
||||
int webGet(const FetchArgument& argument, FetchResult &result);
|
||||
std::string webGet(const std::string &url, const std::string &proxy = "", unsigned int cache_ttl = 0, std::string *response_headers = nullptr, string_icase_map *request_headers = nullptr);
|
||||
void flushCache();
|
||||
int webPost(const std::string &url, const std::string &data, const std::string &proxy, const string_icase_map &request_headers, std::string *retData);
|
||||
int webPatch(const std::string &url, const std::string &data, const std::string &proxy, const string_icase_map &request_headers, std::string *retData);
|
||||
std::string buildSocks5ProxyString(const std::string &addr, int port, const std::string &username, const std::string &password);
|
||||
|
||||
// Unimplemented: (CURLOPT_HTTPHEADER: Host:)
|
||||
std::string httpGet(const std::string &host, const std::string &addr, const std::string &uri);
|
||||
std::string httpsGet(const std::string &host, const std::string &addr, const std::string &uri);
|
||||
|
||||
#endif // WEBGET_H_INCLUDED
|
||||
Reference in new issue
Block a user