diff --git a/Makefile b/Makefile index dbf53e2..8218d2e 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,12 @@ freshclam_LIB += clamscan/freshclam/freshclam.o clamscan/freshclam/notify.o clam cJSON_CFLAGS += -std=c89 -c -fPIC -pedantic -Wall -Werror -Wstrict-prototypes -Wwrite-strings -Wshadow -Winit-self -Wcast-align -Wformat=2 -Wmissing-prototypes -Wstrict-overflow=2 -Wcast-qual -Wc++-compat -Wundef -Wswitch-default -Wconversion -fstack-protector cJSON_LIB += cJSON.o +ip2region_CFLAGS += -Os -g -Wall -I/ip2region -c +ip2region_LIB += ip2region.o xdb_searcher.o + all: conf.o rhost.o libiptc.o ccronexpr.o + $(CC) $(ip2region_CFLAGS) ip2region/ip2region.c + $(CC) $(ip2region_CFLAGS) ip2region/xdb_searcher.c $(CC) $(cJSON_CFLAGS) cJSON/cJSON.c $(CC) $(freshclam_CFLAGS) -c clamscan/freshclam/freshclam.c -fPIC -DPIC -o clamscan/freshclam/freshclam.o $(CC) $(freshclam_CFLAGS) -c clamscan/freshclam/notify.c -fPIC -DPIC -o clamscan/freshclam/notify.o @@ -44,7 +49,7 @@ all: conf.o rhost.o libiptc.o ccronexpr.o $(CC) $(CLAMSCAN_CFLAGS) -c clamscan/clamscan.c -o clamscan/clamscan.o $(CC) $(CLAMSCAN_CFLAGS) -c clamscan/manager.c -o clamscan/manager.o - $(CC) $(CFLAGS) $^ -o $(OBG) $(cJSON_LIB) $(LIB) $(freshclam_LIB) + $(CC) $(CFLAGS) $^ -o $(OBG) $(cJSON_LIB) $(ip2region_LIB) $(LIB) $(freshclam_LIB) chmod +x $(OBG) @@ -61,4 +66,6 @@ clean: rm -rf clamscan/freshclam/*.o rm -rf `find clamscan/libclamav/ -name *.lo` rm -rf `find clamscan/libclamav/ -name *.o` + rm -rf cJSON/*.o + rm -rf ip2region/*.o rm -rf $(OBG) \ No newline at end of file diff --git a/cJSON/cJSON.o b/cJSON/cJSON.o deleted file mode 100644 index ff4b423..0000000 Binary files a/cJSON/cJSON.o and /dev/null differ diff --git a/ip2region/ip2region.c b/ip2region/ip2region.c new file mode 100644 index 0000000..17f9f67 --- /dev/null +++ b/ip2region/ip2region.c @@ -0,0 +1,45 @@ +#include +#include "xdb_searcher.h" +#include "ip2region.h" + +int ip2region(char *xdb_file, char *ip) +{ + char *db_path = xdb_file; + xdb_vector_index_t *v_index; + xdb_searcher_t searcher; + char region_buffer[256], ip_buffer[16]; + long s_time; + + // 1、从 db_path 加载 VectorIndex 索引。 + // 得到 v_index 做成全局缓存，便于后续反复使用。 + // 注意：v_index 不需要每次都加载，建议在服务启动的时候加载一次，然后做成全局资源。 + v_index = xdb_load_vector_index_from_file(db_path); + if (v_index == NULL) { + printf("failed to load vector index from `%s`\n", db_path); + return 1; + } + + // 2、使用全局的 VectorIndex 变量创建带 VectorIndex 缓存的 xdb 查询对象 + int err = xdb_new_with_vector_index(&searcher, db_path, v_index); + if (err != 0) { + printf("failed to create vector index cached searcher with errcode=%d\n", err); + return 2; + } + + // 3、调用 search API 查询 + // 得到的 region 信息会存储到 region_buffer 里面，如果你自定义了数据，请确保给足 buffer 的空间。 + s_time = xdb_now(); + err = xdb_search_by_string(&searcher, ip, region_buffer, sizeof(region_buffer)); + if (err != 0) { + printf("failed search(%s) with errno=%d\n", ip, err); + } else { + printf("{region: %s, took: %d μs}", region_buffer, (int)(xdb_now() - s_time)); + } + + // 备注：并发使用，没一个线程需要单独定义并且初始化一个 searcher 查询对象。 + + // 4、关闭 xdb 查询器，如果是要关闭服务，也需要释放 v_index 的内存。 + xdb_close(&searcher); + xdb_close_vector_index(v_index); + return 0; +} diff --git a/ip2region/ip2region.h b/ip2region/ip2region.h new file mode 100644 index 0000000..f8305d6 --- /dev/null +++ b/ip2region/ip2region.h @@ -0,0 +1,8 @@ +#ifndef IP2REGION_H +#define IP2REGION_H + + +int ip2region(char *xdb_file, char *ip); + + +#endif diff --git a/ip2region/ip2region.xdb b/ip2region/ip2region.xdb new file mode 100644 index 0000000..c78b792 Binary files /dev/null and b/ip2region/ip2region.xdb differ diff --git a/ip2region/xdb_searcher.c b/ip2region/xdb_searcher.c new file mode 100644 index 0000000..6bcae3a --- /dev/null +++ b/ip2region/xdb_searcher.c @@ -0,0 +1,402 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +// --- +// @Author Lion +// @Date 2022/06/27 + +#include "sys/time.h" +#include "xdb_searcher.h" + +// internal function prototype define +XDB_PRIVATE(int) read(xdb_searcher_t *, long offset, char *, size_t length); + +XDB_PRIVATE(int) xdb_new_base(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index, const xdb_content_t *c_buffer) { + memset(xdb, 0x00, sizeof(xdb_searcher_t)); + + // check the content buffer first + if (c_buffer != NULL) { + xdb->v_index = NULL; + xdb->content = c_buffer; + return 0; + } + + // open the xdb binary file + FILE *handle = fopen(db_path, "rb"); + if (handle == NULL) { + return 1; + } + + xdb->handle = handle; + xdb->v_index = v_index; + + return 0; +} + +// xdb searcher new api define +XDB_PUBLIC(int) xdb_new_with_file_only(xdb_searcher_t *xdb, const char *db_path) { + return xdb_new_base(xdb, db_path, NULL, NULL); +} + +XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index) { + return xdb_new_base(xdb, db_path, v_index, NULL); +} + +XDB_PUBLIC(int) xdb_new_with_buffer(xdb_searcher_t *xdb, const xdb_content_t *c_buffer) { + return xdb_new_base(xdb, NULL, NULL, c_buffer); +} + +XDB_PUBLIC(void) xdb_close(void *ptr) { + xdb_searcher_t *xdb = (xdb_searcher_t *) ptr; + if (xdb->handle != NULL) { + fclose(xdb->handle); + xdb->handle = NULL; + } +} + +// --- xdb searcher search api define + +XDB_PUBLIC(int) xdb_search_by_string(xdb_searcher_t *xdb, const char *str_ip, char *region_buffer, size_t length) { + unsigned int ip = 0; + int errcode = xdb_check_ip(str_ip, &ip); + if (errcode != 0) { + return 10 + errcode; + } else { + return xdb_search(xdb, ip, region_buffer, length); + } +} + +XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_buffer, size_t length) { + int il0, il1, idx, err, l, h, m, data_len; + unsigned int s_ptr, e_ptr, p, sip, eip, data_ptr; + char vector_buffer[xdb_vector_index_size], segment_buffer[xdb_segment_index_size]; + + // reset the io counter + xdb->io_count = 0; + + // locate the segment index block based on the vector index + il0 = ((int) (ip >> 24)) & 0xFF; + il1 = ((int) (ip >> 16)) & 0xFF; + idx = il0 * xdb_vector_index_cols * xdb_vector_index_size + il1 * xdb_vector_index_size; + if (xdb->v_index != NULL) { + s_ptr = xdb_get_uint(xdb->v_index->buffer, idx); + e_ptr = xdb_get_uint(xdb->v_index->buffer, idx + 4); + } else if (xdb->content != NULL) { + s_ptr = xdb_get_uint(xdb->content->buffer, xdb_header_info_length + idx); + e_ptr = xdb_get_uint(xdb->content->buffer, xdb_header_info_length + idx + 4); + } else { + err = read(xdb, xdb_header_info_length + idx, vector_buffer, sizeof(vector_buffer)); + if (err != 0) { + return 10 + err; + } + + s_ptr = xdb_get_uint(vector_buffer, 0); + e_ptr = xdb_get_uint(vector_buffer, 4); + } + + // printf("s_ptr=%u, e_ptr=%u\n", s_ptr, e_ptr); + // binary search to get the final region info + data_len = 0, data_ptr = 0; + l = 0, h = ((int) (e_ptr - s_ptr)) / xdb_segment_index_size; + while (l <= h) { + m = (l + h) >> 1; + p = s_ptr + m * xdb_segment_index_size; + + // read the segment index item + err = read(xdb, p, segment_buffer, sizeof(segment_buffer)); + if (err != 0) { + return 20 + err; + } + + // decode the data fields as needed + sip = xdb_get_uint(segment_buffer, 0); + if (ip < sip) { + h = m - 1; + } else { + eip = xdb_get_uint(segment_buffer, 4); + if (ip > eip) { + l = m + 1; + } else { + data_len = xdb_get_ushort(segment_buffer, 8); + data_ptr = xdb_get_uint(segment_buffer, 10); + break; + } + } + } + + // printf("data_len=%u, data_ptr=%u\n", data_len, data_ptr); + if (data_len == 0) { + region_buffer[0] = '\0'; + return 0; + } + + // buffer length checking + if (data_len >= length) { + return 1; + } + + err = read(xdb, data_ptr, region_buffer, data_len); + if (err != 0) { + return 30 + err; + } + + // auto append a NULL-end + region_buffer[data_len] = '\0'; + return 0; +} + +XDB_PRIVATE(int) read(xdb_searcher_t *xdb, long offset, char *buffer, size_t length) { + // check the xdb content cache first + if (xdb->content != NULL) { + memcpy(buffer, xdb->content->buffer + offset, length); + return 0; + } + + // seek to the offset + if (fseek(xdb->handle, offset, SEEK_SET) == -1) { + return 1; + } + + xdb->io_count++; + if (fread(buffer, 1, length, xdb->handle) != length) { + return 2; + } + + return 0; +} + +XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *xdb) { + return xdb->io_count; +} + + +// --- buffer load util functions + +XDB_PUBLIC(xdb_header_t *) xdb_load_header(FILE *handle) { + xdb_header_t *header; + unsigned int size = xdb_header_info_length; + + // entry alloc + header = (xdb_header_t *) xdb_malloc(sizeof(xdb_header_t)); + if (header == NULL) { + return NULL; + } + + if (fseek(handle, 0, SEEK_SET) == -1) { + xdb_free(header); + return NULL; + } + + if (fread(header->buffer, 1,size, handle) != size) { + xdb_free(header); + return NULL; + } + + // fill the fields + header->length = size; + header->version = (unsigned short) xdb_get_ushort(header->buffer, 0); + header->index_policy = (unsigned short) xdb_get_ushort(header->buffer, 2); + header->created_at = xdb_get_uint(header->buffer, 4); + header->start_index_ptr = xdb_get_uint(header->buffer, 8); + header->end_index_ptr = xdb_get_uint(header->buffer,12); + + return header; +} + +XDB_PUBLIC(xdb_header_t *) xdb_load_header_from_file(const char *db_path) { + xdb_header_t *header; + FILE *handle = fopen(db_path, "rb"); + if (handle == NULL) { + return NULL; + } + + header = xdb_load_header(handle); + fclose(handle); + return header; +} + +XDB_PUBLIC(void) xdb_close_header(void *ptr) { + xdb_header_t *header = (xdb_header_t *) ptr; + if (header->length > 0) { + header->length = 0; + xdb_free(header); + } +} + +// --- vector index + +XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index(FILE *handle) { + xdb_vector_index_t *v_index; + unsigned int size = xdb_vector_index_length; + + // seek to the vector index offset + if (fseek(handle, xdb_header_info_length, SEEK_SET) == -1) { + return NULL; + } + + // do the buffer read + v_index = (xdb_vector_index_t *) xdb_malloc(sizeof(xdb_vector_index_t)); + if (v_index == NULL) { + return NULL; + } + + v_index->length = size; + if (fread(v_index->buffer, 1, size, handle) != size) { + xdb_free(v_index); + return NULL; + } + + return v_index; +} + +XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index_from_file(const char *db_path) { + xdb_vector_index_t *v_index; + FILE *handle = fopen(db_path, "rb"); + if (handle == NULL) { + return NULL; + } + + v_index = xdb_load_vector_index(handle); + fclose(handle); + return v_index; +} + +XDB_PUBLIC(void) xdb_close_vector_index(void *ptr) { + xdb_vector_index_t *v_index = (xdb_vector_index_t *) ptr; + if (v_index->length > 0) { + v_index->length = 0; + xdb_free(v_index); + } +} + +// --- content buffer + +XDB_PUBLIC(xdb_content_t *) xdb_load_content(FILE *handle) { + unsigned int size; + xdb_content_t *content; + + // determine the file size + if (fseek(handle, 0, SEEK_END) == -1) { + return NULL; + } + + size = (unsigned int) ftell(handle); + if (fseek(handle, 0, SEEK_SET) == -1) { + return NULL; + } + + // do the file read + content = (xdb_content_t *) xdb_malloc(sizeof(xdb_content_t)); + if (content == NULL) { + return NULL; + } + + // do the buffer alloc + content->buffer = (char *) xdb_malloc(size); + if (content->buffer == NULL) { + xdb_free(content); + return NULL; + } + + // read the content into the buffer + content->length = size; + if (fread(content->buffer, 1, size, handle) != size) { + xdb_free(content); + return NULL; + } + + return content; +} + +XDB_PUBLIC(xdb_content_t *) xdb_load_content_from_file(const char *db_path) { + xdb_content_t *content; + FILE *handle = fopen(db_path, "rb"); + if (handle == NULL) { + return NULL; + } + + content = xdb_load_content(handle); + fclose(handle); + return content; +} + +XDB_PUBLIC(void) xdb_close_content(void *ptr) { + xdb_content_t *content = (xdb_content_t *) ptr; + if (content->length > 0) { + content->length = 0; + xdb_free(content->buffer); + content->buffer = NULL; + xdb_free(content); + } +} + +// --- End + +// get unsigned long (4bytes) from a specified buffer start from the specified offset +XDB_PUBLIC(unsigned int) xdb_get_uint(const char *buffer, int offset) { + return ( + ((buffer[offset ]) & 0x000000FF) | + ((buffer[offset+1] << 8) & 0x0000FF00) | + ((buffer[offset+2] << 16) & 0x00FF0000) | + ((buffer[offset+3] << 24) & 0xFF000000) + ); +} + +// get unsigned short (2bytes) from a specified buffer start from the specified offset +XDB_PUBLIC(int) xdb_get_ushort(const char *buffer, int offset) { + return ( + ((buffer[offset ]) & 0x000000FF) | + ((buffer[offset+1] << 8) & 0x0000FF00) + ); +} + +// string ip to unsigned int +static int shiftIndex[4] = {24, 16, 8, 0}; +XDB_PUBLIC(int) xdb_check_ip(const char *src_ip, unsigned int *dst_ip) { + char c; + int i, n, ip = 0; + const char *ptr = src_ip; + for (i = 0; i < 4; i++) { + n = 0; + while (1) { + c = *ptr; + ptr++; + if (c >= '0' && c <= '9') { + n *= 10; + n += c - '0'; + } else if ((i < 3 && c == '.') || i == 3) { + // stopping at the '.' but ignore the tailing chars + // after the 3rd one (auto clean the tailing none-integer ?). + break; + } else { + return 1; + } + } + + if (n > 0xFF) { + return 2; + } + + ip |= (n << shiftIndex[i]); + } + + *dst_ip = ip; + return 0; +} + +// unsigned int ip to string ip +XDB_PUBLIC(void) xdb_long2ip(unsigned int ip, char *buffer) { + sprintf(buffer, "%d.%d.%d.%d", (ip >> 24) & 0xFF, (ip >> 16) & 0xFF, (ip >> 8) & 0xFF, ip & 0xFF); +} + +// get the middle ip of a and b +XDB_PUBLIC(unsigned int) xdb_mip(unsigned long a, unsigned long b) { + return (unsigned int) ((a + b) >> 1); +} + +XDB_PUBLIC(long) xdb_now() { + struct timeval c_time; + gettimeofday(&c_time, NULL); + return c_time.tv_sec * (int)1e6 + c_time.tv_usec; +} diff --git a/ip2region/xdb_searcher.h b/ip2region/xdb_searcher.h new file mode 100644 index 0000000..12808bc --- /dev/null +++ b/ip2region/xdb_searcher.h @@ -0,0 +1,149 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +// --- +// @Author Lion +// @Date 2022/06/27 + +#ifndef C_XDB_SEARCHER_H +#define C_XDB_SEARCHER_H + +#include +#include +#include + +#if ( defined(WIN32) || defined(_WIN32) || defined(__WINDOWS_) || defined(WINNT) ) +# define XDB_PUBLIC(type) extern __declspec(dllexport) type +# define XDB_PRIVATE(type) static type +# define XDB_WINDOWS +#elif ( defined(linux) || defined(_UNIX) ) +# define XDB_PUBLIC(type) extern type +# define XDB_PRIVATE(type) static inline type +# define XDB_LINUX +#endif + +#define xdb_calloc( _blocks, _bytes ) calloc( _blocks, _bytes ) +#define xdb_malloc( _bytes ) malloc( _bytes ) +#define xdb_free( _ptr ) free( _ptr ) + +// public constants define +#define xdb_header_info_length 256 +#define xdb_vector_index_rows 256 +#define xdb_vector_index_cols 256 +#define xdb_vector_index_size 8 +#define xdb_segment_index_size 14 + +// cache of vector_index_row × vector_index_rows × vector_index_size +#define xdb_vector_index_length 524288 + + +// --- buffer load util functions + +// use the following buffer struct to wrap the binary buffer data +// since the buffer data could not be operated with the string API. +struct xdb_header { + unsigned short version; + unsigned short index_policy; + unsigned int created_at; + unsigned int start_index_ptr; + unsigned int end_index_ptr; + + // the original buffer + unsigned int length; + char buffer[xdb_header_info_length]; +}; +typedef struct xdb_header xdb_header_t; + +XDB_PUBLIC(xdb_header_t *) xdb_load_header(FILE *); + +XDB_PUBLIC(xdb_header_t *) xdb_load_header_from_file(const char *); + +XDB_PUBLIC(void) xdb_close_header(void *); + + +// --- vector index buffer +struct xdb_vector_index { + unsigned int length; + char buffer[xdb_vector_index_length]; +}; +typedef struct xdb_vector_index xdb_vector_index_t; + +XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index(FILE *); + +XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index_from_file(const char *); + +XDB_PUBLIC(void) xdb_close_vector_index(void *); + + +// --- content buffer +struct xdb_content { + unsigned int length; + char *buffer; +}; +typedef struct xdb_content xdb_content_t; + +XDB_PUBLIC(xdb_content_t *) xdb_load_content(FILE *); + +XDB_PUBLIC(xdb_content_t *) xdb_load_content_from_file(const char *); + +XDB_PUBLIC(void) xdb_close_content(void *); + +// --- End buffer load + +// xdb searcher structure +struct xdb_searcher_entry { + FILE *handle; + + // header info + const char *header; + int io_count; + + // vector index buffer cache. + // preload the vector index will reduce the number of IO operations + // thus speedup the search process. + const xdb_vector_index_t *v_index; + + // content buffer. + // cache the whole xdb content. + const xdb_content_t *content; +}; +typedef struct xdb_searcher_entry xdb_searcher_t; + +// xdb searcher new api define +XDB_PUBLIC(int) xdb_new_with_file_only(xdb_searcher_t *, const char *); + +XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_searcher_t *, const char *, const xdb_vector_index_t *); + +XDB_PUBLIC(int) xdb_new_with_buffer(xdb_searcher_t *, const xdb_content_t *); + +XDB_PUBLIC(void) xdb_close(void *); + +// xdb searcher search api define +XDB_PUBLIC(int) xdb_search_by_string(xdb_searcher_t *, const char *, char *, size_t); + +XDB_PUBLIC(int) xdb_search(xdb_searcher_t *, unsigned int, char *, size_t); + +XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *); + + +// get unsigned long (4bytes) from a specified buffer start from the specified offset with little-endian +XDB_PUBLIC(unsigned int) xdb_get_uint(const char *, int); + +// get unsigned short (2bytes) from a specified buffer start from the specified offset with little-endian +XDB_PUBLIC(int) xdb_get_ushort(const char *, int); + +// check the specified string ip and convert it to an unsigned int +XDB_PUBLIC(int) xdb_check_ip(const char *, unsigned int *); + +// unsigned int ip to string ip +XDB_PUBLIC(void) xdb_long2ip(unsigned int, char *); + +// get the middle ip of a and b +XDB_PUBLIC(unsigned int) xdb_mip(unsigned long, unsigned long); + +// get the current time in microseconds +XDB_PUBLIC(long) xdb_now(); + + +#endif //C_XDB_SEARCHER_H diff --git a/rhost.c b/rhost.c index 20ad595..d13c144 100644 --- a/rhost.c +++ b/rhost.c @@ -913,6 +913,9 @@ int main(int argc, char *argv[], char **env) signal(SIGCHLD, sig_child); // 创建捕捉子进程退出信号 + // ip2region 离线IP地址定位库 + //ip2region("ip2region/ip2region.xdb", "1.1.1.1"); + int pid; int i; diff --git a/rhost.h b/rhost.h index 88e36d0..71bcfb5 100644 --- a/rhost.h +++ b/rhost.h @@ -18,6 +18,7 @@ #include #include "./cJSON/cJSON.h" +#include "ip2region/ip2region.h" typedef struct now_next_time