增加ip2region离线IP地址定位库，测试阶段未使用

Makefile

@@ -17,7 +17,12 @@ freshclam_LIB += clamscan/freshclam/freshclam.o clamscan/freshclam/notify.o clam
 cJSON_CFLAGS += -std=c89 -c -fPIC -pedantic -Wall -Werror -Wstrict-prototypes -Wwrite-strings -Wshadow -Winit-self -Wcast-align -Wformat=2 -Wmissing-prototypes -Wstrict-overflow=2 -Wcast-qual -Wc++-compat -Wundef -Wswitch-default -Wconversion -fstack-protector
 cJSON_LIB += cJSON.o
 
+ip2region_CFLAGS += -Os -g -Wall -I/ip2region -c 
+ip2region_LIB += ip2region.o xdb_searcher.o
+
 all: conf.o rhost.o libiptc.o ccronexpr.o 
+	$(CC) $(ip2region_CFLAGS) ip2region/ip2region.c
+	$(CC) $(ip2region_CFLAGS) ip2region/xdb_searcher.c
 	$(CC) $(cJSON_CFLAGS) cJSON/cJSON.c
 	$(CC) $(freshclam_CFLAGS) -c clamscan/freshclam/freshclam.c -fPIC -DPIC -o clamscan/freshclam/freshclam.o
 	$(CC) $(freshclam_CFLAGS) -c clamscan/freshclam/notify.c -fPIC -DPIC -o clamscan/freshclam/notify.o
@@ -44,7 +49,7 @@ all: conf.o rhost.o libiptc.o ccronexpr.o
 	$(CC) $(CLAMSCAN_CFLAGS) -c clamscan/clamscan.c -o clamscan/clamscan.o
 	$(CC) $(CLAMSCAN_CFLAGS) -c clamscan/manager.c -o clamscan/manager.o
 	
-	$(CC) $(CFLAGS) $^ -o $(OBG) $(cJSON_LIB) $(LIB) $(freshclam_LIB)
+	$(CC) $(CFLAGS) $^ -o $(OBG) $(cJSON_LIB) $(ip2region_LIB) $(LIB) $(freshclam_LIB)
 	
 	chmod +x $(OBG)
 
@@ -61,4 +66,6 @@ clean:
 	rm -rf clamscan/freshclam/*.o
 	rm -rf `find clamscan/libclamav/ -name *.lo`
 	rm -rf `find clamscan/libclamav/ -name *.o`
+	rm -rf cJSON/*.o
+	rm -rf ip2region/*.o
 	rm -rf $(OBG)

ip2region/ip2region.c

@@ -0,0 +1,45 @@
+#include <stdio.h>
+#include "xdb_searcher.h"
+#include "ip2region.h"
+
+int ip2region(char *xdb_file, char *ip)
+{
+    char *db_path = xdb_file;
+    xdb_vector_index_t *v_index;
+    xdb_searcher_t searcher;
+    char region_buffer[256], ip_buffer[16];
+    long s_time;
+
+    // 1、从 db_path 加载 VectorIndex 索引。
+    // 得到 v_index 做成全局缓存，便于后续反复使用。
+    // 注意：v_index 不需要每次都加载，建议在服务启动的时候加载一次，然后做成全局资源。
+    v_index = xdb_load_vector_index_from_file(db_path);
+    if (v_index == NULL) {
+        printf("failed to load vector index from `%s`\n", db_path);
+        return 1;
+    }
+    
+    // 2、使用全局的 VectorIndex 变量创建带 VectorIndex 缓存的 xdb 查询对象
+    int err = xdb_new_with_vector_index(&searcher, db_path, v_index);
+    if (err != 0) {
+        printf("failed to create vector index cached searcher with errcode=%d\n", err);
+        return 2;
+    }
+    
+    // 3、调用 search API 查询
+    // 得到的 region 信息会存储到 region_buffer 里面，如果你自定义了数据，请确保给足 buffer 的空间。
+    s_time = xdb_now();
+    err = xdb_search_by_string(&searcher, ip, region_buffer, sizeof(region_buffer));
+    if (err != 0) {
+        printf("failed search(%s) with errno=%d\n", ip, err);
+    } else {
+        printf("{region: %s, took: %d μs}", region_buffer, (int)(xdb_now() - s_time));
+    }
+
+    // 备注：并发使用，没一个线程需要单独定义并且初始化一个 searcher 查询对象。
+
+    // 4、关闭 xdb 查询器，如果是要关闭服务，也需要释放 v_index 的内存。
+    xdb_close(&searcher);
+    xdb_close_vector_index(v_index);
+    return 0;
+}

ip2region/ip2region.h

@@ -0,0 +1,8 @@
+#ifndef IP2REGION_H
+#define IP2REGION_H
+
+
+int ip2region(char *xdb_file, char *ip);
+
+
+#endif

ip2region/xdb_searcher.c

@@ -0,0 +1,402 @@
+// Copyright 2022 The Ip2Region Authors. All rights reserved.
+// Use of this source code is governed by a Apache2.0-style
+// license that can be found in the LICENSE file.
+
+// ---
+// @Author Lion <chenxin619315@gmail.com>
+// @Date   2022/06/27
+
+#include "sys/time.h"
+#include "xdb_searcher.h"
+
+// internal function prototype define
+XDB_PRIVATE(int) read(xdb_searcher_t *, long offset, char *, size_t length);
+
+XDB_PRIVATE(int) xdb_new_base(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index, const xdb_content_t *c_buffer) {
+    memset(xdb, 0x00, sizeof(xdb_searcher_t));
+
+    // check the content buffer first
+    if (c_buffer != NULL) {
+        xdb->v_index = NULL;
+        xdb->content = c_buffer;
+        return 0;
+    }
+
+    // open the xdb binary file
+    FILE *handle = fopen(db_path, "rb");
+    if (handle == NULL) {
+        return 1;
+    }
+
+    xdb->handle = handle;
+    xdb->v_index = v_index;
+
+    return 0;
+}
+
+// xdb searcher new api define
+XDB_PUBLIC(int) xdb_new_with_file_only(xdb_searcher_t *xdb, const char *db_path) {
+    return xdb_new_base(xdb, db_path, NULL, NULL);
+}
+
+XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index) {
+    return xdb_new_base(xdb, db_path, v_index, NULL);
+}
+
+XDB_PUBLIC(int) xdb_new_with_buffer(xdb_searcher_t *xdb, const xdb_content_t *c_buffer) {
+    return xdb_new_base(xdb, NULL, NULL, c_buffer);
+}
+
+XDB_PUBLIC(void) xdb_close(void *ptr) {
+    xdb_searcher_t *xdb = (xdb_searcher_t *) ptr;
+    if (xdb->handle != NULL) {
+        fclose(xdb->handle);
+        xdb->handle = NULL;
+    }
+}
+
+// --- xdb searcher search api define
+
+XDB_PUBLIC(int) xdb_search_by_string(xdb_searcher_t *xdb, const char *str_ip, char *region_buffer, size_t length) {
+    unsigned int ip = 0;
+    int errcode = xdb_check_ip(str_ip, &ip);
+    if (errcode != 0) {
+        return 10 + errcode;
+    } else {
+        return xdb_search(xdb, ip, region_buffer, length);
+    }
+}
+
+XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_buffer, size_t length) {
+    int il0, il1, idx, err, l, h, m, data_len;
+    unsigned int s_ptr, e_ptr, p, sip, eip, data_ptr;
+    char vector_buffer[xdb_vector_index_size], segment_buffer[xdb_segment_index_size];
+
+    // reset the io counter
+    xdb->io_count = 0;
+
+    // locate the segment index block based on the vector index
+    il0 = ((int) (ip >> 24)) & 0xFF;
+    il1 = ((int) (ip >> 16)) & 0xFF;
+    idx = il0 * xdb_vector_index_cols * xdb_vector_index_size + il1 * xdb_vector_index_size;
+    if (xdb->v_index != NULL) {
+        s_ptr = xdb_get_uint(xdb->v_index->buffer, idx);
+        e_ptr = xdb_get_uint(xdb->v_index->buffer, idx + 4);
+    } else if (xdb->content != NULL) {
+        s_ptr = xdb_get_uint(xdb->content->buffer, xdb_header_info_length + idx);
+        e_ptr = xdb_get_uint(xdb->content->buffer, xdb_header_info_length + idx + 4);
+    } else {
+        err = read(xdb, xdb_header_info_length + idx, vector_buffer, sizeof(vector_buffer));
+        if (err != 0) {
+            return 10 + err;
+        }
+
+        s_ptr = xdb_get_uint(vector_buffer, 0);
+        e_ptr = xdb_get_uint(vector_buffer, 4);
+    }
+
+    // printf("s_ptr=%u, e_ptr=%u\n", s_ptr, e_ptr);
+    // binary search to get the final region info
+    data_len = 0, data_ptr = 0;
+    l = 0, h = ((int) (e_ptr - s_ptr)) / xdb_segment_index_size;
+    while (l <= h) {
+        m = (l + h) >> 1;
+        p = s_ptr + m * xdb_segment_index_size;
+
+        // read the segment index item
+        err = read(xdb, p, segment_buffer, sizeof(segment_buffer));
+        if (err != 0) {
+            return 20 + err;
+        }
+
+        // decode the data fields as needed
+        sip = xdb_get_uint(segment_buffer, 0);
+        if (ip < sip) {
+            h = m - 1;
+        } else {
+            eip = xdb_get_uint(segment_buffer, 4);
+            if (ip > eip) {
+                l = m + 1;
+            } else {
+                data_len = xdb_get_ushort(segment_buffer, 8);
+                data_ptr = xdb_get_uint(segment_buffer, 10);
+                break;
+            }
+        }
+    }
+
+    // printf("data_len=%u, data_ptr=%u\n", data_len, data_ptr);
+    if (data_len == 0) {
+        region_buffer[0] = '\0';
+        return 0;
+    }
+
+    // buffer length checking
+    if (data_len >= length) {
+        return 1;
+    }
+
+    err = read(xdb, data_ptr, region_buffer, data_len);
+    if (err != 0) {
+        return 30 + err;
+    }
+
+    // auto append a NULL-end
+    region_buffer[data_len] = '\0';
+    return 0;
+}
+
+XDB_PRIVATE(int) read(xdb_searcher_t *xdb, long offset, char *buffer, size_t length) {
+    // check the xdb content cache first
+    if (xdb->content != NULL) {
+        memcpy(buffer, xdb->content->buffer + offset, length);
+        return 0;
+    }
+
+    // seek to the offset
+    if (fseek(xdb->handle, offset, SEEK_SET) == -1) {
+        return 1;
+    }
+
+    xdb->io_count++;
+    if (fread(buffer, 1, length, xdb->handle) != length) {
+        return 2;
+    }
+
+    return 0;
+}
+
+XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *xdb) {
+    return xdb->io_count;
+}
+
+
+// --- buffer load util functions
+
+XDB_PUBLIC(xdb_header_t *) xdb_load_header(FILE *handle) {
+    xdb_header_t *header;
+    unsigned int size = xdb_header_info_length;
+
+    // entry alloc
+    header = (xdb_header_t *) xdb_malloc(sizeof(xdb_header_t));
+    if (header == NULL) {
+        return NULL;
+    }
+
+    if (fseek(handle, 0, SEEK_SET) == -1) {
+        xdb_free(header);
+        return NULL;
+    }
+
+    if (fread(header->buffer, 1,size, handle) != size) {
+        xdb_free(header);
+        return NULL;
+    }
+
+    // fill the fields
+    header->length = size;
+    header->version = (unsigned short) xdb_get_ushort(header->buffer, 0);
+    header->index_policy = (unsigned short) xdb_get_ushort(header->buffer, 2);
+    header->created_at = xdb_get_uint(header->buffer, 4);
+    header->start_index_ptr = xdb_get_uint(header->buffer, 8);
+    header->end_index_ptr = xdb_get_uint(header->buffer,12);
+
+    return header;
+}
+
+XDB_PUBLIC(xdb_header_t *) xdb_load_header_from_file(const char *db_path) {
+    xdb_header_t *header;
+    FILE *handle = fopen(db_path, "rb");
+    if (handle == NULL) {
+        return NULL;
+    }
+
+    header = xdb_load_header(handle);
+    fclose(handle);
+    return header;
+}
+
+XDB_PUBLIC(void) xdb_close_header(void *ptr) {
+    xdb_header_t *header = (xdb_header_t *) ptr;
+    if (header->length > 0) {
+        header->length = 0;
+        xdb_free(header);
+    }
+}
+
+// --- vector index
+
+XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index(FILE *handle) {
+    xdb_vector_index_t *v_index;
+    unsigned int size = xdb_vector_index_length;
+
+    // seek to the vector index offset
+    if (fseek(handle, xdb_header_info_length, SEEK_SET) == -1) {
+        return NULL;
+    }
+
+    // do the buffer read
+    v_index = (xdb_vector_index_t *) xdb_malloc(sizeof(xdb_vector_index_t));
+    if (v_index == NULL) {
+        return NULL;
+    }
+
+    v_index->length = size;
+    if (fread(v_index->buffer, 1, size, handle) != size) {
+        xdb_free(v_index);
+        return NULL;
+    }
+
+    return v_index;
+}
+
+XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index_from_file(const char *db_path) {
+    xdb_vector_index_t *v_index;
+    FILE *handle = fopen(db_path, "rb");
+    if (handle == NULL) {
+        return NULL;
+    }
+
+    v_index = xdb_load_vector_index(handle);
+    fclose(handle);
+    return v_index;
+}
+
+XDB_PUBLIC(void) xdb_close_vector_index(void *ptr) {
+    xdb_vector_index_t *v_index = (xdb_vector_index_t *) ptr;
+    if (v_index->length > 0) {
+        v_index->length = 0;
+        xdb_free(v_index);
+    }
+}
+
+// --- content buffer
+
+XDB_PUBLIC(xdb_content_t *) xdb_load_content(FILE *handle) {
+    unsigned int size;
+    xdb_content_t *content;
+
+    // determine the file size
+    if (fseek(handle, 0, SEEK_END) == -1) {
+        return NULL;
+    }
+
+    size = (unsigned int) ftell(handle);
+    if (fseek(handle, 0, SEEK_SET) == -1) {
+        return NULL;
+    }
+
+    // do the file read
+    content = (xdb_content_t *) xdb_malloc(sizeof(xdb_content_t));
+    if (content == NULL) {
+        return NULL;
+    }
+
+    // do the buffer alloc
+    content->buffer = (char *) xdb_malloc(size);
+    if (content->buffer == NULL) {
+        xdb_free(content);
+        return NULL;
+    }
+
+    // read the content into the buffer
+    content->length = size;
+    if (fread(content->buffer, 1, size, handle) != size) {
+        xdb_free(content);
+        return NULL;
+    }
+
+    return content;
+}
+
+XDB_PUBLIC(xdb_content_t *) xdb_load_content_from_file(const char *db_path) {
+    xdb_content_t *content;
+    FILE *handle = fopen(db_path, "rb");
+    if (handle == NULL) {
+        return NULL;
+    }
+
+    content = xdb_load_content(handle);
+    fclose(handle);
+    return content;
+}
+
+XDB_PUBLIC(void) xdb_close_content(void *ptr) {
+    xdb_content_t *content = (xdb_content_t *) ptr;
+    if (content->length > 0) {
+        content->length = 0;
+        xdb_free(content->buffer);
+        content->buffer = NULL;
+        xdb_free(content);
+    }
+}
+
+// --- End
+
+// get unsigned long (4bytes) from a specified buffer start from the specified offset
+XDB_PUBLIC(unsigned int) xdb_get_uint(const char *buffer, int offset) {
+    return (
+        ((buffer[offset  ]) & 0x000000FF) |
+        ((buffer[offset+1] <<  8) & 0x0000FF00) |
+        ((buffer[offset+2] << 16) & 0x00FF0000) |
+        ((buffer[offset+3] << 24) & 0xFF000000)
+    );
+}
+
+// get unsigned short (2bytes) from a specified buffer start from the specified offset
+XDB_PUBLIC(int) xdb_get_ushort(const char *buffer, int offset) {
+    return (
+        ((buffer[offset  ]) & 0x000000FF) |
+        ((buffer[offset+1] << 8) & 0x0000FF00)
+    );
+}
+
+// string ip to unsigned int
+static int shiftIndex[4] = {24, 16, 8, 0};
+XDB_PUBLIC(int) xdb_check_ip(const char *src_ip, unsigned int *dst_ip) {
+    char c;
+    int i, n, ip = 0;
+    const char *ptr = src_ip;
+    for (i = 0; i < 4; i++) {
+        n = 0;
+        while (1) {
+            c = *ptr;
+            ptr++;
+            if (c >= '0' && c <= '9') {
+                n *= 10;
+                n += c - '0';
+            } else if ((i < 3 && c == '.') || i == 3) {
+                // stopping at the '.' but ignore the tailing chars
+                // after the 3rd one (auto clean the tailing none-integer ?).
+                break;
+            } else {
+                return 1;
+            }
+        }
+
+        if (n > 0xFF) {
+            return 2;
+        }
+
+        ip |= (n << shiftIndex[i]);
+    }
+
+    *dst_ip = ip;
+    return 0;
+}
+
+// unsigned int ip to string ip
+XDB_PUBLIC(void) xdb_long2ip(unsigned int ip, char *buffer) {
+    sprintf(buffer, "%d.%d.%d.%d", (ip >> 24) & 0xFF, (ip >> 16) & 0xFF, (ip >> 8) & 0xFF, ip & 0xFF);
+}
+
+// get the middle ip of a and b
+XDB_PUBLIC(unsigned int) xdb_mip(unsigned long a, unsigned long b) {
+    return (unsigned int) ((a + b) >> 1);
+}
+
+XDB_PUBLIC(long) xdb_now() {
+    struct timeval c_time;
+    gettimeofday(&c_time, NULL);
+    return c_time.tv_sec * (int)1e6 + c_time.tv_usec;
+}

ip2region/xdb_searcher.h

@@ -0,0 +1,149 @@
+// Copyright 2022 The Ip2Region Authors. All rights reserved.
+// Use of this source code is governed by a Apache2.0-style
+// license that can be found in the LICENSE file.
+
+// ---
+// @Author Lion <chenxin619315@gmail.com>
+// @Date   2022/06/27
+
+#ifndef C_XDB_SEARCHER_H
+#define C_XDB_SEARCHER_H
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#if ( defined(WIN32) || defined(_WIN32) || defined(__WINDOWS_) || defined(WINNT) )
+#   define XDB_PUBLIC(type)    extern __declspec(dllexport) type
+#   define XDB_PRIVATE(type)   static type
+#   define XDB_WINDOWS
+#elif ( defined(linux) || defined(_UNIX) )
+#   define XDB_PUBLIC(type)    extern type
+#   define XDB_PRIVATE(type)   static inline type
+#   define XDB_LINUX
+#endif
+
+#define xdb_calloc( _blocks, _bytes )  calloc( _blocks, _bytes )
+#define xdb_malloc( _bytes )           malloc( _bytes )
+#define xdb_free( _ptr )               free( _ptr )
+
+// public constants define
+#define xdb_header_info_length 256
+#define xdb_vector_index_rows  256
+#define xdb_vector_index_cols  256
+#define xdb_vector_index_size  8
+#define xdb_segment_index_size 14
+
+// cache of vector_index_row × vector_index_rows × vector_index_size
+#define xdb_vector_index_length 524288
+
+
+// --- buffer load util functions
+
+// use the following buffer struct to wrap the binary buffer data
+// since the buffer data could not be operated with the string API.
+struct xdb_header {
+    unsigned short version;
+    unsigned short index_policy;
+    unsigned int created_at;
+    unsigned int start_index_ptr;
+    unsigned int end_index_ptr;
+
+    // the original buffer
+    unsigned int length;
+    char buffer[xdb_header_info_length];
+};
+typedef struct xdb_header xdb_header_t;
+
+XDB_PUBLIC(xdb_header_t *) xdb_load_header(FILE *);
+
+XDB_PUBLIC(xdb_header_t *) xdb_load_header_from_file(const char *);
+
+XDB_PUBLIC(void) xdb_close_header(void *);
+
+
+// --- vector index buffer
+struct xdb_vector_index {
+    unsigned int length;
+    char buffer[xdb_vector_index_length];
+};
+typedef struct xdb_vector_index xdb_vector_index_t;
+
+XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index(FILE *);
+
+XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index_from_file(const char *);
+
+XDB_PUBLIC(void) xdb_close_vector_index(void *);
+
+
+// --- content buffer
+struct xdb_content {
+    unsigned int length;
+    char *buffer;
+};
+typedef struct xdb_content xdb_content_t;
+
+XDB_PUBLIC(xdb_content_t *) xdb_load_content(FILE *);
+
+XDB_PUBLIC(xdb_content_t *) xdb_load_content_from_file(const char *);
+
+XDB_PUBLIC(void) xdb_close_content(void *);
+
+// --- End buffer load
+
+// xdb searcher structure
+struct xdb_searcher_entry {
+    FILE *handle;
+
+    // header info
+    const char *header;
+    int io_count;
+
+    // vector index buffer cache.
+    // preload the vector index will reduce the number of IO operations
+    // thus speedup the search process.
+    const xdb_vector_index_t *v_index;
+
+    // content buffer.
+    // cache the whole xdb content.
+    const xdb_content_t *content;
+};
+typedef struct xdb_searcher_entry xdb_searcher_t;
+
+// xdb searcher new api define
+XDB_PUBLIC(int) xdb_new_with_file_only(xdb_searcher_t *, const char *);
+
+XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_searcher_t *, const char *, const xdb_vector_index_t *);
+
+XDB_PUBLIC(int) xdb_new_with_buffer(xdb_searcher_t *, const xdb_content_t *);
+
+XDB_PUBLIC(void) xdb_close(void *);
+
+// xdb searcher search api define
+XDB_PUBLIC(int) xdb_search_by_string(xdb_searcher_t *, const char *, char *, size_t);
+
+XDB_PUBLIC(int) xdb_search(xdb_searcher_t *, unsigned int, char *, size_t);
+
+XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *);
+
+
+// get unsigned long (4bytes) from a specified buffer start from the specified offset with little-endian
+XDB_PUBLIC(unsigned int) xdb_get_uint(const char *, int);
+
+// get unsigned short (2bytes) from a specified buffer start from the specified offset with little-endian
+XDB_PUBLIC(int) xdb_get_ushort(const char *, int);
+
+// check the specified string ip and convert it to an unsigned int
+XDB_PUBLIC(int) xdb_check_ip(const char *, unsigned int *);
+
+// unsigned int ip to string ip
+XDB_PUBLIC(void) xdb_long2ip(unsigned int, char *);
+
+// get the middle ip of a and b
+XDB_PUBLIC(unsigned int) xdb_mip(unsigned long, unsigned long);
+
+// get the current time in microseconds
+XDB_PUBLIC(long) xdb_now();
+
+
+#endif //C_XDB_SEARCHER_H

rhost.c

@@ -913,6 +913,9 @@ int main(int argc, char *argv[], char **env)
 
     signal(SIGCHLD, sig_child); // 创建捕捉子进程退出信号
     
+    // ip2region 离线IP地址定位库
+    //ip2region("ip2region/ip2region.xdb", "1.1.1.1");
+    
 
     int pid;
     int i;

rhost.h

@@ -18,6 +18,7 @@
 #include <assert.h>
 
 #include "./cJSON/cJSON.h"
+#include "ip2region/ip2region.h"
 
 
 typedef struct now_next_time