smartdns/src/dns_client.c

/*************************************************************************
 *
 * Copyright (C) 2018 Ruilin Peng (Nick) <pymumu@gmail.com>.
 *
 * smartdns is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * smartdns is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "dns_client.h"
#include "atomic.h"
#include "dns.h"
#include "dns_conf.h"
#include "fast_ping.h"
#include "hashtable.h"
#include "list.h"
#include "tlog.h"
#include "util.h"
#include <arpa/inet.h>
#include <errno.h>
#include <fcntl.h>
#include <linux/filter.h>
#include <netdb.h>
#include <netinet/icmp6.h>
#include <netinet/ip.h>
#include <netinet/ip6.h>
#include <netinet/ip_icmp.h>
#include <netinet/tcp.h>
#include <openssl/err.h>
#include <openssl/ssl.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/epoll.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

#define DNS_MAX_HOSTNAME 256
#define DNS_MAX_EVENTS 64
#define DNS_HOSTNAME_LEN 128
#define DNS_TCP_BUFFER (32 * 1024)
#define DNS_TCP_IDLE_TIMEOUT (60 * 10)
#define DNS_TCP_CONNECT_TIMEOUT (5)
#define DNS_QUERY_TIMEOUT (500)

#ifndef TCP_FASTOPEN_CONNECT
#define TCP_FASTOPEN_CONNECT 30
#endif

/* ECS info */
struct dns_client_ecs {
	int enable;
	unsigned int family;
	unsigned int bitlen;
	union {
		unsigned char ipv4_addr[DNS_RR_A_LEN];
		unsigned char ipv6_addr[DNS_RR_AAAA_LEN];
		unsigned char addr[0];
	};
};

/* TCP/TLS buffer */
struct dns_server_buff {
	unsigned char data[DNS_TCP_BUFFER];
	unsigned short len;
};

typedef enum dns_server_status {
	DNS_SERVER_STATUS_INIT = 0,
	DNS_SERVER_STATUS_CONNECTING,
	DNS_SERVER_STATUS_CONNECTIONLESS,
	DNS_SERVER_STATUS_CONNECTED,
	DNS_SERVER_STATUS_DISCONNECTED,
} dns_server_status;

/* dns server information */
struct dns_server_info {
	struct list_head list;
	/* server ping handle */
	struct ping_host_struct *ping_host;

	char ip[DNS_HOSTNAME_LEN];
	/* server type */
	dns_server_type_t type;

	unsigned char *spki;
	int spki_len;

	/* client socket */
	int fd;
	int ttl;
	int ttl_range;
	SSL *ssl;
	SSL_CTX *ssl_ctx;
	SSL_SESSION *ssl_session;
	dns_server_status status;
	unsigned int result_flag;

	struct dns_server_buff send_buff;
	struct dns_server_buff recv_buff;

	time_t last_send;
	time_t last_recv;

	/* server addr info */
	unsigned short ai_family;

	socklen_t ai_addrlen;
	union {
		struct sockaddr_in in;
		struct sockaddr_in6 in6;
		struct sockaddr addr;
	};
};

/* upstream server group member */
struct dns_server_group_member {
	struct list_head list;
	struct dns_server_info *server;
};

/* upstream server groups */
struct dns_server_group {
	char group_name[DNS_GROUP_NAME_LEN];
	struct hlist_node node;
	struct list_head head;
};

/* dns client */
struct dns_client {
	pthread_t tid;
	int run;
	int epoll_fd;

	/* dns server list */
	pthread_mutex_t server_list_lock;
	struct list_head dns_server_list;
	struct dns_server_group *default_group;

	/* query list */
	pthread_mutex_t dns_request_lock;
	struct list_head dns_request_list;
	atomic_t dns_server_num;

	/* ECS */
	struct dns_client_ecs ecs_ipv4;
	struct dns_client_ecs ecs_ipv6;

	/* query doman hash table, key: sid + domain */
	pthread_mutex_t domain_map_lock;
	DECLARE_HASHTABLE(domain_map, 6);
	DECLARE_HASHTABLE(group, 4);
};

/* dns replied server info */
struct dns_query_replied {
	struct hlist_node node;
	socklen_t addr_len;
	union {
		struct sockaddr_in in;
		struct sockaddr_in6 in6;
		struct sockaddr addr;
	};
};

/* query struct */
struct dns_query_struct {
	struct list_head dns_request_list;
	atomic_t refcnt;
	struct dns_server_group *server_group;

	/* query id, hash key sid + domain*/
	char domain[DNS_MAX_CNAME_LEN];
	unsigned short sid;
	struct hlist_node domain_node;

	struct list_head period_list;

	/* dns query type */
	int qtype;

	/* dns query number */
	atomic_t dns_request_sent;
	unsigned long send_tick;

	/* caller notification */
	dns_client_callback callback;
	void *user_ptr;

	/* replied hash table */
	DECLARE_HASHTABLE(replied_map, 4);
};

static struct dns_client client;
static atomic_t dns_client_sid = ATOMIC_INIT(0);

/* get addr info */
static struct addrinfo *_dns_client_getaddr(const char *host, char *port, int type, int protocol)
{
	struct addrinfo hints;
	struct addrinfo *result = NULL;
	int ret = 0;

	memset(&hints, 0, sizeof(hints));
	hints.ai_family = AF_UNSPEC;
	hints.ai_socktype = type;
	hints.ai_protocol = protocol;

	ret = getaddrinfo(host, port, &hints, &result);
	if (ret != 0) {
		tlog(TLOG_ERROR, "get addr info failed. %s\n", gai_strerror(errno));
		tlog(TLOG_ERROR, "host = %s, port = %s, type = %d, protocol = %d", host, port, type, protocol);
		goto errout;
	}

	return result;
errout:
	if (result) {
		freeaddrinfo(result);
	}
	return NULL;
}

/* check whether server exists */
static int _dns_client_server_exist(struct addrinfo *gai, dns_server_type_t server_type)
{
	struct dns_server_info *server_info, *tmp;
	pthread_mutex_lock(&client.server_list_lock);
	list_for_each_entry_safe(server_info, tmp, &client.dns_server_list, list)
	{
		if (server_info->ai_addrlen != gai->ai_addrlen || server_info->ai_family != gai->ai_family) {
			continue;
		}

		if (server_info->type != server_type) {
			continue;
		}

		if (memcmp(&server_info->addr, gai->ai_addr, gai->ai_addrlen) != 0) {
			continue;
		}

		pthread_mutex_unlock(&client.server_list_lock);
		return 0;
	}

	pthread_mutex_unlock(&client.server_list_lock);
	return -1;
}

static void _dns_client_server_update_ttl(struct ping_host_struct *ping_host, const char *host, FAST_PING_RESULT result, struct sockaddr *addr, socklen_t addr_len,
								   int seqno, int ttl, struct timeval *tv, void *userptr)
{
	struct dns_server_info *server_info = userptr;
	if (result != PING_RESULT_RESPONSE || server_info == NULL) {
		return;
	}

	double rtt = tv->tv_sec * 1000.0 + tv->tv_usec / 1000.0;
	tlog(TLOG_DEBUG, "from %s: seq=%d ttl=%d time=%.3f\n", host, seqno, ttl, rtt);
	server_info->ttl = ttl;
}

/* get server control block by ip and port, type */
static struct dns_server_info *_dns_client_get_server(char *server_ip, int port, dns_server_type_t server_type)
{
	struct dns_server_info *server_info, *tmp;
	struct dns_server_info *server_info_return = NULL;
	char port_s[8];
	int sock_type;
	struct addrinfo *gai = NULL;

	if (server_type >= DNS_SERVER_TYPE_END) {
		tlog(TLOG_ERROR, "server type is invalid.");
		return NULL;
	}

	switch (server_type) {
	case DNS_SERVER_UDP:
		sock_type = SOCK_DGRAM;
		break;
	case DNS_SERVER_TLS:
	case DNS_SERVER_TCP:
		sock_type = SOCK_STREAM;
		break;
	default:
		return NULL;
		break;
	}

	/* get addr info */
	snprintf(port_s, 8, "%d", port);
	gai = _dns_client_getaddr(server_ip, port_s, sock_type, 0);
	if (gai == NULL) {
		tlog(TLOG_ERROR, "get address failed, %s:%d", server_ip, port);
		goto errout;
	}

	pthread_mutex_lock(&client.server_list_lock);
	list_for_each_entry_safe(server_info, tmp, &client.dns_server_list, list)
	{
		if (server_info->ai_addrlen != gai->ai_addrlen || server_info->ai_family != gai->ai_family) {
			continue;
		}

		if (server_info->type != server_type) {
			continue;
		}

		if (memcmp(&server_info->addr, gai->ai_addr, gai->ai_addrlen) != 0) {
			continue;
		}

		pthread_mutex_unlock(&client.server_list_lock);
		server_info_return = server_info;
		break;
	}

	pthread_mutex_unlock(&client.server_list_lock);

	freeaddrinfo(gai);
	return server_info_return;
errout:
	if (gai) {
		freeaddrinfo(gai);
	}
	return NULL;
}

/* get server group by name */
static struct dns_server_group *_dns_client_get_group(const char *group_name)
{
	unsigned long key;
	struct dns_server_group *group = NULL;
	struct hlist_node *tmp = NULL;

	if (group_name == NULL) {
		return NULL;
	}

	key = hash_string(group_name);
	hash_for_each_possible_safe(client.group, group, tmp, node, key)
	{
		if (strncmp(group->group_name, group_name, DNS_GROUP_NAME_LEN) != 0) {
			continue;
		}

		return group;
	}

	return NULL;
}

/* get server group by name */
static struct dns_server_group *_dns_client_get_dnsserver_group(const char *group_name)
{
	struct dns_server_group *group = _dns_client_get_group(group_name);

	if (group == NULL) {
		group = client.default_group;
		tlog(TLOG_DEBUG, "send query to group %s", DNS_SERVER_GROUP_DEFAULT);
	} else {
		if (list_empty(&group->head)) {
			group = client.default_group;
			tlog(TLOG_DEBUG, "send query to group %s", DNS_SERVER_GROUP_DEFAULT);
		} else {
			tlog(TLOG_DEBUG, "send query to group %s", group_name);
		}
	}

	return group;
}

/* add server to group */
static int _dns_client_add_to_group(char *group_name, struct dns_server_info *server_info)
{
	struct dns_server_group *group = NULL;
	struct dns_server_group_member *group_member = NULL;

	group = _dns_client_get_group(group_name);
	if (group == NULL) {
		tlog(TLOG_ERROR, "group %s not exist.", group_name);
		return -1;
	}

	group_member = malloc(sizeof(*group_member));
	if (group_member == NULL) {
		tlog(TLOG_ERROR, "malloc memory failed.");
		goto errout;
	}

	memset(group_member, 0, sizeof(*group_member));
	group_member->server = server_info;
	list_add(&group_member->list, &group->head);

	return 0;
errout:
	if (group_member) {
		free(group_member);
	}

	return -1;
}

/* add server to group */
int dns_client_add_to_group(char *group_name, char *server_ip, int port, dns_server_type_t server_type)
{
	struct dns_server_info *server_info = NULL;

	server_info = _dns_client_get_server(server_ip, port, server_type);
	if (server_info == NULL) {
		return -1;
	}

	return _dns_client_add_to_group(group_name, server_info);
}

/* free group member */
static int _dns_client_remove_member(struct dns_server_group_member *group_member)
{
	list_del_init(&group_member->list);
	free(group_member);

	return 0;
}

static int _dns_client_remove_from_group(struct dns_server_group *group, struct dns_server_info *server_info)
{
	struct dns_server_group_member *group_member;
	struct dns_server_group_member *tmp;

	list_for_each_entry_safe(group_member, tmp, &group->head, list)
	{
		if (group_member->server != server_info) {
			continue;
		}

		_dns_client_remove_member(group_member);
	}

	return 0;
}

static int _dns_client_remove_server_from_groups(struct dns_server_info *server_info)
{
	struct dns_server_group *group;
	struct hlist_node *tmp = NULL;
	int i = 0;

	hash_for_each_safe(client.group, i, tmp, group, node)
	{
		_dns_client_remove_from_group(group, server_info);
	}

	return 0;
}

int dns_client_remove_from_group(char *group_name, char *server_ip, int port, dns_server_type_t server_type)
{
	struct dns_server_info *server_info = NULL;
	struct dns_server_group *group = NULL;

	server_info = _dns_client_get_server(server_ip, port, server_type);
	if (server_info == NULL) {
		return -1;
	}

	group = _dns_client_get_group(group_name);
	if (group == NULL) {
		return -1;
	}

	return _dns_client_remove_from_group(group, server_info);
}

int dns_client_add_group(char *group_name)
{
	unsigned long key;
	struct dns_server_group *group = NULL;

	if (_dns_client_get_group(group_name) != NULL) {
		tlog(TLOG_ERROR, "add group %s failed, group already exists", group_name);
		return -1;
	}

	group = malloc(sizeof(*group));
	if (group == NULL) {
		goto errout;
	}

	memset(group, 0, sizeof(*group));
	INIT_LIST_HEAD(&group->head);
	strncpy(group->group_name, group_name, DNS_GROUP_NAME_LEN);
	key = hash_string(group_name);
	hash_add(client.group, &group->node, key);

	return 0;
errout:
	if (group) {
		free(group);
		group = NULL;
	}

	return -1;
}

static int _dns_client_remove_group(struct dns_server_group *group)
{
	struct dns_server_group_member *group_member;
	struct dns_server_group_member *tmp;

	list_for_each_entry_safe(group_member, tmp, &group->head, list)
	{
		_dns_client_remove_member(group_member);
	}

	hash_del(&group->node);
	free(group);

	return 0;
}

int dns_client_remove_group(char *group_name)
{
	unsigned long key;
	struct dns_server_group *group = NULL;
	struct hlist_node *tmp = NULL;

	key = hash_string(group_name);
	hash_for_each_possible_safe(client.group, group, tmp, node, key)
	{
		if (strncmp(group->group_name, group_name, DNS_GROUP_NAME_LEN) != 0) {
			continue;
		}

		_dns_client_remove_group(group);

		return 0;
	}

	return 0;
}

static void _dns_client_group_remove_all(void)
{
	struct dns_server_group *group;
	struct hlist_node *tmp = NULL;
	int i = 0;

	hash_for_each_safe(client.group, i, tmp, group, node)
	{
		_dns_client_remove_group(group);
	}
}

/* add dns server information */
static int _dns_client_server_add(char *server_ip, struct addrinfo *gai, dns_server_type_t server_type, unsigned int server_flag, unsigned int result_flag, int ttl,
						   char *spki)
{
	struct dns_server_info *server_info = NULL;
	unsigned char *spki_data = NULL;
	int spki_data_len = 0;

	/* read SPKI value, base64 sha256 value */
	if (spki && (strlen(spki) < DNS_MAX_SPKI_LEN)) {
		spki_data = malloc(DNS_MAX_SPKI_LEN);
		if (spki_data) {
			memset(spki_data, 0, DNS_MAX_SPKI_LEN);
			spki_data_len = SSL_base64_decode(spki, spki_data);
			if (spki_data_len != SHA256_DIGEST_LENGTH) {
				free(spki_data);
				spki_data = NULL;
				spki_data_len = 0;
			}
		}
	}

	/* if server exist, return */
	if (_dns_client_server_exist(gai, server_type) == 0) {
		return 0;
	}

	server_info = malloc(sizeof(*server_info));
	if (server_info == NULL) {
		goto errout;
	}

	if (server_type != DNS_SERVER_UDP) {
		result_flag &= (~DNSSERVER_FLAG_CHECK_TTL);
	}

	memset(server_info, 0, sizeof(*server_info));
	strncpy(server_info->ip, server_ip, sizeof(server_info->ip));
	server_info->ai_family = gai->ai_family;
	server_info->ai_addrlen = gai->ai_addrlen;
	server_info->type = server_type;
	server_info->fd = 0;
	server_info->status = DNS_SERVER_STATUS_INIT;
	server_info->result_flag = result_flag;
	server_info->ttl = ttl;
	server_info->ttl_range = 0;
	server_info->spki = spki_data;
	server_info->spki_len = spki_data_len;

	/* exclude this server from default group */
	if ((server_flag & SERVER_FLAG_EXCLUDE_DEFAULT) == 0) {
		if (_dns_client_add_to_group(DNS_SERVER_GROUP_DEFAULT, server_info) != 0) {
			tlog(TLOG_ERROR, "add server to default group failed.");
			goto errout;
		}
	}

	/* if server type is TLS, create ssl context */
	if (server_type == DNS_SERVER_TLS) {
		server_info->ssl_ctx = SSL_CTX_new(SSLv23_client_method());
		if (server_info->ssl_ctx == NULL) {
			tlog(TLOG_ERROR, "init ssl failed.");
			goto errout;
		}
	}

	/* safe address info */
	if (gai->ai_addrlen > sizeof(server_info->in6)) {
		tlog(TLOG_ERROR, "addr len invalid, %d, %zd, %d", gai->ai_addrlen, sizeof(server_info->addr), server_info->ai_family);
		goto errout;
	}
	memcpy(&server_info->addr, gai->ai_addr, gai->ai_addrlen);

	/* start ping task */
	if (ttl <= 0 && (result_flag & DNSSERVER_FLAG_CHECK_TTL)) {
		server_info->ping_host = fast_ping_start(PING_TYPE_DNS, server_ip, 0, 60000, 1000, _dns_client_server_update_ttl, server_info);
		if (server_info->ping_host == NULL) {
			tlog(TLOG_ERROR, "start ping failed.");
			goto errout;
		}

		if (ttl < 0) {
			server_info->ttl_range = -ttl;
		}
	}

	/* add to list */
	pthread_mutex_lock(&client.server_list_lock);
	list_add(&server_info->list, &client.dns_server_list);
	pthread_mutex_unlock(&client.server_list_lock);

	atomic_inc(&client.dns_server_num);
	return 0;
errout:
	if (spki_data) {
		free(spki_data);
	}

	if (server_info) {
		if (server_info->ssl_ctx) {
			SSL_CTX_free(server_info->ssl_ctx);
			server_info->ssl_ctx = NULL;
		}
		if (server_info->ping_host) {
			fast_ping_stop(server_info->ping_host);
		}

		free(server_info);
	}

	return -1;
}

static void _dns_client_close_socket(struct dns_server_info *server_info)
{
	if (server_info->fd <= 0) {
		return;
	}

	if (server_info->ssl) {
		/* Shutdown ssl */
		SSL_shutdown(server_info->ssl);
		SSL_free(server_info->ssl);
		server_info->ssl = NULL;
	}

	/* remove fd from epoll */
	epoll_ctl(client.epoll_fd, EPOLL_CTL_DEL, server_info->fd, NULL);
	close(server_info->fd);

	server_info->fd = -1;
	server_info->status = DNS_SERVER_STATUS_DISCONNECTED;
	/* update send recv time */
	time(&server_info->last_send);
	time(&server_info->last_recv);
	tlog(TLOG_DEBUG, "server %s closed.", server_info->ip);
}

static void _dns_client_server_close(struct dns_server_info *server_info)
{
	/* stop ping task */
	if (server_info->ping_host) {
		if (fast_ping_stop(server_info->ping_host) != 0) {
			tlog(TLOG_ERROR, "stop ping failed.\n");
		}
	}

	_dns_client_close_socket(server_info);

	if (server_info->ssl_session) {
		SSL_SESSION_free(server_info->ssl_session);
		server_info->ssl_session = NULL;
	}

	if (server_info->ssl_ctx) {
		SSL_CTX_free(server_info->ssl_ctx);
		server_info->ssl_ctx = NULL;
	}
}

/* remove all servers information */
static void _dns_client_server_remove_all(void)
{
	struct dns_server_info *server_info, *tmp;
	pthread_mutex_lock(&client.server_list_lock);
	list_for_each_entry_safe(server_info, tmp, &client.dns_server_list, list)
	{
		list_del(&server_info->list);
		_dns_client_server_close(server_info);
		if (server_info->spki) {
			free(server_info->spki);
			server_info->spki = NULL;
		}
		free(server_info);
	}
	pthread_mutex_unlock(&client.server_list_lock);
}

/* remove single server */
static int _dns_client_server_remove(char *server_ip, struct addrinfo *gai, dns_server_type_t server_type)
{
	struct dns_server_info *server_info, *tmp;

	/* find server and remove */
	pthread_mutex_lock(&client.server_list_lock);
	list_for_each_entry_safe(server_info, tmp, &client.dns_server_list, list)
	{
		if (server_info->ai_addrlen != gai->ai_addrlen || server_info->ai_family != gai->ai_family) {
			continue;
		}

		if (memcmp(&server_info->addr, gai->ai_addr, gai->ai_addrlen) != 0) {
			continue;
		}

		list_del(&server_info->list);
		_dns_client_server_close(server_info);
		pthread_mutex_unlock(&client.server_list_lock);
		_dns_client_remove_server_from_groups(server_info);
		free(server_info);
		atomic_dec(&client.dns_server_num);
		return 0;
	}
	pthread_mutex_unlock(&client.server_list_lock);
	return -1;
}

static int _dns_client_server_operate(char *server_ip, int port, dns_server_type_t server_type, unsigned int server_flag, unsigned int result_flag, int ttl,
							   char *spki, int operate)
{
	char port_s[8];
	int sock_type;
	int ret;
	struct addrinfo *gai = NULL;

	if (server_type >= DNS_SERVER_TYPE_END) {
		tlog(TLOG_ERROR, "server type is invalid.");
		return -1;
	}

	switch (server_type) {
	case DNS_SERVER_UDP:
		sock_type = SOCK_DGRAM;
		break;
	case DNS_SERVER_TLS:
	case DNS_SERVER_TCP:
		sock_type = SOCK_STREAM;
		break;
	default:
		return -1;
		break;
	}

	/* get addr info */
	snprintf(port_s, 8, "%d", port);
	gai = _dns_client_getaddr(server_ip, port_s, sock_type, 0);
	if (gai == NULL) {
		tlog(TLOG_ERROR, "get address failed, %s:%d", server_ip, port);
		goto errout;
	}

	if (operate == 0) {
		/* add server */
		ret = _dns_client_server_add(server_ip, gai, server_type, server_flag, result_flag, ttl, spki);
		if (ret != 0) {
			goto errout;
		}
	} else {
		/* remove server */
		ret = _dns_client_server_remove(server_ip, gai, server_type);
		if (ret != 0) {
			goto errout;
		}
	}
	freeaddrinfo(gai);
	return 0;
errout:
	if (gai) {
		freeaddrinfo(gai);
	}
	return -1;
}

int dns_client_add_server(char *server_ip, int port, dns_server_type_t server_type, unsigned server_flag, unsigned int result_flag, int ttl, char *spki)
{
	return _dns_client_server_operate(server_ip, port, server_type, server_flag, result_flag, ttl, spki, 0);
}

int dns_client_remove_server(char *server_ip, int port, dns_server_type_t server_type)
{
	return _dns_client_server_operate(server_ip, port, server_type, 0, 0, 0, NULL, 1);
}

int dns_server_num(void)
{
	return atomic_read(&client.dns_server_num);
}

static void _dns_client_query_get(struct dns_query_struct *query)
{
	if (atomic_inc_return(&query->refcnt) <= 0) {
		tlog(TLOG_ERROR, "BUG: query ref is invalid, domain: %s", query->domain);
		abort();
	}
}

static void _dns_client_query_release(struct dns_query_struct *query)
{
	int refcnt = atomic_dec_return(&query->refcnt);
	int bucket = 0;
	struct dns_query_replied *replied_map;
	struct hlist_node *tmp;

	if (refcnt) {
		if (refcnt < 0) {
			tlog(TLOG_ERROR, "BUG: refcnt is %d", refcnt);
			abort();
		}
		return;
	}

	/* notify caller query end */
	if (query->callback) {
		query->callback(query->domain, DNS_QUERY_END, 0, NULL, NULL, 0, query->user_ptr);
	}

	/* free resource */
	pthread_mutex_lock(&client.domain_map_lock);
	list_del_init(&query->dns_request_list);
	hash_del(&query->domain_node);
	pthread_mutex_unlock(&client.domain_map_lock);

	hash_for_each_safe(query->replied_map, bucket, tmp, replied_map, node)
	{
		hash_del(&replied_map->node);
		free(replied_map);
	}
	memset(query, 0, sizeof(*query));
	free(query);
}

static void _dns_client_query_remove(struct dns_query_struct *query)
{
	/* remove query from period check list, and release reference*/
	pthread_mutex_lock(&client.domain_map_lock);
	if (list_empty(&query->dns_request_list)) {
		pthread_mutex_unlock(&client.domain_map_lock);
		return;
	}
	list_del_init(&query->dns_request_list);
	hash_del(&query->domain_node);
	pthread_mutex_unlock(&client.domain_map_lock);

	_dns_client_query_release(query);
}

static void _dns_client_query_remove_all(void)
{
	struct dns_query_struct *query, *tmp;
	LIST_HEAD(check_list);

	pthread_mutex_lock(&client.domain_map_lock);
	list_for_each_entry_safe(query, tmp, &client.dns_request_list, dns_request_list)
	{
		list_add(&query->period_list, &check_list);
	}
	pthread_mutex_unlock(&client.domain_map_lock);

	list_for_each_entry_safe(query, tmp, &check_list, period_list)
	{
		list_del_init(&query->period_list);
		_dns_client_query_remove(query);
	}

	return;
}

static void _dns_client_check_udp_nat(struct dns_query_struct *query)
{
	struct dns_server_info *server_info = NULL;
	struct dns_server_group_member *group_member = NULL;

	/* For udp nat case.
	 * when router reconnect to internet, udp port may always marked as UNREPLIED.
	 * dns query will timeout, and cannot reconnect again,
	 * create a new socket to communicate.
	 */
	pthread_mutex_lock(&client.server_list_lock);
	list_for_each_entry(group_member, &query->server_group->head, list)
	{
		server_info = group_member->server;
		if (server_info->type != DNS_SERVER_UDP) {
			continue;
		}

		if (server_info->last_send - 5 > server_info->last_recv) {
			server_info->recv_buff.len = 0;
			server_info->send_buff.len = 0;
			tlog(TLOG_DEBUG, "query server %s timeout.", server_info->ip);
			_dns_client_close_socket(server_info);
		}
	}
	pthread_mutex_unlock(&client.server_list_lock);
}

static void _dns_client_check_tcp(void)
{
	struct dns_server_info *server_info;
	time_t now;

	time(&now);

	pthread_mutex_lock(&client.server_list_lock);
	list_for_each_entry(server_info, &client.dns_server_list, list)
	{
		if (server_info->type == DNS_SERVER_UDP) {
			/* no need to check udp server */
			continue;
		}

		if (server_info->status == DNS_SERVER_STATUS_CONNECTING) {
			if (server_info->last_send + DNS_TCP_CONNECT_TIMEOUT < now) {
				tlog(TLOG_DEBUG, "server %s connect timeout.", server_info->ip);
				_dns_client_close_socket(server_info);
			}
		} else if (server_info->status == DNS_SERVER_STATUS_CONNECTED) {
			if (server_info->last_recv + DNS_TCP_IDLE_TIMEOUT < now) {
				/*disconnect if the server is not responding */
				server_info->recv_buff.len = 0;
				server_info->send_buff.len = 0;
				_dns_client_close_socket(server_info);
			}
		}
	}
	pthread_mutex_unlock(&client.server_list_lock);
}

static void _dns_client_period_run_second(void)
{
	_dns_client_check_tcp();
}

static void _dns_client_period_run(void)
{
	struct dns_query_struct *query, *tmp;
	static unsigned int msec = 0;
	msec++;

	LIST_HEAD(check_list);

	unsigned long now = get_tick_count();

	/* get query which timed out to check list */
	pthread_mutex_lock(&client.domain_map_lock);
	list_for_each_entry_safe(query, tmp, &client.dns_request_list, dns_request_list)
	{
		if ((now - DNS_QUERY_TIMEOUT >= query->send_tick) && query->send_tick > 0) {
			list_add(&query->period_list, &check_list);
			_dns_client_query_get(query);
		}
	}
	pthread_mutex_unlock(&client.domain_map_lock);

	list_for_each_entry_safe(query, tmp, &check_list, period_list)
	{
		/* free timed out query, and notify caller */
		list_del_init(&query->period_list);
		_dns_client_check_udp_nat(query);
		_dns_client_query_remove(query);
		_dns_client_query_release(query);
	}

	if (msec % 10 == 0) {
		_dns_client_period_run_second();
	}

	return;
}

static struct dns_query_struct *_dns_client_get_request(unsigned short sid, char *domain)
{
	struct dns_query_struct *query = NULL;
	struct dns_query_struct *query_result = NULL;
	struct hlist_node *tmp = NULL;
	uint32_t key;

	/* get query by hash key : id + domain */
	key = hash_string(domain);
	key = jhash(&sid, sizeof(sid), key);
	pthread_mutex_lock(&client.domain_map_lock);
	hash_for_each_possible_safe(client.domain_map, query, tmp, domain_node, key)
	{
		if (sid != query->sid) {
			continue;
		}

		if (strncmp(query->domain, domain, DNS_MAX_CNAME_LEN) != 0) {
			continue;
		}

		query_result = query;
		_dns_client_query_get(query_result);
		break;
	}
	pthread_mutex_unlock(&client.domain_map_lock);

	return query_result;
}

static int _dns_replied_check_add(struct dns_query_struct *dns_query, struct sockaddr *addr, socklen_t addr_len)
{
	uint32_t key = 0;
	struct dns_query_replied *replied_map = NULL;

	if (addr_len > sizeof(struct sockaddr_in6)) {
		tlog(TLOG_ERROR, "addr length is invalid.");
		return -1;
	}

	/* avoid multiple replies from one server */
	key = jhash(addr, addr_len, 0);
	hash_for_each_possible(dns_query->replied_map, replied_map, node, key)
	{
		/* already replied, ignore this reply */
		if (memcmp(&replied_map->addr, addr, addr_len) == 0) {
			return -1;
		}
	}

	replied_map = malloc(sizeof(*replied_map));
	if (replied_map == NULL) {
		tlog(TLOG_ERROR, "malloc failed");
		return -1;
	}

	/* add address info to check hashtable */
	memcpy(&replied_map->addr, addr, addr_len);
	hash_add(dns_query->replied_map, &replied_map->node, key);
	return 0;
}

static int _dns_client_recv(struct dns_server_info *server_info, unsigned char *inpacket, int inpacket_len, struct sockaddr *from, socklen_t from_len)
{
	int len;
	int i;
	int qtype;
	int qclass;
	char domain[DNS_MAX_CNAME_LEN];
	int rr_count;
	struct dns_rrs *rrs = NULL;
	unsigned char packet_buff[DNS_PACKSIZE];
	struct dns_packet *packet = (struct dns_packet *)packet_buff;
	int ret = 0;
	struct dns_query_struct *query;
	int request_num = 0;
	int has_opt = 0;

	packet->head.tc = 0;

	/* decode domain from udp packet */
	len = dns_decode(packet, DNS_PACKSIZE, inpacket, inpacket_len);
	if (len != 0) {
		char host_name[DNS_MAX_CNAME_LEN];
		tlog(TLOG_WARN, "decode failed, packet len = %d, tc = %d, id = %d, from = %s\n", inpacket_len, packet->head.tc, packet->head.id,
			 gethost_by_addr(host_name, sizeof(host_name), from));
		return -1;
	}

	/* not answer, return error */
	if (packet->head.qr != DNS_OP_IQUERY) {
		tlog(TLOG_DEBUG, "message type error.\n");
		return -1;
	}

	tlog(TLOG_DEBUG, "qdcount = %d, ancount = %d, nscount = %d, nrcount = %d, len = %d, id = %d, tc = %d, rd = %d, ra = %d, rcode = %d, payloadsize = %d\n",
		 packet->head.qdcount, packet->head.ancount, packet->head.nscount, packet->head.nrcount, inpacket_len, packet->head.id, packet->head.tc,
		 packet->head.rd, packet->head.ra, packet->head.rcode, dns_get_OPT_payload_size(packet));

	/* get question */
	rrs = dns_get_rrs_start(packet, DNS_RRS_QD, &rr_count);
	for (i = 0; i < rr_count && rrs; i++, rrs = dns_get_rrs_next(packet, rrs)) {
		dns_get_domain(rrs, domain, DNS_MAX_CNAME_LEN, &qtype, &qclass);
		tlog(TLOG_DEBUG, "domain: %s qtype: %d  qclass: %d\n", domain, qtype, qclass);
	}

	if (dns_get_OPT_payload_size(packet) > 0) {
		has_opt = 1;
	}

	/* get query reference */
	query = _dns_client_get_request(packet->head.id, domain);
	if (query == NULL || (query && has_opt == 0 && server_info->result_flag & DNSSERVER_FLAG_CHECK_EDNS)) {
		if (query) {
			_dns_client_query_release(query);
		}
		return 0;
	}

	/* avoid multiple replies */
	if (_dns_replied_check_add(query, (struct sockaddr *)from, from_len) != 0) {
		_dns_client_query_release(query);
		return 0;
	}

	request_num = atomic_dec_return(&query->dns_request_sent);
	if (request_num < 0) {
		_dns_client_query_release(query);
		tlog(TLOG_ERROR, "send count is invalid, %d", request_num);
		return -1;
	}

	/* notify caller dns query result */
	if (query->callback) {
		ret = query->callback(query->domain, DNS_QUERY_RESULT, server_info->result_flag, packet, inpacket, inpacket_len, query->user_ptr);
		if (request_num == 0 || ret) {
			/* if all server replied, or done, stop query, release resource */
			_dns_client_query_remove(query);
		}
	}

	_dns_client_query_release(query);
	return ret;
}

static int _dns_client_create_socket_udp(struct dns_server_info *server_info)
{
	int fd = 0;
	struct epoll_event event;
	const int on = 1;
	const int val = 255;

	fd = socket(server_info->ai_family, SOCK_DGRAM, 0);
	if (fd < 0) {
		tlog(TLOG_ERROR, "create socket failed, %s", strerror(errno));
		goto errout;
	}

	memset(&event, 0, sizeof(event));
	event.events = EPOLLIN;
	event.data.ptr = server_info;
	if (epoll_ctl(client.epoll_fd, EPOLL_CTL_ADD, fd, &event) != 0) {
		tlog(TLOG_ERROR, "epoll ctl failed.");
		return -1;
	}

	server_info->fd = fd;
	server_info->status = DNS_SERVER_STATUS_CONNECTIONLESS;
	setsockopt(server_info->fd, IPPROTO_IP, IP_RECVTTL, &on, sizeof(on));
	setsockopt(server_info->fd, SOL_IP, IP_TTL, &val, sizeof(val));
	if (server_info->ai_family == AF_INET6) {
		/* for recving ip ttl value */
		setsockopt(server_info->fd, IPPROTO_IPV6, IPV6_RECVHOPLIMIT, &on, sizeof(on));
		setsockopt(server_info->fd, IPPROTO_IPV6, IPV6_2292HOPLIMIT, &on, sizeof(on));
		setsockopt(server_info->fd, IPPROTO_IPV6, IPV6_HOPLIMIT, &on, sizeof(on));
	}

	return 0;
errout:
	if (fd > 0) {
		close(fd);
	}

	return -1;
}

static int _DNS_client_create_socket_tcp(struct dns_server_info *server_info)
{
	int fd = 0;
	struct epoll_event event;
	int yes = 1;

	fd = socket(server_info->ai_family, SOCK_STREAM, 0);
	if (fd < 0) {
		tlog(TLOG_ERROR, "create socket failed.");
		goto errout;
	}

	if (set_fd_nonblock(fd, 1) != 0) {
		tlog(TLOG_ERROR, "set socket non block failed, %s", strerror(errno));
		goto errout;
	}

	/* enable tcp fast open */
	if (setsockopt(fd, IPPROTO_TCP, TCP_FASTOPEN_CONNECT, &yes, sizeof(yes)) != 0) {
		tlog(TLOG_DEBUG, "enable TCP fast open failed.");
	}

	setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes));

	if (connect(fd, (struct sockaddr *)&server_info->addr, server_info->ai_addrlen) != 0) {
		if (errno != EINPROGRESS) {
			tlog(TLOG_ERROR, "connect failed.");
			goto errout;
		}
	}

	memset(&event, 0, sizeof(event));
	event.events = EPOLLIN | EPOLLOUT;
	event.data.ptr = server_info;
	if (epoll_ctl(client.epoll_fd, EPOLL_CTL_ADD, fd, &event) != 0) {
		tlog(TLOG_ERROR, "epoll ctl failed.");
		return -1;
	}

	server_info->fd = fd;
	server_info->status = DNS_SERVER_STATUS_CONNECTING;

	tlog(TLOG_DEBUG, "tcp server %s connecting.\n", server_info->ip);

	return 0;
errout:
	if (fd > 0) {
		close(fd);
	}

	return -1;
}

static int _DNS_client_create_socket_tls(struct dns_server_info *server_info)
{
	int fd = 0;
	struct epoll_event event;
	SSL *ssl = NULL;
	int yes = 1;

	if (server_info->ssl_ctx == NULL) {
		tlog(TLOG_ERROR, "create ssl ctx failed.");
		goto errout;
	}

	ssl = SSL_new(server_info->ssl_ctx);
	if (ssl == NULL) {
		tlog(TLOG_ERROR, "new ssl failed.");
		goto errout;
	}

	fd = socket(server_info->ai_family, SOCK_STREAM, 0);
	if (fd < 0) {
		tlog(TLOG_ERROR, "create socket failed.");
		goto errout;
	}

	if (set_fd_nonblock(fd, 1) != 0) {
		tlog(TLOG_ERROR, "set socket non block failed, %s", strerror(errno));
		goto errout;
	}

	if (setsockopt(fd, IPPROTO_TCP, TCP_FASTOPEN_CONNECT, &yes, sizeof(yes)) != 0) {
		tlog(TLOG_DEBUG, "enable TCP fast open failed.");
	}

	// ? this cause ssl crash ?
	// setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes));

	if (connect(fd, (struct sockaddr *)&server_info->addr, server_info->ai_addrlen) != 0) {
		if (errno != EINPROGRESS) {
			tlog(TLOG_ERROR, "connect failed.");
			goto errout;
		}
	}

	if (SSL_set_fd(ssl, fd) == 0) {
		tlog(TLOG_ERROR, "ssl set fd failed.");
		goto errout;
	}

	/* reuse ssl session */
	if (server_info->ssl_session) {
		SSL_set_session(ssl, server_info->ssl_session);
	}
	SSL_set_mode(ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);

	memset(&event, 0, sizeof(event));
	event.events = EPOLLIN | EPOLLOUT;
	event.data.ptr = server_info;
	if (epoll_ctl(client.epoll_fd, EPOLL_CTL_ADD, fd, &event) != 0) {
		tlog(TLOG_ERROR, "epoll ctl failed.");
		goto errout;
	}

	server_info->fd = fd;
	server_info->ssl = ssl;
	server_info->status = DNS_SERVER_STATUS_CONNECTING;

	tlog(TLOG_DEBUG, "tls server %s connecting.\n", server_info->ip);

	return 0;
errout:
	if (fd > 0) {
		close(fd);
	}

	if (ssl) {
		SSL_free(ssl);
	}

	return -1;
}

static int _dns_client_create_socket(struct dns_server_info *server_info)
{
	time(&server_info->last_send);
	time(&server_info->last_recv);

	if (server_info->fd > 0) {
		return -1;
	}

	if (server_info->type == DNS_SERVER_UDP) {
		return _dns_client_create_socket_udp(server_info);
	} else if (server_info->type == DNS_SERVER_TCP) {
		return _DNS_client_create_socket_tcp(server_info);
	} else if (server_info->type == DNS_SERVER_TLS) {
		return _DNS_client_create_socket_tls(server_info);
	} else {
		return -1;
	}

	return 0;
}

static int _dns_client_process_udp(struct dns_server_info *server_info, struct epoll_event *event, unsigned long now)
{
	int len;
	unsigned char inpacket[DNS_IN_PACKSIZE];
	struct sockaddr_storage from;
	socklen_t from_len = sizeof(from);
	char from_host[DNS_MAX_CNAME_LEN];
	struct msghdr msg;
	struct iovec iov;
	char ans_data[4096];
	int ttl = 0;
	struct cmsghdr *cmsg;

	memset(&msg, 0, sizeof(msg));
	iov.iov_base = (char *)inpacket;
	iov.iov_len = sizeof(inpacket);
	msg.msg_name = &from;
	msg.msg_namelen = sizeof(from);
	msg.msg_iov = &iov;
	msg.msg_iovlen = 1;
	msg.msg_control = ans_data;
	msg.msg_controllen = sizeof(ans_data);

	len = recvmsg(server_info->fd, &msg, MSG_DONTWAIT);
	if (len < 0) {
		tlog(TLOG_ERROR, "recvfrom failed, %s\n", strerror(errno));
		return -1;
	}
	from_len = msg.msg_namelen;

	/* Get the TTL of the IP header */
	for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
		if (cmsg->cmsg_level == SOL_IP && cmsg->cmsg_type == IP_TTL) {
			if (cmsg->cmsg_len >= sizeof(int)) {
				int *ttlPtr = (int *)CMSG_DATA(cmsg);
				ttl = *ttlPtr;
			}
		} else if (cmsg->cmsg_level == IPPROTO_IPV6 && cmsg->cmsg_type == IPV6_HOPLIMIT) {
			if (cmsg->cmsg_len >= sizeof(int)) {
				int *ttlPtr = (int *)CMSG_DATA(cmsg);
				ttl = *ttlPtr;
			}
		}
	}

	tlog(TLOG_DEBUG, "recv udp packet from %s, len: %d, ttl: %d", gethost_by_addr(from_host, sizeof(from_host), (struct sockaddr *)&from), len, ttl);

	if ((ttl != server_info->ttl) && (server_info->ttl > 0) && (server_info->result_flag & DNSSERVER_FLAG_CHECK_TTL)) {
		/* If TTL check is enabled but the TTL is inconsistent, it is considered to be a fake dns packet */
		if ((ttl < server_info->ttl - server_info->ttl_range) || (ttl > server_info->ttl + server_info->ttl_range)) {
			/* tlog(TLOG_DEBUG, "TTL mismatch, from:%d, local %d, discard result", ttl, server_info->ttl); */
			return 0;
		}
	}

	/* update recv time */
	time(&server_info->last_recv);

	/* processing dns packet */
	if (_dns_client_recv(server_info, inpacket, len, (struct sockaddr *)&from, from_len) != 0) {
		return -1;
	}

	return 0;
}

static int _dns_client_socket_ssl_send(SSL *ssl, const void *buf, int num)
{
	int ret = 0;
	int ssl_ret = 0;
	unsigned long ssl_err = 0;

	if (ssl == NULL) {
		return -1;
	}

	ret = SSL_write(ssl, buf, num);
	if (ret >= 0) {
		return ret;
	}

	ssl_ret = SSL_get_error(ssl, ret);
	switch (ssl_ret) {
	case SSL_ERROR_NONE:
		errno = EAGAIN;
		return -1;
		break;
	case SSL_ERROR_ZERO_RETURN:
		return 0;
		break;
	case SSL_ERROR_WANT_READ:
		errno = EAGAIN;
		ret = -1;
		break;
	case SSL_ERROR_WANT_WRITE:
		errno = EAGAIN;
		ret = -1;
		break;
	case SSL_ERROR_SSL:
		ssl_err = ERR_get_error();
		if (ERR_GET_REASON(ssl_err) == SSL_R_UNINITIALIZED) {
			errno = EAGAIN;
			return -1;
		}

		tlog(TLOG_ERROR, "SSL write fail error no:  %s(%ld)\n", ERR_reason_error_string(ssl_err), ssl_err);
		errno = EFAULT;
		ret = -1;
		break;
	case SSL_ERROR_SYSCALL:
		tlog(TLOG_ERROR, "SSL syscall failed, %s", strerror(errno));
		return ret;
	default:
		errno = EFAULT;
		ret = -1;
		break;
	}

	return ret;
}

static int _dns_client_socket_ssl_recv(SSL *ssl, void *buf, int num)
{
	int ret = 0;
	int ssl_ret = 0;
	unsigned long ssl_err = 0;

	if (ssl == NULL) {
		errno = EFAULT;
		return -1;
	}

	ret = SSL_read(ssl, buf, num);
	if (ret >= 0) {
		return ret;
	}

	ssl_ret = SSL_get_error(ssl, ret);
	switch (ssl_ret) {
	case SSL_ERROR_NONE:
		errno = EAGAIN;
		return -1;
		break;
	case SSL_ERROR_ZERO_RETURN:
		return 0;
		break;
	case SSL_ERROR_WANT_READ:
		errno = EAGAIN;
		ret = -1;
		break;
	case SSL_ERROR_WANT_WRITE:
		errno = EAGAIN;
		ret = -1;
		break;
	case SSL_ERROR_SSL:
		ssl_err = ERR_get_error();
		if (ERR_GET_REASON(ssl_err) == SSL_R_UNINITIALIZED) {
			errno = EAGAIN;
			return -1;
		}

		tlog(TLOG_ERROR, "SSL read fail error no: %s(%ld)\n", ERR_reason_error_string(ssl_err), ssl_err);
		errno = EFAULT;
		ret = -1;
		break;
	case SSL_ERROR_SYSCALL:
		if (errno != ECONNRESET) {
			tlog(TLOG_INFO, "SSL syscall failed, %s ", strerror(errno));
		}
		ret = -1;
		return ret;
	default:
		errno = EFAULT;
		ret = -1;
		break;
	}

	return ret;
}

static int _dns_client_socket_send(struct dns_server_info *server_info)
{
	if (server_info->type == DNS_SERVER_UDP) {
		return -1;
	} else if (server_info->type == DNS_SERVER_TCP) {
		return send(server_info->fd, server_info->send_buff.data, server_info->send_buff.len, MSG_NOSIGNAL);
	} else if (server_info->type == DNS_SERVER_TLS) {
		return _dns_client_socket_ssl_send(server_info->ssl, server_info->send_buff.data, server_info->send_buff.len);
	} else {
		return -1;
	}
}

static int _dns_client_socket_recv(struct dns_server_info *server_info)
{
	if (server_info->type == DNS_SERVER_UDP) {
		return -1;
	} else if (server_info->type == DNS_SERVER_TCP) {
		return recv(server_info->fd, server_info->recv_buff.data + server_info->recv_buff.len, DNS_TCP_BUFFER - server_info->recv_buff.len, 0);
	} else if (server_info->type == DNS_SERVER_TLS) {
		return _dns_client_socket_ssl_recv(server_info->ssl, server_info->recv_buff.data + server_info->recv_buff.len,
										   DNS_TCP_BUFFER - server_info->recv_buff.len);
	} else {
		return -1;
	}
}

static int _dns_client_process_tcp(struct dns_server_info *server_info, struct epoll_event *event, unsigned long now)
{
	int len;
	int ret = -1;
	unsigned char *inpacket_data = NULL;

	if (event->events & EPOLLIN) {
		/* receive from tcp */
		len = _dns_client_socket_recv(server_info);
		if (len < 0) {
			/* no data to recv, try again */
			if (errno == EAGAIN) {
				return 0;
			}

			/* FOR GFW */
			if (errno == ECONNRESET) {
				goto errout;
			}

			tlog(TLOG_ERROR, "recv failed, %s\n", strerror(errno));
			goto errout;
		}

		/* peer server close */
		if (len == 0) {
			pthread_mutex_lock(&client.server_list_lock);
			_dns_client_close_socket(server_info);
			server_info->recv_buff.len = 0;
			if (server_info->send_buff.len > 0) {
				/* still remain request data, reconnect and send*/
				ret = _dns_client_create_socket(server_info);
			} else {
				ret = 0;
			}
			pthread_mutex_unlock(&client.server_list_lock);
			tlog(TLOG_DEBUG, "peer close, %s", server_info->ip);
			return ret;
		}

		time(&server_info->last_recv);

		server_info->recv_buff.len += len;
		if (server_info->recv_buff.len < 2) {
			/* wait and recv */
			return 0;
		}

		while (1) {
			/* tcp result format
			 * | len (short) | dns query result |
			 */
			inpacket_data = server_info->recv_buff.data;
			len = ntohs(*((unsigned short *)(inpacket_data)));
			if (len <= 0 || len >= DNS_IN_PACKSIZE) {
				/* data len is invalid */
				goto errout;
			}

			if (len > server_info->recv_buff.len - 2) {
				/* len is not expceded, wait and recv */
				break;
			}

			inpacket_data = server_info->recv_buff.data + 2;
			tlog(TLOG_DEBUG, "recv tcp packet from %s, len = %d", server_info->ip, len);

			/* process result */
			if (_dns_client_recv(server_info, inpacket_data, len, &server_info->addr, server_info->ai_addrlen) != 0) {
				goto errout;
			}
			len += 2;
			server_info->recv_buff.len -= len;

			/* move to next result */
			if (server_info->recv_buff.len > 0) {
				memmove(server_info->recv_buff.data, server_info->recv_buff.data + len, server_info->recv_buff.len);
			} else {
				break;
			}
		}
	}

	/* when connected */
	if (event->events & EPOLLOUT) {
		struct epoll_event event;

		if (server_info->status == DNS_SERVER_STATUS_CONNECTING) {
			server_info->status = DNS_SERVER_STATUS_CONNECTED;
			tlog(TLOG_DEBUG, "tcp server %s connected", server_info->ip);
		}

		if (server_info->status != DNS_SERVER_STATUS_CONNECTED) {
			server_info->status = DNS_SERVER_STATUS_DISCONNECTED;
		}
		pthread_mutex_lock(&client.server_list_lock);
		if (server_info->send_buff.len > 0) {
			/* send existing send_buffer data  */
			len = _dns_client_socket_send(server_info);
			if (len < 0) {
				if (errno == EAGAIN) {
					pthread_mutex_unlock(&client.server_list_lock);
					return 0;
				}
				pthread_mutex_unlock(&client.server_list_lock);
				goto errout;
			}

			server_info->send_buff.len -= len;
			if (server_info->send_buff.len > 0) {
				memmove(server_info->send_buff.data, server_info->send_buff.data + len, server_info->send_buff.len);
			}
		}
		pthread_mutex_unlock(&client.server_list_lock);

		/* still remain data, retry */
		if (server_info->send_buff.len > 0) {
			return 0;
		}

		/* clear epllout event */
		memset(&event, 0, sizeof(event));
		event.events = EPOLLIN;
		event.data.ptr = server_info;
		if (epoll_ctl(client.epoll_fd, EPOLL_CTL_MOD, server_info->fd, &event) != 0) {
			tlog(TLOG_ERROR, "epoll ctl failed.");
			goto errout;
		}

		return 0;
	}

	return 0;

errout:
	pthread_mutex_lock(&client.server_list_lock);
	server_info->recv_buff.len = 0;
	server_info->send_buff.len = 0;
	_dns_client_close_socket(server_info);
	pthread_mutex_unlock(&client.server_list_lock);

	return -1;
}

static inline int _dns_client_to_hex(int c)
{
	if (c > 0x9) {
		return 'A' + c - 0xA;
	} else {
		return '0' + c;
	}
}

static int _dns_client_tls_verify(struct dns_server_info *server_info)
{
	X509 *cert = NULL;
	char peer_CN[256];
	char cert_fingerprint[256];
	int i = 0;
	int key_len = 0;
	unsigned char *key_data = NULL;
	unsigned char *key_data_tmp = NULL;
	unsigned char *key_sha256 = NULL;

	cert = SSL_get_peer_certificate(server_info->ssl);
	if (cert == NULL) {
		tlog(TLOG_ERROR, "get peer certificate failed.");
		return -1;
	}

	X509_NAME_get_text_by_NID(X509_get_subject_name(cert), NID_commonName, peer_CN, 256);
	tlog(TLOG_DEBUG, "peer CN: %s", peer_CN);

	/* get spki pin */
	key_len = i2d_X509_PUBKEY(X509_get_X509_PUBKEY(cert), NULL);
	if (key_len <= 0) {
		tlog(TLOG_ERROR, "get x509 public key failed.");
		goto errout;
	}

	key_data = OPENSSL_malloc(key_len);
	key_data_tmp = key_data;
	if (key_data == NULL) {
		tlog(TLOG_ERROR, "malloc memory failed.");
		goto errout;
	}

	i2d_X509_PUBKEY(X509_get_X509_PUBKEY(cert), &key_data_tmp);

	/* Get the SHA256 value of SPKI */
	key_sha256 = SSL_SHA256(key_data, key_len, NULL);
	if (key_sha256 == NULL) {
		tlog(TLOG_ERROR, "get sha256 failed.");
		goto errout;
	}

	char *ptr = cert_fingerprint;
	for (i = 0; i < SHA256_DIGEST_LENGTH; i++) {
		*ptr = _dns_client_to_hex(key_sha256[i] >> 4 & 0xF);
		ptr++;
		*ptr = _dns_client_to_hex(key_sha256[i] & 0xF);
		ptr++;
		*ptr = ':';
		ptr++;
	}
	ptr--;
	*ptr = 0;
	tlog(TLOG_DEBUG, "cert SPKI pin(%s): %s", "sha256", cert_fingerprint);

	if (server_info->spki) {
		/* check SPKI */
		if (memcmp(server_info->spki, key_sha256, server_info->spki_len) != 0) {
			tlog(TLOG_INFO, "server %s cert spki is invalid", server_info->ip);
			goto errout;
		} else {
			tlog(TLOG_DEBUG, "server %s cert spki verify succeed", server_info->ip);
		}
	}

	OPENSSL_free(key_data);
	X509_free(cert);
	return 0;

errout:
	if (key_data) {
		OPENSSL_free(key_data);
	}

	if (cert) {
		X509_free(cert);
	}

	return -1;
}

static int _dns_client_process_tls(struct dns_server_info *server_info, struct epoll_event *event, unsigned long now)
{
	int ret = -1;
	struct epoll_event fd_event;
	int ssl_ret;

	if (server_info->status == DNS_SERVER_STATUS_CONNECTING) {
		/* do SSL hand shake */
		ret = SSL_connect(server_info->ssl);
		if (ret == 0) {
			goto errout;
		} else if (ret < 0) {
			memset(&fd_event, 0, sizeof(fd_event));
			ssl_ret = SSL_get_error(server_info->ssl, ret);
			if (ssl_ret == SSL_ERROR_WANT_READ) {
				fd_event.events = EPOLLIN;
			} else if (ssl_ret == SSL_ERROR_WANT_WRITE) {
				fd_event.events = EPOLLOUT;
			} else {
				goto errout;
			}

			fd_event.data.ptr = server_info;
			if (epoll_ctl(client.epoll_fd, EPOLL_CTL_MOD, server_info->fd, &fd_event) != 0) {
				tlog(TLOG_ERROR, "epoll ctl failed.");
				goto errout;
			}

			return 0;
		}

		tlog(TLOG_DEBUG, "tls server %s connected.\n", server_info->ip);
		/* Was the stored session reused? */
		if (SSL_session_reused(server_info->ssl)) {
			tlog(TLOG_DEBUG, "reused session");
		} else {
			tlog(TLOG_DEBUG, "new session");
			if (server_info->ssl_session) {
				/* free session */
				SSL_SESSION_free(server_info->ssl_session);
				server_info->ssl_session = NULL;
			}

			if (_dns_client_tls_verify(server_info) != 0) {
				tlog(TLOG_WARN, "peer verify failed.");
				goto errout;
			}

			/* save ssl session for next request */
			server_info->ssl_session = SSL_get1_session(server_info->ssl);
		}

		server_info->status = DNS_SERVER_STATUS_CONNECTED;
		memset(&fd_event, 0, sizeof(fd_event));
		fd_event.events = EPOLLIN | EPOLLOUT;
		fd_event.data.ptr = server_info;
		if (epoll_ctl(client.epoll_fd, EPOLL_CTL_MOD, server_info->fd, &fd_event) != 0) {
			tlog(TLOG_ERROR, "epoll ctl failed.");
			goto errout;
		}
	}

	return _dns_client_process_tcp(server_info, event, now);
errout:
	pthread_mutex_lock(&client.server_list_lock);
	server_info->recv_buff.len = 0;
	server_info->send_buff.len = 0;
	_dns_client_close_socket(server_info);
	pthread_mutex_unlock(&client.server_list_lock);

	return -1;
}

static int _dns_client_process(struct dns_server_info *server_info, struct epoll_event *event, unsigned long now)
{
	if (server_info->type == DNS_SERVER_UDP) {
		/* receive from udp */
		return _dns_client_process_udp(server_info, event, now);
	} else if (server_info->type == DNS_SERVER_TCP) {
		/* receive from tcp */
		return _dns_client_process_tcp(server_info, event, now);
	} else if (server_info->type == DNS_SERVER_TLS) {
		/* recive from tls */
		return _dns_client_process_tls(server_info, event, now);
	} else {
		return -1;
	}

	return 0;
}

static void *_dns_client_work(void *arg)
{
	struct epoll_event events[DNS_MAX_EVENTS + 1];
	int num;
	int i;
	unsigned long now = {0};
	unsigned int sleep = 100;
	int sleep_time;
	unsigned long expect_time = 0;

	sleep_time = sleep;
	now = get_tick_count() - sleep;
	expect_time = now + sleep;
	while (client.run) {
		now = get_tick_count();
		if (now >= expect_time) {
			_dns_client_period_run();
			sleep_time = sleep - (now - expect_time);
			if (sleep_time < 0) {
				sleep_time = 0;
				expect_time = now;
			}
			expect_time += sleep;
		}

		num = epoll_wait(client.epoll_fd, events, DNS_MAX_EVENTS, sleep_time);
		if (num < 0) {
			usleep(100000);
			continue;
		}

		for (i = 0; i < num; i++) {
			struct epoll_event *event = &events[i];
			struct dns_server_info *server_info = (struct dns_server_info *)event->data.ptr;
			if (server_info == NULL) {
				tlog(TLOG_WARN, "server info is invalid.");
				continue;
			}

			_dns_client_process(server_info, event, now);
		}
	}

	close(client.epoll_fd);
	client.epoll_fd = -1;

	return NULL;
}

static int _dns_client_send_udp(struct dns_server_info *server_info, void *packet, int len)
{
	int send_len = 0;
	if (server_info->fd <= 0) {
		return -1;
	}

	send_len = sendto(server_info->fd, packet, len, 0, (struct sockaddr *)&server_info->addr, server_info->ai_addrlen);
	if (send_len != len) {
		return -1;
	}

	return 0;
}

static int _dns_client_send_data_to_buffer(struct dns_server_info *server_info, void *packet, int len)
{
	struct epoll_event event;

	if (DNS_TCP_BUFFER - server_info->send_buff.len < len) {
		errno = ENOMEM;
		return -1;
	}

	memcpy(server_info->send_buff.data + server_info->send_buff.len, packet, len);
	server_info->send_buff.len += len;

	memset(&event, 0, sizeof(event));
	event.events = EPOLLIN | EPOLLOUT;
	event.data.ptr = server_info;
	if (epoll_ctl(client.epoll_fd, EPOLL_CTL_MOD, server_info->fd, &event) != 0) {
		tlog(TLOG_ERROR, "epoll ctl failed.");
		return -1;
	}

	return 0;
}

static int _dns_client_send_tcp(struct dns_server_info *server_info, void *packet, unsigned short len)
{
	int send_len = 0;
	unsigned char inpacket_data[DNS_IN_PACKSIZE];
	unsigned char *inpacket = inpacket_data;

	if (len > sizeof(inpacket_data) - 2) {
		tlog(TLOG_ERROR, "packet size is invalid.");
		return -1;
	}

	/* TCP query format
	 * | len (short) | dns query data |
	 */
	*((unsigned short *)(inpacket)) = htons(len);
	memcpy(inpacket + 2, packet, len);
	len += 2;

	if (server_info->status != DNS_SERVER_STATUS_CONNECTED) {
		return _dns_client_send_data_to_buffer(server_info, inpacket, len);
	}

	if (server_info->fd <= 0) {
		return -1;
	}

	send_len = send(server_info->fd, inpacket, len, MSG_NOSIGNAL);
	if (send_len < 0) {
		if (errno == EAGAIN) {
			/* save data to buffer, and retry when EPOLLOUT is available */
			return _dns_client_send_data_to_buffer(server_info, inpacket, len);
		} else if (errno == EPIPE) {
			shutdown(server_info->fd, SHUT_RDWR);
		}
		return -1;
	} else if (send_len < len) {
		/* save remain data to buffer, and retry when EPOLLOUT is available */
		return _dns_client_send_data_to_buffer(server_info, inpacket + send_len, len - send_len);
	}

	return 0;
}

static int _dns_client_send_tls(struct dns_server_info *server_info, void *packet, unsigned short len)
{
	int send_len = 0;
	unsigned char inpacket_data[DNS_IN_PACKSIZE];
	unsigned char *inpacket = inpacket_data;

	if (len > sizeof(inpacket_data) - 2) {
		tlog(TLOG_ERROR, "packet size is invalid.");
		return -1;
	}

	/* TCP query format
	 * | len (short) | dns query data |
	 */
	*((unsigned short *)(inpacket)) = htons(len);
	memcpy(inpacket + 2, packet, len);
	len += 2;

	if (server_info->status != DNS_SERVER_STATUS_CONNECTED) {
		return _dns_client_send_data_to_buffer(server_info, inpacket, len);
	}

	if (server_info->ssl == NULL) {
		return -1;
	}

	send_len = _dns_client_socket_ssl_send(server_info->ssl, inpacket, len);
	if (send_len < 0) {
		if (errno == EAGAIN || server_info->ssl == NULL) {
			/* save data to buffer, and retry when EPOLLOUT is available */
			return _dns_client_send_data_to_buffer(server_info, inpacket, len);
		} else if (server_info->ssl && errno != ENOMEM) {
			SSL_shutdown(server_info->ssl);
		}
		return -1;
	} else if (send_len < len) {
		/* save remain data to buffer, and retry when EPOLLOUT is available */
		return _dns_client_send_data_to_buffer(server_info, inpacket + send_len, len - send_len);
	}

	return 0;
}

static int _dns_client_send_packet(struct dns_query_struct *query, void *packet, int len)
{
	struct dns_server_info *server_info = NULL;
	struct dns_server_group_member *group_member = NULL;
	struct dns_server_group_member *tmp = NULL;
	int ret = 0;
	int send_err = 0;

	query->send_tick = get_tick_count();

	/* send query to all dns servers */
	pthread_mutex_lock(&client.server_list_lock);
	list_for_each_entry_safe(group_member, tmp, &query->server_group->head, list)
	{
		server_info = group_member->server;
		if (server_info->fd <= 0) {
			ret = _dns_client_create_socket(server_info);
			if (ret != 0) {
				continue;
			}
		}

		atomic_inc(&query->dns_request_sent);
		switch (server_info->type) {
		case DNS_SERVER_UDP:
			/* udp query */
			ret = _dns_client_send_udp(server_info, packet, len);
			send_err = errno;
			break;
		case DNS_SERVER_TCP:
			/* tcp query */
			ret = _dns_client_send_tcp(server_info, packet, len);
			send_err = errno;
			break;
		case DNS_SERVER_TLS:
			/* tls query */
			ret = _dns_client_send_tls(server_info, packet, len);
			send_err = errno;
			break;
		default:
			/* unsupport query type */
			ret = -1;
			break;
		}

		if (ret != 0) {
			if (send_err != ENOMEM) {
				tlog(TLOG_ERROR, "send query to %s failed, %s, type: %d", server_info->ip, strerror(send_err), server_info->type);
			} else {
				tlog(TLOG_DEBUG, "send query to %s failed, %s, type: %d", server_info->ip, strerror(send_err), server_info->type);
			}
			atomic_dec(&query->dns_request_sent);
			continue;
		}
		time(&server_info->last_send);
	}
	pthread_mutex_unlock(&client.server_list_lock);
	return 0;
}

static int _dns_client_dns_add_ecs(struct dns_packet *packet, int qtype)
{
	if (qtype == DNS_T_A && client.ecs_ipv4.enable) {
		struct dns_opt_ecs ecs;
		ecs.family = DNS_ADDR_FAMILY_IP;
		ecs.source_prefix = client.ecs_ipv4.bitlen;
		ecs.scope_prefix = 0;
		memcpy(ecs.addr, client.ecs_ipv4.ipv4_addr, DNS_RR_A_LEN);
		return dns_add_OPT_ECS(packet, &ecs);
	} else if (qtype == DNS_T_AAAA && client.ecs_ipv6.enable) {
		struct dns_opt_ecs ecs;
		ecs.family = DNS_ADDR_FAMILY_IPV6;
		ecs.source_prefix = client.ecs_ipv6.bitlen;
		ecs.scope_prefix = 0;
		memcpy(ecs.addr, client.ecs_ipv6.ipv6_addr, DNS_RR_AAAA_LEN);
		return dns_add_OPT_ECS(packet, &ecs);
	}
	return 0;
}

static int _dns_client_send_query(struct dns_query_struct *query, char *doamin)
{
	unsigned char packet_buff[DNS_PACKSIZE];
	unsigned char inpacket[DNS_IN_PACKSIZE];
	struct dns_packet *packet = (struct dns_packet *)packet_buff;
	int encode_len;

	/* init dns packet head */
	struct dns_head head;
	memset(&head, 0, sizeof(head));
	head.id = query->sid;
	head.qr = DNS_QR_QUERY;
	head.opcode = DNS_OP_QUERY;
	head.aa = 0;
	head.rd = 1;
	head.ra = 0;
	head.rcode = 0;

	if (dns_packet_init(packet, DNS_PACKSIZE, &head) != 0) {
		tlog(TLOG_ERROR, "init packet failed.");
		return -1;
	}

	/* add question */
	if (dns_add_domain(packet, doamin, query->qtype, DNS_C_IN) != 0) {
		tlog(TLOG_ERROR, "add domain to packet failed.");
		return -1;
	}

	dns_set_OPT_payload_size(packet, DNS_IN_PACKSIZE);
	/* dns_add_OPT_TCP_KEEYALIVE(packet, 600); */
	if (_dns_client_dns_add_ecs(packet, query->qtype) != 0) {
		tlog(TLOG_ERROR, "add ecs failed.");
		return -1;
	}

	/* encode packet */
	encode_len = dns_encode(inpacket, DNS_IN_PACKSIZE, packet);
	if (encode_len <= 0) {
		tlog(TLOG_ERROR, "encode query failed.");
		return -1;
	}

	if (encode_len > DNS_IN_PACKSIZE) {
		tlog(TLOG_ERROR, "size is invalid.");
		abort();
		return -1;
	}

	/* send query packet */
	return _dns_client_send_packet(query, inpacket, encode_len);
}

int dns_client_query(char *domain, int qtype, dns_client_callback callback, void *user_ptr, const char *group_name)
{
	struct dns_query_struct *query = NULL;
	int ret = 0;
	uint32_t key = 0;

	query = malloc(sizeof(*query));
	if (query == NULL) {
		goto errout;
	}
	memset(query, 0, sizeof(*query));

	INIT_HLIST_NODE(&query->domain_node);
	INIT_LIST_HEAD(&query->dns_request_list);
	atomic_set(&query->refcnt, 0);
	atomic_set(&query->dns_request_sent, 0);
	hash_init(query->replied_map);
	strncpy(query->domain, domain, DNS_MAX_CNAME_LEN);
	query->user_ptr = user_ptr;
	query->callback = callback;
	query->qtype = qtype;
	query->send_tick = 0;
	query->sid = atomic_inc_return(&dns_client_sid);
	query->server_group = _dns_client_get_dnsserver_group(group_name);
	if (query->server_group == NULL) {
		tlog(TLOG_ERROR, "get dns server group %s failed.", group_name);
		goto errout;
	}

	_dns_client_query_get(query);
	/* add query to hashtable */
	key = hash_string(domain);
	key = jhash(&query->sid, sizeof(query->sid), key);
	pthread_mutex_lock(&client.domain_map_lock);
	hash_add(client.domain_map, &query->domain_node, key);
	pthread_mutex_unlock(&client.domain_map_lock);

	/* send query */
	_dns_client_query_get(query);
	ret = _dns_client_send_query(query, domain);
	if (ret != 0) {
		_dns_client_query_release(query);
		goto errout_del_list;
	}

	pthread_mutex_lock(&client.domain_map_lock);
	list_add_tail(&query->dns_request_list, &client.dns_request_list);
	pthread_mutex_unlock(&client.domain_map_lock);

	tlog(TLOG_INFO, "send request %s, qtype %d, id %d\n", domain, qtype, query->sid);
	_dns_client_query_release(query);

	return 0;
errout_del_list:
	query->callback = NULL;
	_dns_client_query_remove(query);
	query = NULL;
errout:
	if (query) {
		tlog(TLOG_ERROR, "release %p", query);
		free(query);
	}
	return -1;
}

int dns_client_set_ecs(char *ip, int subnet)
{
	return 0;
}

int dns_client_init(void)
{
	pthread_attr_t attr;
	int epollfd = -1;
	int ret;

	if (client.epoll_fd > 0) {
		return -1;
	}

	memset(&client, 0, sizeof(client));
	pthread_attr_init(&attr);
	atomic_set(&client.dns_server_num, 0);

	epollfd = epoll_create1(EPOLL_CLOEXEC);
	if (epollfd < 0) {
		tlog(TLOG_ERROR, "create epoll failed, %s\n", strerror(errno));
		goto errout;
	}

	pthread_mutex_init(&client.server_list_lock, NULL);
	INIT_LIST_HEAD(&client.dns_server_list);

	pthread_mutex_init(&client.domain_map_lock, NULL);
	hash_init(client.domain_map);
	hash_init(client.group);
	INIT_LIST_HEAD(&client.dns_request_list);

	if (dns_client_add_group(DNS_SERVER_GROUP_DEFAULT) != 0) {
		tlog(TLOG_ERROR, "add default server group failed.");
		goto errout;
	}

	client.default_group = _dns_client_get_group(DNS_SERVER_GROUP_DEFAULT);
	client.epoll_fd = epollfd;
	client.run = 1;

	/* start work task */
	ret = pthread_create(&client.tid, &attr, _dns_client_work, NULL);
	if (ret != 0) {
		tlog(TLOG_ERROR, "create client work thread failed, %s\n", strerror(errno));
		goto errout;
	}

	return 0;
errout:
	if (client.tid > 0) {
		void *retval = NULL;
		client.run = 0;
		pthread_join(client.tid, &retval);
	}

	if (epollfd) {
		close(epollfd);
	}

	pthread_mutex_destroy(&client.server_list_lock);
	pthread_mutex_destroy(&client.domain_map_lock);

	return -1;
}

void dns_client_exit(void)
{
	if (client.tid > 0) {
		void *ret = NULL;
		client.run = 0;
		pthread_join(client.tid, &ret);
	}

	/* free all resouces */
	_dns_client_server_remove_all();
	_dns_client_query_remove_all();
	_dns_client_group_remove_all();

	pthread_mutex_destroy(&client.server_list_lock);
	pthread_mutex_destroy(&client.domain_map_lock);
}