cpu-usage: reduce cpu usage when idle

.gitignore

@@ -2,3 +2,4 @@
 *.o
 .DS_Store
 *.swp.
+systemd/smartdns.service

src/dns_client.c

@@ -184,8 +184,8 @@ struct dns_client {
 	int ssl_verify_skip;
 
 	/* query list */
-	pthread_mutex_t dns_request_lock;
 	struct list_head dns_request_list;
+	pthread_cond_t run_cond;
 	atomic_t dns_server_num;
 
 	/* ECS */
@@ -3129,6 +3129,10 @@ int dns_client_query(const char *domain, int qtype, dns_client_callback callback
 	}
 
 	pthread_mutex_lock(&client.domain_map_lock);
+	if (list_empty(&client.dns_request_list)) {
+		pthread_cond_signal(&client.run_cond);
+	}
+
 	list_add_tail(&query->dns_request_list, &client.dns_request_list);
 	pthread_mutex_unlock(&client.domain_map_lock);
 
@@ -3402,15 +3406,25 @@ static void *_dns_client_work(void *arg)
 	int num = 0;
 	int i = 0;
 	unsigned long now = {0};
+	unsigned long last = {0};
 	unsigned int sleep = 100;
 	int sleep_time = 0;
 	unsigned long expect_time = 0;
 
 	sleep_time = sleep;
 	now = get_tick_count() - sleep;
+	last = now;
 	expect_time = now + sleep;
 	while (atomic_read(&client.run)) {
 		now = get_tick_count();
+		if (sleep_time > 0) {
+			sleep_time -= now - last;
+			if (sleep_time <= 0) {
+				sleep_time = 0;
+			}
+		}
+		last = now;
+
 		if (now >= expect_time) {
 			_dns_client_period_run();
 			sleep_time = sleep - (now - expect_time);
@@ -3421,6 +3435,15 @@ static void *_dns_client_work(void *arg)
 			expect_time += sleep;
 		}
 
+		pthread_mutex_lock(&client.domain_map_lock);
+		if (list_empty(&client.dns_request_list)) {
+			pthread_cond_wait(&client.run_cond, &client.domain_map_lock);
+			expect_time = get_tick_count();
+			pthread_mutex_unlock(&client.domain_map_lock);
+			continue;
+		}
+		pthread_mutex_unlock(&client.domain_map_lock);
+
 		num = epoll_wait(client.epoll_fd, events, DNS_MAX_EVENTS, sleep_time);
 		if (num < 0) {
 			usleep(100000);
@@ -3511,6 +3534,8 @@ int dns_client_init(void)
 	hash_init(client.group);
 	INIT_LIST_HEAD(&client.dns_request_list);
 
+	pthread_cond_init(&client.run_cond, NULL);
+
 	if (dns_client_add_group(DNS_SERVER_GROUP_DEFAULT) != 0) {
 		tlog(TLOG_ERROR, "add default server group failed.");
 		goto errout;
@@ -3542,6 +3567,7 @@ errout:
 
 	pthread_mutex_destroy(&client.server_list_lock);
 	pthread_mutex_destroy(&client.domain_map_lock);
+	pthread_cond_destroy(&client.run_cond);
 
 	return -1;
 }
@@ -3551,6 +3577,9 @@ void dns_client_exit(void)
 	if (client.tid) {
 		void *ret = NULL;
 		atomic_set(&client.run, 0);
+		pthread_mutex_lock(&client.domain_map_lock);
+		pthread_cond_signal(&client.run_cond);
+		pthread_mutex_unlock(&client.domain_map_lock);
 		pthread_join(client.tid, &ret);
 		client.tid = 0;
 	}
@@ -3563,6 +3592,7 @@ void dns_client_exit(void)
 
 	pthread_mutex_destroy(&client.server_list_lock);
 	pthread_mutex_destroy(&client.domain_map_lock);
+	pthread_cond_destroy(&client.run_cond);
 	if (client.ssl_ctx) {
 		SSL_CTX_free(client.ssl_ctx);
 		client.ssl_ctx = NULL;

src/dns_server.c

@@ -40,6 +40,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/epoll.h>
+#include <sys/eventfd.h>
 #include <sys/socket.h>
 #include <sys/types.h>
 
@@ -246,6 +247,7 @@ struct dns_request {
 struct dns_server {
 	atomic_t run;
 	int epoll_fd;
+	int event_fd;
 	struct list_head conn_list;
 
 	/* dns request list */
@@ -269,7 +271,14 @@ static void _dns_server_request_get(struct dns_request *request);
 static void _dns_server_request_release(struct dns_request *request);
 static void _dns_server_request_release_complete(struct dns_request *request, int do_complete);
 static int _dns_server_reply_passthrouth(struct dns_server_post_context *context);
-static int _dns_server_do_query(struct dns_request *request);
+static int _dns_server_do_query(struct dns_request *request, int skip_notify_event);
+
+static void _dns_server_wakup_thread(void)
+{
+	uint64_t u = 1;
+	int unused __attribute__((unused));
+	unused = write(server.event_fd, &u, sizeof(u));
+}
 
 static int _dns_server_forward_request(unsigned char *inpacket, int inpacket_len)
 {
@@ -4054,7 +4063,7 @@ static int _dns_server_query_dualstack(struct dns_request *request)
 	request_dualstack->dualstack_request = request;
 	_dns_server_request_set_callback(request_dualstack, dns_server_dualstack_callback, request);
 	request->request_wait++;
-	ret = _dns_server_do_query(request_dualstack);
+	ret = _dns_server_do_query(request_dualstack, 0);
 	if (ret != 0) {
 		request->request_wait--;
 		tlog(TLOG_ERROR, "do query %s type %d failed.\n", request->domain, qtype);
@@ -4074,7 +4083,7 @@ errout:
 	return ret;
 }
 
-static int _dns_server_do_query(struct dns_request *request)
+static int _dns_server_do_query(struct dns_request *request, int skip_notify_event)
 {
 	int ret = -1;
 	const char *group_name = NULL;
@@ -4150,6 +4159,9 @@ static int _dns_server_do_query(struct dns_request *request)
 	_dns_server_setup_query_option(request, &options);
 
 	pthread_mutex_lock(&server.request_list_lock);
+	if (list_empty(&server.request_list) && skip_notify_event == 1) {
+		_dns_server_wakup_thread();
+	}
 	list_add_tail(&request->list, &server.request_list);
 	pthread_mutex_unlock(&server.request_list_lock);
 
@@ -4275,7 +4287,7 @@ static int _dns_server_recv(struct dns_server_conn_head *conn, unsigned char *in
 	_dns_server_request_set_client(request, conn);
 	_dns_server_request_set_client_addr(request, from, from_len);
 	_dns_server_request_set_id(request, packet->head.id);
-	ret = _dns_server_do_query(request);
+	ret = _dns_server_do_query(request, 1);
 	if (ret != 0) {
 		tlog(TLOG_ERROR, "do query %s failed.\n", request->domain);
 		goto errout;
@@ -4327,7 +4339,7 @@ static int _dns_server_prefetch_request(char *domain, dns_type_t qtype, int expi
 	request->qtype = qtype;
 	_dns_server_setup_server_query_options(request, server_query_option);
 	_dns_server_request_set_enable_prefetch(request, expired_domain);
-	ret = _dns_server_do_query(request);
+	ret = _dns_server_do_query(request, 0);
 	if (ret != 0) {
 		tlog(TLOG_ERROR, "do query %s failed.\n", request->domain);
 		goto errout;
@@ -4359,7 +4371,7 @@ int dns_server_query(const char *domain, int qtype, struct dns_server_query_opti
 	request->qtype = qtype;
 	_dns_server_setup_server_query_options(request, server_query_option);
 	_dns_server_request_set_callback(request, callback, user_ptr);
-	ret = _dns_server_do_query(request);
+	ret = _dns_server_do_query(request, 0);
 	if (ret != 0) {
 		tlog(TLOG_ERROR, "do query %s failed.\n", domain);
 		goto errout;
@@ -4865,14 +4877,12 @@ static void _dns_server_period_run_second(void)
 	}
 }
 
-static void _dns_server_period_run(void)
+static void _dns_server_period_run(unsigned int msec)
 {
 	struct dns_request *request = NULL;
 	struct dns_request *tmp = NULL;
-	static unsigned int msec = 0;
 	LIST_HEAD(check_list);
 
-	msec++;
 	if (msec % 10 == 0) {
 		_dns_server_period_run_second();
 	}
@@ -4940,22 +4950,50 @@ int dns_server_run(void)
 	int num = 0;
 	int i = 0;
 	unsigned long now = {0};
+	unsigned long last = {0};
+	unsigned int msec = 0;
 	int sleep = 100;
 	int sleep_time = 0;
 	unsigned long expect_time = 0;
 
 	sleep_time = sleep;
 	now = get_tick_count() - sleep;
+	last = now;
 	expect_time = now + sleep;
 	while (atomic_read(&server.run)) {
 		now = get_tick_count();
+		if (sleep_time > 0) {
+			sleep_time -= now - last;
+			if (sleep_time <= 0) {
+				sleep_time = 0;
+			}
+
+			int cnt = sleep_time / sleep;
+			msec -= cnt;
+			expect_time -= cnt * sleep;
+			sleep_time -= cnt * sleep;
+		}
+		last = now;
+
 		if (now >= expect_time) {
-			_dns_server_period_run();
+			msec++;
+			_dns_server_period_run(msec);
 			sleep_time = sleep - (now - expect_time);
 			if (sleep_time < 0) {
 				sleep_time = 0;
 				expect_time = now;
 			}
+
+			/* When server is idle, the sleep time is 1000ms, to reduce CPU usage */
+			pthread_mutex_lock(&server.request_list_lock);
+			if (list_empty(&server.request_list)) {
+				int cnt = 10 - (msec % 10) - 1;
+				sleep_time += sleep * cnt;
+				msec += cnt;
+				/* sleep to next second */
+				expect_time += sleep * cnt;
+			}
+			pthread_mutex_unlock(&server.request_list_lock);
 			expect_time += sleep;
 		}
 
@@ -4971,6 +5009,14 @@ int dns_server_run(void)
 
 		for (i = 0; i < num; i++) {
 			struct epoll_event *event = &events[i];
+			/* read event */
+			if (event->data.fd == server.event_fd) {
+				uint64_t value;
+				int unused __attribute__((unused));
+				unused = read(server.event_fd, &value, sizeof(uint64_t));
+				continue;
+			}
+
 			struct dns_server_conn_head *conn_head = event->data.ptr;
 			if (conn_head == NULL) {
 				tlog(TLOG_ERROR, "invalid fd\n");
@@ -5294,6 +5340,31 @@ static int _dns_server_cache_save(void)
 	return 0;
 }
 
+static int _dns_server_init_wakeup_event(void)
+{
+	int fdevent = -1;
+	fdevent = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
+	if (fdevent < 0) {
+		tlog(TLOG_ERROR, "create eventfd failed, %s\n", strerror(errno));
+		goto errout;
+	}
+
+	struct epoll_event event;
+	memset(&event, 0, sizeof(event));
+	event.events = EPOLLIN | EPOLLERR;
+	event.data.fd = fdevent;
+	if (epoll_ctl(server.epoll_fd, EPOLL_CTL_ADD, fdevent, &event) != 0) {
+		tlog(TLOG_ERROR, "set eventfd failed, %s\n", strerror(errno));
+		goto errout;
+	}
+
+	server.event_fd = fdevent;
+
+	return 0;
+errout:
+	return -1;
+}
+
 int dns_server_init(void)
 {
 	pthread_attr_t attr;
@@ -5344,6 +5415,11 @@ int dns_server_init(void)
 	tlog(TLOG_INFO, "%s",
 		 (is_ipv6_ready) ? "IPV6 is ready, enable IPV6 features" : "IPV6 is not ready, disable IPV6 features");
 
+	if (_dns_server_init_wakeup_event() != 0) {
+		tlog(TLOG_ERROR, "init wakeup event failed.");
+		goto errout;
+	}
+
 	return 0;
 errout:
 	atomic_set(&server.run, 0);
@@ -5363,10 +5439,15 @@ errout:
 void dns_server_stop(void)
 {
 	atomic_set(&server.run, 0);
+	_dns_server_wakup_thread();
 }
 
 void dns_server_exit(void)
 {
+	if (server.event_fd > 0) {
+		close(server.event_fd);
+		server.event_fd = -1;
+	}
 	_dns_server_close_socket();
 	_dns_server_cache_save();
 	_dns_server_request_remove_all();

src/fast_ping.c

@@ -36,6 +36,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/epoll.h>
+#include <sys/eventfd.h>
 #include <sys/socket.h>
 #include <sys/types.h>
 #include <unistd.h>
@@ -140,6 +141,8 @@ struct fast_ping_struct {
 	int fd_udp6;
 	struct ping_host_struct udp6_host;
 
+	int event_fd;
+
 	pthread_mutex_t map_lock;
 	DECLARE_HASHTABLE(addrmap, 6);
 };
@@ -148,6 +151,13 @@ static struct fast_ping_struct ping;
 static atomic_t ping_sid = ATOMIC_INIT(0);
 static int bool_print_log = 1;
 
+static void _fast_ping_wakup_thread(void)
+{
+	uint64_t u = 1;
+	int unused __attribute__((unused));
+	unused = write(ping.event_fd, &u, sizeof(u));
+}
+
 static uint16_t _fast_ping_checksum(uint16_t *header, size_t len)
 {
 	uint32_t sum = 0;
@@ -1122,6 +1132,9 @@ struct ping_host_struct *fast_ping_start(PING_TYPE type, const char *host, int c
 
 	pthread_mutex_lock(&ping.map_lock);
 	_fast_ping_host_get(ping_host);
+	if (hash_empty(ping.addrmap)) {
+		_fast_ping_wakup_thread();
+	}
 	hash_add(ping.addrmap, &ping_host->addr_node, addrkey);
 	ping_host->run = 1;
 	pthread_mutex_unlock(&ping.map_lock);
@@ -1627,6 +1640,7 @@ static void *_fast_ping_work(void *arg)
 	int num = 0;
 	int i = 0;
 	unsigned long now = {0};
+	unsigned long last = {0};
 	struct timeval tvnow = {0};
 	int sleep = 100;
 	int sleep_time = 0;
@@ -1634,9 +1648,18 @@ static void *_fast_ping_work(void *arg)
 
 	sleep_time = sleep;
 	now = get_tick_count() - sleep;
+	last = now;
 	expect_time = now + sleep;
 	while (atomic_read(&ping.run)) {
 		now = get_tick_count();
+		if (sleep_time > 0) {
+			sleep_time -= now - last;
+			if (sleep_time <= 0) {
+				sleep_time = 0;
+			}
+		}
+		last = now;
+
 		if (now >= expect_time) {
 			_fast_ping_period_run();
 			sleep_time = sleep - (now - expect_time);
@@ -1647,12 +1670,22 @@ static void *_fast_ping_work(void *arg)
 			expect_time += sleep;
 		}
 
+		pthread_mutex_lock(&ping.map_lock);
+		if (hash_empty(ping.addrmap)) {
+			sleep_time = -1;
+		}
+		pthread_mutex_unlock(&ping.map_lock);
+
 		num = epoll_wait(ping.epoll_fd, events, PING_MAX_EVENTS, sleep_time);
 		if (num < 0) {
 			usleep(100000);
 			continue;
 		}
 
+		if (sleep_time == -1) {
+			expect_time = get_tick_count();
+		}
+
 		if (num == 0) {
 			continue;
 		}
@@ -1660,6 +1693,14 @@ static void *_fast_ping_work(void *arg)
 		gettimeofday(&tvnow, NULL);
 		for (i = 0; i < num; i++) {
 			struct epoll_event *event = &events[i];
+			/* read event */
+			if (event->data.fd == ping.event_fd) {
+				uint64_t value;
+				int unused __attribute__((unused));
+				unused = read(ping.event_fd, &value, sizeof(uint64_t));
+				continue;
+			}
+
 			struct ping_host_struct *ping_host = (struct ping_host_struct *)event->data.ptr;
 			_fast_ping_process(ping_host, event, &tvnow);
 		}
@@ -1671,6 +1712,31 @@ static void *_fast_ping_work(void *arg)
 	return NULL;
 }
 
+static int _fast_ping_init_wakeup_event(void)
+{
+	int fdevent = -1;
+	fdevent = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
+	if (fdevent < 0) {
+		tlog(TLOG_ERROR, "create eventfd failed, %s\n", strerror(errno));
+		goto errout;
+	}
+
+	struct epoll_event event;
+	memset(&event, 0, sizeof(event));
+	event.events = EPOLLIN | EPOLLERR;
+	event.data.fd = fdevent;
+	if (epoll_ctl(ping.epoll_fd, EPOLL_CTL_ADD, fdevent, &event) != 0) {
+		tlog(TLOG_ERROR, "set eventfd failed, %s\n", strerror(errno));
+		goto errout;
+	}
+
+	ping.event_fd = fdevent;
+
+	return 0;
+errout:
+	return -1;
+}
+
 int fast_ping_init(void)
 {
 	pthread_attr_t attr;
@@ -1694,7 +1760,6 @@ int fast_ping_init(void)
 	pthread_mutex_init(&ping.map_lock, NULL);
 	pthread_mutex_init(&ping.lock, NULL);
 	hash_init(ping.addrmap);
-	ping.epoll_fd = epollfd;
 	ping.no_unprivileged_ping = !has_unprivileged_ping();
 	ping.ident = (getpid() & 0XFFFF);
 	atomic_set(&ping.run, 1);
@@ -1704,6 +1769,13 @@ int fast_ping_init(void)
 		goto errout;
 	}
 
+	ping.epoll_fd = epollfd;
+	ret = _fast_ping_init_wakeup_event();
+	if (ret != 0) {
+		tlog(TLOG_ERROR, "init wakeup event failed, %s\n", strerror(errno));
+		goto errout;
+	}
+
 	return 0;
 errout:
 	if (ping.tid) {
@@ -1713,12 +1785,18 @@ errout:
 		ping.tid = 0;
 	}
 
-	if (epollfd) {
+	if (epollfd > 0) {
 		close(epollfd);
 	}
 
+	if (ping.event_fd) {
+		close(ping.event_fd);
+		ping.event_fd = -1;
+	}
+
 	pthread_mutex_destroy(&ping.lock);
 	pthread_mutex_destroy(&ping.map_lock);
+	memset(&ping, 0, sizeof(ping));
 
 	return -1;
 }
@@ -1751,10 +1829,16 @@ void fast_ping_exit(void)
 	if (ping.tid) {
 		void *ret = NULL;
 		atomic_set(&ping.run, 0);
+		_fast_ping_wakup_thread();
 		pthread_join(ping.tid, &ret);
 		ping.tid = 0;
 	}
 
+	if (ping.event_fd > 0) {
+		close(ping.event_fd);
+		ping.event_fd = -1;
+	}
+
 	_fast_ping_close_fds();
 	_fast_ping_remove_all();