本文介绍了Linux C++网络编程中使用Muduo库实现基于epoll的事件驱动模型。讲解了epoll的由来、EventLoop、Channel、Poller、Acceptor、TcpServer和TcpConnection等关键组件，阐述了它们在处理网络连接、数据读写中的作用。
前言
在上一节中以拆分的方式学习完 Linux 、C++、网络等知识后，这节会将这三个模块糅合起来，站在项目的基础上再次去学习这三个模块。Linux 网路编程比较经典的有 Redis、Muduo、TeamTalk等开源项目。本文将以 Muduo 来介绍 网络编程的框架，学习完 Muduo 再去学习其他框架就容易许多了。注意， 文中贴的代码均为核心代码，且注释详细，认真阅读。Linux C++ 网络编程 (一)Linux C++ 开发环境搭建下载路径
github 地址：https://github.com/smilew12/muduo.git项目结构
项目主要分为两个模块：base 模块：主要封装互斥锁、条件变量、线程池、日志等基础类；net 模块：主要根据 reactor 模型对 Linux 平台下 Epoll 的封装；本文主要介绍服务器模型，所以只引导读者学习 net 模块， base 模块主要使用 RAII 技法封装的常用类。epoll 由来
先来说说传统的迭代式服务器模型以及一个连接一个线程的模型的缺点，以下图一个连接一个线程为例说明：
- 我将我想要监听的链接、读写等事件全权委托给它；
- 有连接到来事件的时候通知我？我直接调用 accept 函数去接收。
- 有数据到来的时候也通知我？ 我直接调用 recv 函数去收取数据。
- 发送缓冲区有空间时也通知我？我直接调用 send 函数去发送数据。
| epoll_create() | 函数用来创建一个代理对象； |
| epoll_wait() | 函数就是当有事件到来时，会返回响应事件的代理； |
| epoll_ctl() | 主要是往这个代理中注册你想要监听的事件； |
//头文件#ifndef __MYREACTOR_H__#define __MYREACTOR_H__#include #include #include #include #include #define WORKER_THREAD_NUM 5class CMyReactor
{public:
CMyReactor();
~CMyReactor();bool init(const char* ip, short nport);bool uninit();bool close_client(int clientfd);static void* main_loop(void* p);private://no copyable
CMyReactor(const CMyReactor& rhs);
CMyReactor& operator = (const CMyReactor& rhs);bool create_server_listener(const char* ip, short port);static void accept_thread_proc(CMyReactor* pReatcor);static void worker_thread_proc(CMyReactor* pReatcor);private://C11语法可以在这里初始化int m_listenfd = 0;int m_epollfd = 0;bool m_bStop = false;std::shared_ptr<std::thread> m_acceptthread;std::shared_ptr<std::thread> m_workerthreads[WORKER_THREAD_NUM];std::condition_variable m_acceptcond;std::mutex m_acceptmutex;std::condition_variable m_workercond ;std::mutex m_workermutex;std::list<int> m_listClients;
};#endif //!__MYREACTOR_H__//.cpp文件#include "myreactor.h"#include #include #include #include #include #include //for htonl() and htons()#include #include #include #include #include #include #include //for std::setw()/setfill()#include #define min(a, b) ((a <= b) ? (a) : (b))
CMyReactor::CMyReactor()
{//m_listenfd = 0;//m_epollfd = 0;//m_bStop = false;
}
CMyReactor::~CMyReactor()
{
}bool CMyReactor::init(const char* ip, short nport)
{//创建监听socket，并将监听socket挂载到 epoll 上if (!create_server_listener(ip, nport))
{std::cout <"Unable to bind: " <":" <"." <std::endl;return false;
}//打印当前线程idstd::cout <"main thread id = " <std::this_thread::get_id() <std::endl;//启动接收新连接的线程
m_acceptthread.reset(new std::thread(CMyReactor::accept_thread_proc, this));//启动工作线程(收发数据)for (auto& t : m_workerthreads)
{
t.reset(new std::thread(CMyReactor::worker_thread_proc, this));
}return true;
}//释放资源bool CMyReactor::uninit()
{
m_bStop = true;
m_acceptcond.notify_one();
m_workercond.notify_all();
m_acceptthread->join();for (auto& t : m_workerthreads)
{
t->join();
}
::epoll_ctl(m_epollfd, EPOLL_CTL_DEL, m_listenfd, NULL);//TODO: 是否需要先调用shutdown()一下？
::shutdown(m_listenfd, SHUT_RDWR);
::close(m_listenfd);
::close(m_epollfd);return true;
}bool CMyReactor::close_client(int clientfd)
{if (::epoll_ctl(m_epollfd, EPOLL_CTL_DEL, clientfd, NULL) == -1)
{std::cout <"close client socket failed as call epoll_ctl failed" <std::endl;//return false;
}
::close(clientfd);return true;
}//主 loopvoid* CMyReactor::main_loop(void* p)
{std::cout <"main thread id = " <std::this_thread::get_id() <std::endl;
CMyReactor* pReatcor = static_cast(p);//在一个 while 循环中 不断根据 epoll_wait的返回值去处理相应的事件while (!pReatcor->m_bStop)
{struct epoll_event ev[1024];int n = ::epoll_wait(pReatcor->m_epollfd, ev, 1024, 10);if (n == 0)continue;else if (n 0)
{
}int m = min(n, 1024);for (int i = 0; i {//判断返回的如果是接受链接事件，则通知接收连接线程接收新连接if (ev[i].data.fd == pReatcor->m_listenfd)
pReatcor->m_acceptcond.notify_one();//如果是收发数据事件，则通知普通工作线程接收数据else
{//使用大括号将锁的粒度变细，将临界区变小
{//m_listClients队列是共享资源，需要加锁std::unique_lock<std::mutex> guard(pReatcor->m_workermutex);
pReatcor->m_listClients.push_back(ev[i].data.fd);
}//通知消费者 m_workercond 消费
pReatcor->m_workercond.notify_one();//std::cout <
}// end if
}// end for-loop
}// end whilestd::cout <"main loop exit ..." <std::endl;return NULL;
}void CMyReactor::accept_thread_proc(CMyReactor* pReatcor)
{std::cout <"accept thread, thread id = " <std::this_thread::get_id() <std::endl;//接受链接线程在一个死循环中不断接受客户端的连接while (true)
{int newfd;struct sockaddr_in clientaddr;socklen_t addrlen;
{std::unique_lock<std::mutex> guard(pReatcor->m_acceptmutex);//如果没有连接，进行将阻塞在这里等待。当m_acceptcond被唤醒时，//说明有新连接到来，那么调用 accept 接受连接
pReatcor->m_acceptcond.wait(guard);if (pReatcor->m_bStop)break;//std::cout <
newfd = ::accept(pReatcor->m_listenfd, (struct sockaddr *)&clientaddr, &addrlen);
}if (newfd == -1)continue;std::cout <"new client connected: " <":" <std::endl;//将新socket设置为non-blockingint oldflag = ::fcntl(newfd, F_GETFL, 0);int newflag = oldflag | O_NONBLOCK;if (::fcntl(newfd, F_SETFL, newflag) == -1)
{std::cout <"fcntl error, oldflag =" <", newflag = " <std::endl;continue;
}struct epoll_event e;memset(&e, 0, sizeof(e));
e.events = EPOLLIN | EPOLLRDHUP | EPOLLET;
e.data.fd = newfd;//将 accept 的连接fd ，继续加入 epoll 中监听他的 读写事件if (::epoll_ctl(pReatcor->m_epollfd, EPOLL_CTL_ADD, newfd, &e) == -1)
{std::cout <"epoll_ctl error, fd =" <std::endl;
}
}std::cout <"accept thread exit ..." <std::endl;
}void CMyReactor::worker_thread_proc(CMyReactor* pReatcor)
{std::cout <"new worker thread, thread id = " <std::this_thread::get_id() <std::endl;while (true)
{int clientfd;
{std::unique_ lock<std::mutex> guard(pReatcor->m_workermutex);//注意此处应使用while， 避免虚假唤醒while (pReatcor->m_listClients.empty())
{if (pReatcor->m_bStop)
{std::cout <"worker thread exit ..." <std::endl;return;
}
pReatcor->m_workercond.wait(guard);
}
clientfd = pReatcor->m_listClients.front();
pReatcor->m_listClients.pop_front();
}//gdb调试时不能实时刷新标准输出，用这个函数刷新标准输出，使信息在屏幕上实时显示出来std::cout <std::endl;std::string strclientmsg;char buff[256];bool bError = false;while (true)
{memset(buff, 0, sizeof(buff));int nRecv = ::recv(clientfd, buff, 256, 0);if (nRecv == -1)
{if (errno == EWOULDBLOCK)break;else
{std::cout <"recv error, client disconnected, fd = " <std::endl;
pReatcor->close_client(clientfd);
bError = true;break;
}
}//对端关闭了socket，这端也关闭。else if (nRecv == 0)
{std::cout <"peer closed, client disconnected, fd = " <std::endl;
pReatcor->close_client(clientfd);
bError = true;break;
}
strclientmsg += buff;
}//出错了，就不要再继续往下执行了if (bError)continue;std::cout <"client msg: " <//将消息加上时间标签后发回time_t now = time(NULL);struct tm* nowstr = localtime(&now);std::ostringstream ostimestr;
ostimestr <"[" <tm_year + 1900 <"-"
<std::setw(2) <std::setfill('0') <tm_mon + 1 <"-"
<std::setw(2) <std::setfill('0') <tm_mday <" "
<std::setw(2) <std::setfill('0') <tm_hour <":"
<std::setw(2) <std::setfill('0') <tm_min <":"
<std::setw(2) <std::setfill('0') <tm_sec <"]server reply: ";
strclientmsg.insert(0, ostimestr.str());//对于数据的发送：LT不注册可写事件，有数据直接发送，调用send和write的时候可能会阻塞，但是没有关系//睡一会继续发，一直尝试，到数据发送出去while (true)
{int nSent = ::send(clientfd, strclientmsg.c_str(), strclientmsg.length(), 0);if (nSent == -1)
{if (errno == EWOULDBLOCK)
{std::this_thread::sleep_for(std::chrono::milliseconds(10));continue;
}else
{std::cout <"send error, fd = " <std::endl;
pReatcor->close_client(clientfd);break;
}
}std::cout <"send: " < strclientmsg.erase(0, nSent);if (strclientmsg.empty())break;
}
}
}//根据 ip、port 创建监听socket，和 epollfd， 并将监听socket 挂载到 epollfd 上bool CMyReactor::create_server_listener(const char* ip, short port)
{
m_listenfd = ::socket(AF_INET, SOCK_STREAM | SOCK_NONBLOCK, 0);if (m_listenfd == -1)return false;int on = 1;//设置监听 socket 的地址和端口的可重用
::setsockopt(m_listenfd, SOL_SOCKET, SO_REUSEADDR, (char *)&on, sizeof(on));
::setsockopt(m_listenfd, SOL_SOCKET, SO_REUSEPORT, (char *)&on, sizeof(on));struct sockaddr_in servaddr;memset(&servaddr, 0, sizeof(servaddr));
servaddr.sin_family = AF_INET;
servaddr.sin_addr.s_addr = inet_addr(ip);
servaddr.sin_port = htons(port);//绑定if (::bind(m_listenfd, (sockaddr *)&servaddr, sizeof(servaddr)) == -1)return false;//监听if (::listen(m_listenfd, 50) == -1)return false;//创建 epollfd
m_epollfd = ::epoll_create(1);if (m_epollfd == -1)return false;struct epoll_event e;memset(&e, 0, sizeof(e));
e.events = EPOLLIN | EPOLLRDHUP;
e.data.fd = m_listenfd;//将m_listenfd(监听socketfd)挂载到 epollfd 上面，让epoll_wait 进行监听if (::epoll_ctl(m_epollfd, EPOLL_CTL_ADD, m_listenfd, &e) == -1)return false;return true;
}//main.cpp #include #include //for signal()#include#include //for exit()#include #include #include #include "myreactor.h"
CMyReactor g_reator;void prog_exit(int signo){std::cout <"program recv signal " <" to exit." <std::endl;
g_reator.uninit();
}void daemon_run(){int pid;
signal(SIGCHLD, SIG_IGN);//1)在父进程中，fork返回新创建子进程的进程ID；//2)在子进程中，fork返回0；//3)如果出现错误，fork返回一个负值；
pid = fork();if (pid 0)
{std:: cout <"fork error" <std::endl;exit(-1);
}//父进程退出，子进程独立运行else if (pid > 0) {exit(0);
}//之前parent和child运行在同一个session里,parent是会话(session)的领头进程,//parent进程作为会话的领头进程，如果exit结束执行的话，那么子进程会成为孤儿进程，并被init收养。//执行setsid()之后,child将重新获得一个新的会话(session)id。//这时parent退出之后,将不会影响到child了。
setsid();int fd;
fd = open("/dev/null", O_RDWR, 0);if (fd != -1)
{
dup2(fd, STDIN_FILENO);
dup2(fd, STDOUT_FILENO);
dup2(fd, STDERR_FILENO);
}if (fd > 2)
close(fd);
}int main(int argc, char* argv[]){ //设置信号处理
signal(SIGCHLD, SIG_DFL);
signal(SIGPIPE, SIG_IGN);
signal(SIGINT, prog_exit);
signal(SIGKILL, prog_exit);
signal(SIGTERM, prog_exit);short port = 0;int ch;bool bdaemon = false;//根据传入参数，判断是否开启守护进程模式 -dwhile ((ch = getopt(argc, argv, "p:d")) != -1)
{switch (ch)
{case 'd':
bdaemon = true;break;case 'p':
port = atol(optarg);break;
}
}if (bdaemon)
daemon_run();if (port == 0)
port = 12345;//根据ip、port 初始化socket，创建 epoll_wait()if (!g_reator.init("0.0.0.0", 12345))return -1;//进入 loop 循环
g_reator.main_loop(&g_reator);return 0;
}
至此，一个 reactor 模型基本已经完成了，相信你应该已经理解了，大体的流程了。那么在来看 Muduo 是怎养封装这个 epoll 的？
EventLoop
先来介绍最重要的模块EventLoop : 他是上面第四步的实现：//在一个循环中让 epoll_wait() 不断检测事件void EventLoop::loop() {while() {//vector, 用来返回 epoll_wait 中监听到的 有活动的事件(fd)
m_activeChannels.clear();//m_activeChannels 是一个输入输出参数
m_poller->poll(kPollTimeMs, &m_activeChannels);//遍历 epoll_wait 返回的结果,然后进行 accept / send / recvfor(ChannelList::iterator it = m_activeChannels.begin();
it != m_activeChannels.end(); ++it)
{
(*it)->handleEvent();
}//处理其他事件， 此时不必关心
doPendingFunctors();
}
}Channel
对于上面 epoll_wait() 返回的就绪事件，怎么去处理呢？那么Channel 类主要负责把不同的 IO 事件分发给不同的回调，例如 ReadCallback、 WriteCallBack 等；同时提供向 epollfd 中注册可读可写事件的接口。每个 Channel 自始至终只负责一个文件描述符的 IO 事件分发。//主要处理 epoll_wait 返回的事件，并将他们分发到不同的回调void Channel::handleEvent()
{//如果出错，那么分发到错误的回调if(m_revents & (POLLERR | POLLNVAL)){if(m_errorCallBack) m_errorCallBack();
}//如果是 POLLIN， 说明是数据收发的回调if(m_revents & (POLLIN | POLLPRI | POLLRDHUP)){if(m_readCallBack) m_readCallBack();
}//如果是 POLLOUT， 那么是可写事件的回调if(m_revents & POLLOUT){if(m_writeCallBack) m_writeCallBack();
}
}//设置回调函数，供其他模块注册回调函数void setReadCallback(const ReadEventCallback& cb){ readCallback_ = cb; }void setWriteCallback(const EventCallback& cb){ writeCallback_ = cb; }void setCloseCallback(const EventCallback& cb){ closeCallback_ = cb; }void setErrorCallback(const EventCallback& cb){ errorCallback_ = cb; }//往 epoll 中注册可读事件void enableReading() { m_events |= kReadEvent; update(); }//从 epoll 中移除可读事件void disableReading() { m_events &= ~kReadEvent; update(); }void enableWriting() { m_events |= kWriteEvent; update(); }bool isWriting() { return m_events &= kWriteEvent; }bool isReading() { return m_events &= kReadEvent; }void disableWriting() { m_events &= ~kWriteEvent; update(); }//最终都调用 uodate函数，该函数会调用 EventLoop::uodateChannel(), 后者在调用 Poller::updateChannel();void Channel::update()
{
m_addedToLoop = true;
p_loop->runInLoop(std::bind(&EventLoop::updateChannel, p_loop, this));
}Poller
Poller类是 IO multiplexing 的封装。在 Muduo 中是一个抽象类，因为 Muduo 同时支持 poll 和 epoll 两种 IO 多路复用机制，他们是真正调用 epoll_wait() 的地方TimeStamp Poller::poll(int timeoutMs, ChannelList* activeChannels)
{//真正的调用 epoll_wait / poll 的地方。int numEvents = ::poll(/*&*m_pollfds.begin()*/m_pollfds.data(), m_pollfds.size(), timeoutMs);TimeStamp now(TimeStamp::now());if(numEvents > 0){//将返回的结果封装成 channel 返回给 Eventloop::loop 函数
fillActiveChannels(numEvents, activeChannels);
}else if(numEvents == 0){
LOG_TRACE <" nothing happended";
}else{
LOG_SYSERR <"Poller::poll()";
}return now;
}void Poller::fillActiveChannels(int numEvents, ChannelList* activeChannels) const
{for(PollFdList::const_iterator pfd = m_pollfds.begin();
pfd != m_pollfds.end() && numEvents > 0; ++pfd)
{if(pfd->revents > 0)
{
--numEvents;
ChannelMap::const_iterator ch = m_channels.find(pfd->fd);
assert(ch != m_channels.end());
Channel* channel = ch->second;
assert(channel->fd() == pfd->fd);
channel->set_revents(pfd->revents);//将返回的结果封装成 channel 返回给 Eventloop::loop 函数
activeChannels->push_back(channel);
}
}
}
- 接受客户端连接到来的类 ，需要注册 Channel::setReadCallback(), 以调用accept 进行接受；
- 接受有数据到来的类 ，需要注册 Channel::setReadCallback()， 以调用 recv 接收数据；
- 检测发送缓冲区是否可写， 需要注册 Channel::WriteCallback(), 以调用 send 函数发送数据；
Acceptor
接受客户端连接到来的类就是 Muduo::net::Acceptor 类Acceptor::Acceptor(EventLoop* loop, const InetAddress& listenAddr)
: loop_(loop),//在构造函数中同时创建 acceptsocket，并调用下面的 acceptChannel_.setReadCallback 将acceptSocket添加到 epoll 中，监听可读事件
acceptSocket_(sockets::createNonblockingOrDie()),
acceptChannel_(loop, acceptSocket_.fd()),
listenning_(false),
idleFd_(::open("/dev/null", O_RDONLY | O_CLOEXEC))
{
assert(idleFd_ >= 0);
acceptSocket_.setReuseAddr(true);//绑定监听
acceptSocket_.bindAddress(listenAddr);//注册可读事件，当有事件到来时， 调用 handleRead() 回调接受连接
acceptChannel_.setReadCallback(
boost::bind(&Acceptor::handleRead, this));
}void Acceptor::handleRead()
{
loop_->assertInLoopThread();InetAddress peerAddr(0);//真正 调用 accept 的地方int connfd = acceptSocket_.accept(&peerAddr);if (connfd >= 0)
{// string hostport = peerAddr.toIpPort();// LOG_TRACE <if (newConnectionCallback_)
{//将接受的新连接，分配给 TcpConnection 函数。即代表一路连接
newConnectionCallback_(connfd, peerAddr);
}else
{
sockets::close(connfd);
}
}else
{// Read the section named "The special problem of// accept()ing when you can't" in libev's doc.// By Marc Lehmann, author of livev.if (errno == EMFILE)
{
::close(idleFd_);
idleFd_ = ::accept(acceptSocket_.fd(), NULL, NULL);
::close(idleFd_);
idleFd_ = ::open("/dev/null", O_RDONLY | O_CLOEXEC);
}
}
}
TcpServer
TcpServer 类会注册 Acceptor::setNewConnectionCallback，同时会为每个连接创建一个 TcpConnection 类；//TcpServer 类
TcpServer::TcpServer(EventLoop* loop,const InetAddress& listenAddr,const string& nameArg)
: loop_(CHECK_NOTNULL(loop)),
hostport_(listenAddr.toIpPort()),
name_(nameArg),
acceptor_(new Acceptor(loop, listenAddr)),
threadPool_(new EventLoopThreadPool(loop)),
connectionCallback_(defaultConnectionCallback),
messageCallback_(defaultMessageCallback),
started_(false),
nextConnId_(1)
{//在构造函数中 注册 Acceptor 的 connfd 回调函数，同时调用 newconnection 函数
acceptor_->setNewConnectionCallback(
boost::bind(&TcpServer::newConnection, this, _1, _2));
}void TcpServer::newConnection(int sockfd, const InetAddress& peerAddr)
{
loop_->assertInLoopThread();// 按照轮叫的方式选择一个EventLoop
EventLoop* ioLoop = threadPool_->getNextLoop();char buf[32];snprintf(buf, sizeof buf, ":%s#%d", hostport_.c_str(), nextConnId_);
++nextConnId_;string connName = name_ + buf;
LOG_INFO <"TcpServer::newConnection [" < <"] - new connection [" < <"] from " < InetAddress localAddr(sockets::getLocalAddr(sockfd));//每一个连接 新建一个 TcpConnection 类，同时设置他们的用来数据收发的回调函数TcpConnectionPtr conn(new TcpConnection(ioLoop,
connName,
sockfd,
localAddr,
peerAddr));
connections_[connName] = conn;
conn->setConnectionCallback(connectionCallback_);
conn->setMessageCallback(messageCallback_);
conn->setWriteCompleteCallback(writeCompleteCallback_);
conn->setCloseCallback(
boost::bind(&TcpServer::removeConnection, this, _1));// conn->connectEstablished();
ioLoop->runInLoop(boost::bind(&TcpConnection::connectEstablished, conn));
LOG_TRACE <"[5] usecount=" <
}TcpConnection
TcpConnection 类会向 分发事件的类 Channel 注册接受数据和发送数据的回调 setReadCallback、setWriteCallback等，// TcpConeciton 的构造函数
TcpConnection::TcpConnection(EventLoop* loop,const string& nameArg,int sockfd,const InetAddress& localAddr,const InetAddress& peerAddr)
: loop_(CHECK_NOTNULL(loop)),
name_(nameArg),
state_(kConnecting),
socket_(new Socket(sockfd)),
channel_(new Channel(loop, sockfd)),
localAddr_(localAddr),
peerAddr_(peerAddr),
highWaterMark_(64*1024*1024)
{// 通道可读事件到来的时候，回调TcpConnection::handleRead，_1是事件发生时间
channel_->setReadCallback(
boost::bind(&TcpConnection::handleRead, this, _1));// 通道可写事件到来的时候，回调TcpConnection::handleWrite
channel_->setWriteCallback(
boost::bind(&TcpConnection::handleWrite, this));// 连接关闭，回调TcpConnection::handleClose
channel_->setCloseCallback(
boost::bind(&TcpConnection::handleClose, this));// 发生错误，回调TcpConnection::handleError
channel_->setErrorCallback(
boost::bind(&TcpConnection::handleError, this));
LOG_DEBUG <"TcpConnection::ctor[" <"] at " <this
<" fd=" < socket_->setKeepAlive(true);
}//用于发送数据的函数void TcpConnection::handleWrite()//当有事件到来，Channel::handleEvent() 会分发事件，同时提供回调函数， TcpConnection 会注册 setReadCallback 完成真正的数据收发；void TcpConnection::handleRead(Timestamp receiveTime)
扫一扫
3799
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
