Linux高性能服务器编程第十五章线程池

Hoshea Zhang2023-11-132023-11-13

我们这里不介绍进程池

概述

创建一组子线程，运行着相同的代码和属性，任务到来时，主线程通过某种方式选择一个子线程为之服务，方法有两种：

随机算法或轮流选取算法
工作队列

处理多客户

这里用进程的概念去解释：

在使用进程池处理多客户任务时，首先要考虑的一个问题是﹔监听 socket和连接socket是否都由主进程来统一管理。回忆第8章中我们介绍过的几种并发模式，其中半同步/半反应堆模式是由主进程统一管理这两种 socket的;而半同步/半异步模式，以及领导者/追随者模式，则是由主进程管理所有监听socket，而各个子进程分别管理属于自己的连接socket 的。对于前一种情况，主进程接受新的连接以得到连接socket，然后它需要将该socket传递给子进程（对于线程池而言，父线程将socket传递给子线程是很简单的，因为它们可以很容易地共享该socket。)。后一种情况的灵活性更大一些，因为子进程可以自己调用accept来接受新的连接，这样父进程就无须向子进程传递socket，而只需要简单地通知一声:“我检测到新的连接，你来接受它。”

如果客户任务存在上下文关系，最好使用同一个子线程去服务，多线程中可以使用EPOLLONESHOT来处理

线程池的实现

具体看注释

同步机制包装类

/线程同步机制包装类
#ifndef LOCKER_H
#define LOCKER_H
 
#include <exception>
#include <pthread.h>
#include <semaphore.h>
 
//封装信号量的类
class sem
{
public:
    sem()
    {
        if(sem_init(&m_sem, 0, 0) != 0)
        {
            throw std::exception();
        }
    }
    //销毁信号量
    ~sem()
    {
        sem_destroy(&m_sem);
    }
    //等待信号量
    bool wait()
    {
        return sem_wait(&m_sem) == 0;
    }
    bool post()
    {
        return sem_post(&m_sem) == 0;
    }
private:
    sem_t m_sem;
};
 
 
class locker
{
public:
    locker()
    {
        if(pthread_mutex_init(&m_mutex, NULL) != 0)
        {
            throw std::exception();
        }
    }
    ~locker()
    {
        pthread_mutex_destroy(&m_mutex);
    }
 
    bool lock()
    {
        return pthread_mutex_lock(&m_mutex) == 0;
    }
 
    bool unlock()
    {
        return pthread_mutex_unlock(&m_mutex) == 0;
    }
 
private:
    pthread_mutex_t m_mutex;
 
};
//phtread
 
class cond
{
 
public:
    cond()
    {
        if(pthread_mutex_init(&m_mutex, NULL) != 0)
        {
            throw std::exception();
        }
        if(pthread_cond_init(&m_cond, NULL) != 0)
        {
            //出现问题得释放已经分配的资源
            pthread_mutex_destroy(&m_mutex);
            throw std::exception();
        }
    }
 
    ~cond()
    {
        pthread_mutex_destroy(&m_mutex);
        pthread_cond_destroy(&m_cond);
    }
 
    bool wait()
    {
        int ret = 0;
        pthread_mutex_lock(&m_mutex);
        ret = pthread_cond_wait(&m_cond, &m_mutex);
        pthread_mutex_unlock(&m_mutex);
        return ret == 0;
    }
 
    bool signal()
    {
        return pthread_cond_signal(&m_cond) == 0;
    }
 
private:
    pthread_mutex_t m_mutex;
    pthread_cond_t m_cond;
 
};
 
 
#endif

线程池类

#ifndef THREADPOOL_H
#define THREADPOOL_H
 
#include<list>
#include<cstdio>
#include<exception>
#include<pthread.h>
#include"locker.h"

//模板参数是任务类
template<typename T>
class threadpool
{
private:
    //请求队列
    std::list<T*> m_workequeue;
    //描述线程池的数组，大小为m_thread_number
    pthread_t* m_threads;
    //线程池中的线程数
    int m_thread_number;
    //请求队列中最大的请求数
    int m_max_requests;
    //保护请求队列的互斥锁
    locker m_queuelocker;
    //是否有任务需要处理
    sem m_queuestat;
    //是否结束线程
    bool m_stop;    
 
private:
    //工作进程运行的函数
    static void* worker(void* arg);
    void run();
 
public:
    //线程数量默认为8，最大工作请求数默认为10000
    threadpool(int thread_number = 8 , int max_requests = 10000);
    ~threadpool();
    bool append(T* request);
};
 
template<typename T>
threadpool<T>::threadpool(int thread_number,int max_requests):m_thread_number(thread_number),m_max_requests(max_requests),m_threads(NULL),m_stop(false)
{
    if((thread_number<=0) || (max_requests<=0)){
        throw std::exception();
    }
 
    m_threads = new pthread_t[m_thread_number];
    if(!m_threads){
        throw std::exception();
    }
    //创建thread_number个线程
    for(int i=0;i<thread_number;++i){
        printf("create the %dth thread\n",i+1);
        pthread_create(m_threads+i,NULL,worker,this);
        //线程分离就是当线程被设置为分离状态后，线程结束时，它的资源会被系统自动的回收，而不再需要在其它线程中对其进行 pthread_join() 操作
        pthread_detach(m_threads[i]);      
    }
}
template<typename T>
threadpool<T>::~threadpool()
{
    delete[] m_threads;
    m_stop = true;
}
template<typename T>
bool threadpool<T>::append(T* request){
    //记得上锁，因为所有线程共享该队列
    m_queuelocker.lock();
    if(m_workequeue.size()>=m_max_requests){
        m_queuelocker.unlock();
        return false;
    }
    m_workequeue.push_back(request);
    m_queuelocker.unlock();
    m_queuestat.post();
    return true;
}
template<typename T>
void* threadpool<T>::worker(void* arg){
    threadpool* pool = (threadpool*)arg;
    pool->run();
    return pool;
}
template<typename T>
void threadpool<T>::run(){
    while(!m_stop){
        //从工作队列中取出任务
        m_queuestat.wait();
        m_queuelocker.lock();
        if(m_workequeue.empty()){
            m_queuelocker.unlock();
            continue;
        }
        T* request = m_workequeue.front();
        m_workequeue.pop_front();
        m_queuelocker.unlock();
        if(!request){
            continue;
        }
        request->process();
    }
}
 
#endif

http_conn

头文件

#ifndef HTTP_CONN_h
#define HTTP_CONN_h
 
#include <unistd.h>
#include <signal.h>
#include <sys/types.h>
#include <sys/epoll.h>
#include <fcntl.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <assert.h>
#include <sys/stat.h>
#include <string.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <stdarg.h>
#include <errno.h>
#include <sys/uio.h>
#include "locker.h"
 
class http_conn
{
public:
    
    static const int FILENAME_LEN = 200;//文件名的最大长度
    
    static const int READ_BUFFER_SIZE = 2048;//缓冲区的大小
    
    static const int WRITE_BUFFER_SIZE = 1024;//写缓冲区的大小
 
 
    //HTTP请求方法
    //HTTP请求方法,但本代码中仅仅支持GET
    enum METHOD{
        GET = 0,
        POST,
        HEAD,
        PUT,
        DELETE,
        TRACE,
        OPTIONS,
        CONNECT,
        PATCH
    };
 
 
    //主状态机可能的状态
    enum CHECK_STATE
    {
        CHECK_STATE_REQUESTION = 0,//正在分析当前请求行
        CHECK_STATE_HEADER,//正在分析头部字段
        CHECK_STATE_CONTENT
    };
 
    //从状态机可能的状态
    enum LINE_STATUS
    {
        LINE_OK = 0,//读取到一个完整的行
        LINE_BAD,//行出错
        LINE_OPEN//行数据尚且不完整
    };
 
    //服务器处理http请求的结果
    enum HTTP_CODE
    {
        NO_REQUEST,//请求不完整需要继续读取
        GET_REQUEST,//得到了一个完整的请求
        BAD_REQUEST,//请求有语法错误
        NO_RESOURCE,//没有资源
        FORBIDDEN_REQUEST,//没有足够的权限
        FILE_REQUEST,//文件已被请求
        INTERNAL_ERROR,//服务器内部错误
        CLOSED_CONNECTION//客户端连接已关闭
    };
 
public:
    http_conn(){}
    ~http_conn(){}
public:
    void init(int sockfd,const sockaddr_in& addr);
    void close_conn(bool real_close = true);
    bool read();
    bool write();
    void process();
 
private:
    void init();
    HTTP_CODE process_read();
    bool process_write(HTTP_CODE ret);
 
    //下面一组函数被process_read调用以分析HTTP请求
    HTTP_CODE parse_request_line(char* text);
    HTTP_CODE parse_header(char* text);
    HTTP_CODE parse_content(char* text);
    HTTP_CODE do_request();
    char* get_line(){ return m_read_buf + m_start_line; }
    LINE_STATUS parse_line();
 
    //下面一组函数被process_write调用以填充HTTP应答
    void unmap();
    bool add_response(const char* format, ...);
    bool add_content(const char* content);
    bool add_status_line(int status, const char* title);
    bool add_headers(int content_length);
    bool add_content_length(int content_length);
    bool add_linger();
    bool add_blank_line();
 
public:
    static int m_epollfd;
    static int m_user_count;
 
private:
    int m_sockfd;
    sockaddr_in m_address;
 
    char m_read_buf[READ_BUFFER_SIZE];
    int m_read_idx;
    int m_check_idx;
    int m_start_line;
    char m_write_buf[WRITE_BUFFER_SIZE];
    int m_write_idx;
 
    CHECK_STATE m_check_state;
    METHOD m_method;
    
    char m_real_file[FILENAME_LEN];
    char* m_url;
    char* m_version;
    char* m_host;
    int m_content_length;
    bool m_linger;
 
    char* m_file_address;
    struct stat m_file_stat;
    struct iovec m_iv[2];
    int m_iv_count;
};
 
#endif

源文件

#include"http_conn.h"
//定义一些HTTP响应的状态信息
const char* ok_200_title = "OK";
const char* error_400_title = "Bad Request";
const char* error_400_form = "Your request has bad syntax or is inherently impossible to satisfy.\n";
const char* error_403_title = "Forbidden";
const char* error_403_form = "you do not have permisson to get file from this server.\n";
const char* error_404_title = "Not Found";
const char* error_404_form = "The requested file was not found on this server.\n";
const char* error_500_title = "Internal Error";
const char* error_500_form = "There was an unusual problem serving the requested file.\n";
 
//网站的根目录
const char* doc_root = "/home/ross/Documents";
 
int setnonblocking(int fd){
    int old_option = fcntl(fd,F_GETFL);
    int new_option = old_option | O_NONBLOCK;
    fcntl(fd,F_SETFL,new_option);
    return old_option;
}
 
void addfd(int epollfd,int fd,bool one_shot){
    epoll_event event;
    event.data.fd = fd;
    event.events = EPOLLIN | EPOLLET | EPOLLRDHUP;
    if(one_shot){
        event.events |= EPOLLONESHOT;
    }
    epoll_ctl(epollfd,EPOLL_CTL_ADD,fd,&event);
    setnonblocking(fd);
}
 
void removefd(int epollfd,int fd){
    epoll_ctl(epollfd,EPOLL_CTL_DEL,fd,0);
    close(fd);
}
 
void modfd(int epollfd,int fd,int ev){
    epoll_event event;
    event.data.fd = fd;
    event.events = ev | EPOLLET | EPOLLONESHOT | EPOLLRDHUP;
    epoll_ctl(epollfd,EPOLL_CTL_MOD,fd,&event);
}
 
int http_conn::m_user_count = 0;
int http_conn::m_epollfd = -1;
 
void http_conn::close_conn(bool real_close){
    if(real_close && (m_sockfd != -1)){
        removefd(m_epollfd,m_sockfd);
        m_sockfd = -1;
        m_user_count --;
    }
}
 
void http_conn::init(int sockfd,const sockaddr_in& addr){
    m_sockfd = sockfd;
    m_address = addr;
    addfd(m_epollfd,sockfd,true);
    m_user_count ++;
    init();
}
 
void http_conn::init(){
    m_check_state = CHECK_STATE_REQUESTION;
    m_linger = false;
 
    m_method = GET;
    m_url = 0;
    m_version = 0;
    m_content_length = 0;
    m_host = 0;
    m_start_line = 0;
    m_check_idx = 0;
    m_read_idx = 0;
    m_write_idx = 0;
    memset(m_read_buf,'\0',READ_BUFFER_SIZE);
    memset(m_write_buf,'\0',WRITE_BUFFER_SIZE);
    memset(m_real_file,'\0',FILENAME_LEN);
}
 
http_conn::LINE_STATUS http_conn::parse_line(){
    char temp;
    for(;m_check_idx<m_read_idx;++m_check_idx){
        temp = m_read_buf[m_check_idx];
        if(temp == '\r'){
            if(m_read_buf[m_check_idx+1] == '\n'){
                m_read_buf[m_check_idx++] = '\0';
                m_read_buf[m_check_idx++] = '\0';
                return LINE_OK;
            }
            else if( (m_check_idx+1) == m_read_idx){
                return LINE_OPEN;
            }
            else
                return LINE_BAD;
        }
        else if(temp == '\n'){
            if((m_check_idx>1) && (m_read_buf[m_check_idx-1] == '\r'))
            {
                m_read_buf[m_check_idx-1] = '\0';
                m_read_buf[m_check_idx++] = '\0';
                return LINE_OK;
            }
            else{
                return LINE_BAD;
            }
        }
    }
    return LINE_OPEN;
}
 
bool http_conn::read(){
    if(m_read_idx >= READ_BUFFER_SIZE){
        return false;
    }
    int bytes_read = 0;
    while(true){
        bytes_read = recv(m_sockfd,m_read_buf,READ_BUFFER_SIZE-m_read_idx,0);
        if(bytes_read == -1){
            if( errno == EAGAIN || errno == EWOULDBLOCK){
                break;
            }
            return false;
        }
        else if(bytes_read == 0){
            return false;
        }
        m_read_idx += bytes_read;
    }
    return true;
}
 
http_conn::HTTP_CODE http_conn::parse_request_line(char* text){
    m_url = strpbrk(text," \t");
    if(!m_url){
        return BAD_REQUEST;
    }
    *m_url++ = '\0';
    m_url += strspn(m_url," \t");
    
    char* method = text;
    if(strcasecmp(method,"GET") == 0){
        m_method = GET;
    }
    else{
        return BAD_REQUEST;
    }
 
    m_version = strpbrk(m_url," \t");
    if(!m_version){
        return BAD_REQUEST;
    }
    *m_version++ = '\0';
    m_version+=strspn(m_version," \t");
    if(strcasecmp(m_version,"HTTP/1.1") != 0){
        return BAD_REQUEST;
    }
 
    if(strncasecmp(m_url,"http://",7)==0){
        m_url+=7;
        m_url = strchr(m_url,'/');
    }
    if(!m_url || m_url[0]!= '/'){
        return BAD_REQUEST;
    }
 
    m_check_state = CHECK_STATE_HEADER;
    return NO_REQUEST;
}
 
http_conn::HTTP_CODE http_conn::parse_header(char* text){
    if(text[0] == '\0'){
        if(m_content_length != 0){
            m_check_state = CHECK_STATE_CONTENT;
            return NO_REQUEST;
        }
        printf("parse_header OK\n");
        return GET_REQUEST;
    }
    else if(strncasecmp(text,"Connection:",11) == 0){
        text+=11;
        text += strspn(text," \t");
        if(strcasecmp(text,"keep-alive") == 0){
            m_linger = true;
        }
    }
    else if(strncasecmp(text,"Content-Length",15)==0){
        text += 15;
        text += strspn(text," \t");
        m_content_length += atol(text);
    }
    else if(strncasecmp(text,"Host:",5) == 0){
        text+=5;
        text+=strspn(text," \t");
        m_host = text;
    }
    else{
        printf("oh no! I can't handle this header %s\n",text);
    }
    return NO_REQUEST;
}
 
http_conn::HTTP_CODE http_conn::parse_content(char* text){
    if(m_read_idx >= m_content_length+m_check_idx){
        text[m_content_length] = '\0';
        return GET_REQUEST;
    }
    return NO_REQUEST;
}
 
http_conn::HTTP_CODE http_conn::process_read(){
    LINE_STATUS line_status = LINE_OK;
    HTTP_CODE ret = NO_REQUEST;
    char* text = 0;
    while(((m_check_state == CHECK_STATE_CONTENT) && (line_status == LINE_OK)) || ((line_status=parse_line()) ==LINE_OK) ){
        text = get_line();
        m_start_line = m_check_idx;
        printf("I got 1 line %s\n",text);
        switch(m_check_state)
        {
        case CHECK_STATE_REQUESTION:{
            ret = parse_request_line(text);
            if(ret == BAD_REQUEST){
                return BAD_REQUEST;
            }
            break;
        }
        case CHECK_STATE_HEADER:{
            ret = parse_header(text);
            if(ret == BAD_REQUEST){
                return BAD_REQUEST;
            }
            else if(ret == GET_REQUEST){
                return do_request();
            }
            break;
        }
        case CHECK_STATE_CONTENT:{
            ret = parse_content(text);
            if(ret == GET_REQUEST){
                return do_request();
            }
            line_status = LINE_OPEN;
            break;
        }
        default:
            return INTERNAL_ERROR;
        }
    }
    return NO_REQUEST;
}
 
http_conn::HTTP_CODE http_conn::do_request(){
    strcpy(m_real_file,doc_root);
    int len = strlen(doc_root);
    strncpy(m_real_file+len,m_url,FILENAME_LEN-len-1);
    if(stat(m_real_file,&m_file_stat) < 0){
        return NO_RESOURCE;
    }
    if(!(m_file_stat.st_mode & S_IROTH)){
        return FORBIDDEN_REQUEST;
    }
    if(S_ISDIR(m_file_stat.st_mode)){
        return BAD_REQUEST;
    }
 
    int fd = open(m_real_file,O_RDONLY);
    m_file_address = (char*)mmap(0,m_file_stat.st_size,PROT_READ,MAP_PRIVATE,fd,0);
    close(fd);
    return FILE_REQUEST;
}
 
void http_conn::unmap(){
    if(m_file_address){
        munmap(m_file_address,m_file_stat.st_size);
        m_file_address = 0;
    }
}
 
bool http_conn::add_response(const char* format, ...){
    if(m_write_idx>WRITE_BUFFER_SIZE){
        return false;
    }
    va_list arg_list;
    va_start(arg_list,format);
    int len = vsnprintf(m_write_buf+m_write_idx,WRITE_BUFFER_SIZE-1-m_write_idx,format,arg_list);
    if(len >= (WRITE_BUFFER_SIZE-1-m_write_idx)){
        return false;
    }
    m_write_idx+=len;
    va_end(arg_list);
    return true;
}
 
bool http_conn::add_status_line(int status,const char* title){
    return add_response("%s %d %s\r\n","HTTP/1.1",status,title);
}
 
bool http_conn::add_headers(int content_len){
    add_content_length(content_len);
    add_linger();
    add_blank_line();
    return true;    
}
 
bool http_conn::add_content_length(int content_len){
    return add_response("Content-Length: %d\r\n",content_len);
}
 
bool http_conn::add_linger(){
    return add_response("Connection: %s\r\n",(m_linger == true)?"keep-alive":"close");
}
 
bool http_conn::add_blank_line(){
    return add_response("%s","\r\n");
}
 
bool http_conn::add_content(const char* content){
    return add_response("%s",content);
}
 
bool http_conn::process_write(HTTP_CODE ret){
    switch ((ret))
    {
    case INTERNAL_ERROR:
    {
        add_status_line(500,error_500_title);
        add_headers(strlen(error_500_form));
        if(!add_content(error_500_form)){
            return false;
        }
        break;
    }
    case BAD_REQUEST:
    {
        add_status_line(400,error_400_title);
        add_headers(strlen(error_400_form));
        if(! add_content(error_500_form)){
            return false;
        }
        break;
    }
    case NO_RESOURCE:
    {
        add_status_line(404,error_404_title);
        add_headers(strlen(error_404_form));
        if(! add_content(error_404_form)){
            return false;
        }
        break;
    }
    case FORBIDDEN_REQUEST:
    {
        add_status_line(403,error_403_title);
        add_headers(strlen(error_403_form));
        if(! add_content(error_403_form)){
            return false;
        }
        break;
    }
    case FILE_REQUEST:
    {
        add_status_line(200,ok_200_title);
        if(m_file_stat.st_size !=0){
            add_headers(m_file_stat.st_size);
            m_iv[0].iov_base = m_write_buf;
            m_iv[0].iov_len = m_write_idx;
            m_iv[1].iov_base = m_file_address;
            m_iv[1].iov_len = m_file_stat.st_size;
            m_iv_count = 2;
            printf("process_write ok\n");
            return true;
        }
        else{
            const char* ok_string = "<html><body></body></html>";
            add_headers(strlen(ok_string));
            if(! add_content(ok_string)){
                return false;
            }
            break;
        }
    }
 
    default:
    {
        return false;
    }
    }
    m_iv[0].iov_base = m_write_buf;
    m_iv[0].iov_len = m_write_idx;
    m_iv_count = 1;
    return true;
}
 
bool http_conn::write(){
    int temp = 0;
    int bytes_have_send = 0;
    int bytes_to_send = m_write_idx;
    if(bytes_to_send == 0){
        modfd(m_epollfd,m_sockfd,EPOLLOUT);
        init();
        return true;
    }
    while(1)
    {
        temp = writev(m_sockfd,m_iv,m_iv_count);
        if(temp < 0){
            if(errno == EAGAIN){
                modfd(m_epollfd,m_sockfd,EPOLLOUT);
                return true;
            }
            unmap();
            return false;
        }
 
        bytes_have_send+=temp;
        bytes_to_send-= temp;
        if(bytes_to_send<=bytes_have_send){
            unmap();
            if(m_linger){
                init();
                modfd(m_epollfd,m_sockfd,EPOLLIN);
                return true;
            }
            else{
                modfd(m_epollfd,m_sockfd,EPOLLIN);
                return false;
            }
        }
    }
}
 
void http_conn::process()
{
    HTTP_CODE read_ret = process_read();
    if(read_ret == NO_REQUEST){
        modfd(m_epollfd,m_sockfd,EPOLLIN);
        return;
    }
    bool write_ret = process_write(read_ret);
    if(!write_ret){
        close_conn();
    }
    modfd(m_epollfd,m_sockfd,EPOLLOUT);
}

main函数

#include <unistd.h>
#include <signal.h>
#include <sys/types.h>
#include <sys/epoll.h>
#include <fcntl.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <assert.h>
#include <sys/stat.h>
#include <string.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <stdarg.h>
#include <errno.h>
 
#include "locker.h"
#include "threadpool.h"
#include "http_conn.h"
 
#define MAX_FD 65536
#define MAX_EVENT_NUMBER 10000
 
extern int addfd(int epollfd, int fd, bool one_shot);
extern int removefd(int epollfd, int fd);
 
void addsig(int sig,void(handler)(int),bool restart = true){
    struct sigaction sa;
    memset(&sa,'\0',sizeof(sa));
    sa.sa_handler =  handler;
    if(restart){
        sa.sa_flags |= SA_RESTART;
    }
    sigfillset(&sa.sa_mask);
    assert(sigaction(sig,&sa,NULL) != -1);
}
 
void show_error(int sockfd,const char* info){
    printf("%s\n",info);
    send(sockfd,info,strlen(info),0);
    close(sockfd);
}
 
int main(int argc,char* argv[]){
    if(argc <= 2){
        printf("usage:%s ip port\n",basename(argv[0]));
        return 1;
    }
 
    const char* ip = argv[1];
    int port = atoi(argv[2]);
 
    addsig(SIGPIPE,SIG_IGN);
 
    threadpool<http_conn>* pool = NULL;
    try
    {
        pool = new threadpool<http_conn>;
    }
    catch(...)
    {
        printf("error\n");
        return 1;   
    }
    printf("creat a threadpool\n");
 
    http_conn* users = new http_conn[MAX_FD];
    assert(users);
    int user_count = 0;
    printf("create some users\n");
 
    sockaddr_in addr;
    bzero(&addr,sizeof(addr));
    addr.sin_family = AF_INET;
    inet_pton(AF_INET6,ip,&addr.sin_addr);
    addr.sin_port = htons(port);
 
    int listenfd = socket(PF_INET,SOCK_STREAM,0);
    assert(listenfd>=0);
    linger tmp = {1,0};
    setsockopt(listenfd,SOL_SOCKET,SO_LINGER,&tmp,sizeof(tmp));
 
    int ret = bind(listenfd,(sockaddr*)&addr,sizeof(addr));
    assert(ret != -1);
 
    ret = listen(listenfd,5);
    assert(ret != -1);
    printf("listening\n");
 
    int epollfd = epoll_create(5);
    assert(epollfd != -1);
    epoll_event events[MAX_EVENT_NUMBER];
    addfd(epollfd,listenfd,false);
    http_conn::m_epollfd = epollfd;
 
    while(true){
        int number = epoll_wait(epollfd,events,MAX_EVENT_NUMBER,-1);
        if((number < 0) && (errno!=EAGAIN)){
            printf("epoll failure\n");
            break;
        }
        
        for(int i=0;i<number;++i){
            int sockfd = events[i].data.fd;
            if(sockfd == listenfd){
                sockaddr_in clientAddr;
                socklen_t len = sizeof(clientAddr);
                int connfd = accept(listenfd,(sockaddr*)&clientAddr,&len);
                if(connfd < 0){
                    printf("accept errno\n");
                    continue;
                }
                if(http_conn::m_user_count >= MAX_FD){
                    show_error(connfd,"Internal server busy\n");
                    continue;
                }
                users[connfd].init(connfd,clientAddr);
            }
            else if(events[i].events & (EPOLLRDBAND | EPOLLHUP | EPOLLERR)){
                users[sockfd].close_conn();
            }
            else if(events[i].events & EPOLLIN){
                if(users[sockfd].read()){
                    pool->append(users + sockfd);
                }
                else{
                    users[sockfd].close_conn();
                }
            }
            else if(events[i].events & EPOLLOUT){
                if(!users[sockfd].write()){
                    users[sockfd].close_conn();
                }
                printf("write ok\n");
            }
            else{}
        }
    }
    close(epollfd);
    close(listenfd);
    delete[] users;
    delete pool;
    return 0;
}