WIP CGI monitered by epoll

+ OK, but some errors case need to be lookup

memo.txt

@@ -21,6 +21,7 @@ For non blocking CGI :
 	// when waitpid() tell us its finish (or maybe when epoll return EPOLLHUP)
 	// then actually parse the script_output and send it to the client.
 
+- check status in autoindex
 
 ----Priorité modérée------------------------
 - namespace utils ?
@@ -28,7 +29,12 @@ For non blocking CGI :
 and add "const" if apropriate.
 - peut-être check si ip > 32bits
 
+
 ----Priorité faible------------------------
+- idealy, we should not clear() raw_request after a response,
+	but just the part we actually parsed for this response.
+	I think the client could send multiples request on the same connection one after the other without waiting for response.
+	So raw_request could contain more than the first request we handle.
 - chunked request (need testing)
 - client_body_limit 0 valeur special pour desactiver dans config
 - gerer le champ "Accept" du client

srcs/Client.cpp

@@ -10,9 +10,9 @@ Client::Client()
 	header_complete(false),
 	body_complete(false),
 	request_complete(false),
-	read_body_size(0),
 	assigned_server(NULL),
 	assigned_location(NULL),
+	cgi_pipe_rfd(0),
 	_fd(0),
 	_port(""),
 	_ip(""),
@@ -26,9 +26,9 @@ Client::Client(int afd, listen_socket *lsocket, std::string aport, std::string a
 	header_complete(false),
 	body_complete(false),
 	request_complete(false),
-	read_body_size(0),
 	assigned_server(NULL),
 	assigned_location(NULL),
+	cgi_pipe_rfd(0),
 	_fd(afd),
 	_port(aport),
 	_ip(aip),
@@ -45,7 +45,6 @@ Client::~Client() {
 Client::Client( Client const & src )
 	: status			( src.status ),
 	header_complete		( src.header_complete ),
-	read_body_size		( src.read_body_size ),
 	assigned_server		( src.assigned_server ),
 	assigned_location	( src.assigned_location ),
 	_fd					( src._fd ),
@@ -232,15 +231,16 @@ void 	Client::fill_script_path(std::string &path, size_t pos)
 void	Client::clear()
 {
 	clear_request();
-	header_complete = false;
-	body_complete = false;
-	request_complete = false;
-	read_body_size = 0;
-	assigned_server = NULL;
-	assigned_location = NULL;	
 	raw_request.clear();
 	response.clear();
 	status = 0;
+	header_complete = false;
+	body_complete = false;
+	request_complete = false;
+	assigned_server = NULL;
+	assigned_location = NULL;
+	cgi_pipe_rfd = 0;
+	cgi_output.clear();
 }
 
 void	Client::clear_request()

srcs/Client.hpp

@@ -55,9 +55,11 @@ class Client
 		bool					header_complete;
 		bool					body_complete;
 		bool					request_complete;
-		size_t					read_body_size; // unused for now
+		// size_t					read_body_size; // unused for now
 		ServerConfig			*assigned_server; // cant be const cause of error_pages.operator[]
 		const LocationConfig	*assigned_location;
+		int						cgi_pipe_rfd;
+		std::string				cgi_output;
 
 		// getters
 		int						get_cl_fd() const;

srcs/webserv/Webserv.hpp

@@ -44,6 +44,15 @@ void		signal_handler(int signum);
 
 # define MIME_TYPE_DEFAULT "application/octet-stream"
 
+struct	cgi_pipe_rfd
+{
+	int			fd;
+	pid_t		cgi_pid;
+	Client		*client;
+};
+bool	operator==(const cgi_pipe_rfd& lhs, int fd);
+bool	operator==(int fd, const cgi_pipe_rfd& rhs);
+
 class Webserv
 {
 	public:
@@ -65,6 +74,7 @@ class Webserv
 	private:
 		int	_epfd;
 		std::vector<listen_socket>	_listen_sockets;
+		std::vector<cgi_pipe_rfd>	_cgi_pipe_rfds;
 		std::vector<ServerConfig>	_servers;
 		std::vector<Client>			_clients;
 		std::map<int, std::string>	_http_status;
@@ -107,6 +117,8 @@ class Webserv
 		void		_close_client(int fd);
 		void		_close_all_clients();
 		void		_close_all_clients_fd();
+		void		_close_cgi_pipe_rfd(int fd);
+		void		_close_all_cgi_pipe_rfd();
 		void		_close_all_listen_sockets();
 		void		_reopen_lsocket(std::vector<listen_socket>::iterator it);
 		void		_handle_epoll_error_lsocket(uint32_t events, std::vector<listen_socket>::iterator it);
@@ -121,18 +133,26 @@ class Webserv
 	// cgi.cpp
 		bool		_is_cgi(Client *client, std::string path);
 		size_t		_cgi_pos(Client *client, std::string &path, size_t pos);
-		std::string	_exec_cgi(Client *client);
+		void		_exec_cgi(Client *client);
 		void		_set_env_vector(Client *client, std::vector<std::string> &env_vector);
 		void		_set_env_cstr(char *env_cstr[], std::vector<std::string> &env_vector);
 		std::string	_dup_env(std::string var, std::string val);
 		std::string	_dup_env(std::string var, int i);
-		std::string	_exec_script(Client *client, char *env[]);
+		void		_exec_script(Client *client, char *env[]);
 		void		_check_script_output(Client *client, std::string & output);
 		void		_check_script_status(Client *client, std::string & output);
 		void		_check_script_fields(Client *client, std::string & output);
 		void		_add_script_body_length_header(std::string & output);
 		void		_remove_body_leading_empty_lines(std::string & output);
 
+
+
+	void	_read_cgi_output(cgi_pipe_rfd &cgi_fd);
+	void	_handle_epoll_error_cgi_fd(uint32_t events, std::vector<cgi_pipe_rfd>::iterator it);
+	void	_cgi_epollhup(uint32_t events, std::vector<cgi_pipe_rfd>::iterator it);
+
+
+
 	///////////////////////
 		class ExecFail : public std::exception
 		{

srcs/webserv/cgi.cpp

@@ -66,9 +66,8 @@ size_t	Webserv::_cgi_pos(Client *client, std::string &path, size_t pos)
 	return NPOS;
 }
 
-std::string	Webserv::_exec_cgi(Client *client)
+void	Webserv::_exec_cgi(Client *client)
 {
-	std::string					script_output;
 	char*						env_cstr[19] = {NULL};
 	std::vector<std::string>	env_vector;
 	env_vector.reserve(18);
@@ -77,12 +76,11 @@ std::string	Webserv::_exec_cgi(Client *client)
 	_set_env_vector(client, env_vector);
 	try {
 		_set_env_cstr(env_cstr, env_vector);
-		script_output = _exec_script(client, env_cstr);
+		_exec_script(client, env_cstr);
 
 		while (env_cstr[i] != NULL)
 			delete[] env_cstr[i++];
-
+		return;
-		return script_output;
 	}
 	catch (const Webserv::ExecFail& e)
 	{
@@ -151,43 +149,17 @@ void	Webserv::_set_env_cstr(char *env_cstr[], std::vector<std::string> &env_vect
 	env_cstr[i] = NULL;
 }
 
-/* void	Webserv::_set_env_cstr(char *env_cstr[], std::vector<std::string> &env_vector)
-{
-	env_cstr[0]  = const_cast<char*>(env_vector[0].c_str());
-	env_cstr[1]  = const_cast<char*>(env_vector[1].c_str());
-	env_cstr[2]  = const_cast<char*>(env_vector[2].c_str());
-	env_cstr[3]  = const_cast<char*>(env_vector[3].c_str());
-	env_cstr[4]  = const_cast<char*>(env_vector[4].c_str());
-	env_cstr[5]  = const_cast<char*>(env_vector[5].c_str());
-	env_cstr[6]  = const_cast<char*>(env_vector[6].c_str());
-	env_cstr[7]  = const_cast<char*>(env_vector[7].c_str());
-	env_cstr[8]  = const_cast<char*>(env_vector[8].c_str());
-	env_cstr[9]  = const_cast<char*>(env_vector[9].c_str());
-	env_cstr[10] = const_cast<char*>(env_vector[10].c_str());
-	env_cstr[11] = const_cast<char*>(env_vector[11].c_str());
-	env_cstr[12] = const_cast<char*>(env_vector[12].c_str());
-	env_cstr[13] = const_cast<char*>(env_vector[13].c_str());
-	env_cstr[14] = const_cast<char*>(env_vector[14].c_str());
-	env_cstr[15] = const_cast<char*>(env_vector[15].c_str());
-	env_cstr[16] = const_cast<char*>(env_vector[16].c_str());
-	env_cstr[17] = const_cast<char*>(env_vector[17].c_str());
-	env_cstr[18] = NULL;
-} */
 
-#define STATUS_500 std::string("Status: 500" CRLF CRLF);
+void	Webserv::_exec_script(Client *client, char *env[])
-
-std::string	Webserv::_exec_script(Client *client, char *env[])
 {
 	#define RD 0
 	#define WR 1
-	#define CGI_BUF_SIZE 10
 	#define FD_WR_TO_CHLD fd_in[WR]
 	#define FD_WR_TO_PRNT fd_out[WR]
 	#define FD_RD_FR_CHLD fd_out[RD]
 	#define FD_RD_FR_PRNT fd_in[RD]
 
 	pid_t			pid;
-	char			buf[CGI_BUF_SIZE]; // WIP define buffer
 	char * const nll[1]	= {NULL};
 	std::string		script_output;
 	std::string		body = client->get_rq_body();
@@ -195,12 +167,12 @@ std::string	Webserv::_exec_script(Client *client, char *env[])
 	int				fd_out[2];
 	std::string		path;
 
-	pipe(fd_in);
+	::pipe(fd_in);
-	pipe(fd_out);
+	::pipe(fd_out);
 
 	pid = fork();
 	if (pid == -1)
-		perror("err fork()");
+		std::perror("err fork()");
 	else if (pid == 0) // child
 	{
 		std::signal(SIGPIPE, SIG_DFL);
@@ -212,14 +184,14 @@ std::string	Webserv::_exec_script(Client *client, char *env[])
 		::close(FD_RD_FR_CHLD);
 		if (dup2(FD_RD_FR_PRNT, STDIN_FILENO) == -1)
 		{
-			perror("err dup2()");
+			std::perror("err dup2()");
 			::close(FD_RD_FR_PRNT); // Valgind debug, not essential
 			::close(FD_WR_TO_PRNT); // Valgind debug, not essential
 			throw ExecFail();
 		}
 		if (dup2(FD_WR_TO_PRNT, STDOUT_FILENO) == -1)
 		{
-			perror("err dup2()");
+			std::perror("err dup2()");
 			::close(FD_RD_FR_PRNT); // Valgind debug, not essential
 			::close(FD_WR_TO_PRNT); // Valgind debug, not essential
 			throw ExecFail();
@@ -230,9 +202,9 @@ std::string	Webserv::_exec_script(Client *client, char *env[])
 
 		path = "." + client->get_rq_script_path(); // Wut ? Only relative path ?
 /*DEBUG*/std::cerr << "execve:[" << path << "]\n";
-		if (execve(path.c_str(), nll, env) == -1) // replace path for debug error forcing
+		if (::execve(path.c_str(), nll, env) == -1) // replace path for debug error forcing
 		{
-			perror("err execve()");
+			std::perror("err execve()");
 			::close(STDIN_FILENO); // Valgind debug, not essential
 			::close(STDOUT_FILENO); // Valgind debug, not essential
 			throw ExecFail();
@@ -240,37 +212,23 @@ std::string	Webserv::_exec_script(Client *client, char *env[])
 	}
 	else //parent
 	{
-		close(FD_RD_FR_PRNT);
+		::close(FD_RD_FR_PRNT);
-		close(FD_WR_TO_PRNT);
+		::close(FD_WR_TO_PRNT);
-		write(FD_WR_TO_CHLD, body.c_str(), body.size());
+		::write(FD_WR_TO_CHLD, body.c_str(), body.size()); // move this before the fork ?
-		close(FD_WR_TO_CHLD);
+		::close(FD_WR_TO_CHLD);
-		waitpid(-1, NULL, 0);
+
-		// We could maybe,
 		// add FD_RD_FR_CHLD to epoll,
-		// return to the main loop,
+		_epoll_update(FD_RD_FR_CHLD, EPOLLIN, EPOLL_CTL_ADD);
-		// read FD_RD_FR_CHLD each time epoll say its ready,
+		// stop monitoring client->fd until the cgi-script as done is job
-		// then try waitpid() with WNOHANG after each read.
+		_epoll_update(client->get_cl_fd(), 0, EPOLL_CTL_DEL);
-		// when waitpid() tell us its finish (or maybe when epoll return EPOLLHUP)
-		// then actually parse the script_output and send it to the client.
 
-		ssize_t	ret = 1;
+		client->cgi_pipe_rfd = FD_RD_FR_CHLD;
-		while (ret > 0)
+		struct cgi_pipe_rfd	new_cgi_fd;
-		{
+		new_cgi_fd.fd = FD_RD_FR_CHLD;
-			ret = read(FD_RD_FR_CHLD, buf, CGI_BUF_SIZE);
+		new_cgi_fd.cgi_pid = pid;
-			if (ret == -1)
+		new_cgi_fd.client = client;
-			{		
+		_cgi_pipe_rfds.push_back(new_cgi_fd);
-				std::perror("err recv()");
-				script_output = STATUS_500;
-				break;
 	}
-			script_output.append(buf, ret);
-		}
-		close(FD_RD_FR_CHLD);
-	}
-	if (script_output.empty())
-		script_output = STATUS_500;
-
-	return script_output;
 }
 
 void	Webserv::_check_script_output(Client *client, std::string & output)

srcs/webserv/close.cpp

@@ -4,7 +4,8 @@
 void	Webserv::_close_client(int fd)
 {
 	std::vector<Client>::iterator	it = _clients.begin();
-	while (it != _clients.end())
+	std::vector<Client>::iterator	it_end = _clients.end();
+	while (it != it_end)
 	{
 		if (*it == fd)
 		{
@@ -32,15 +33,48 @@ void	Webserv::_close_all_clients_fd()
 	while (it != it_end)
 	{
 		// _epoll_update(_clients.back().fd, 0, EPOLL_CTL_DEL); // normalement superflu, DEBUG
-		std::cerr << "close fd " << _clients.back().get_cl_fd() << "\n";
+		std::cerr << "close fd " << it->get_cl_fd() << "\n";
-		if (::close(_clients.back().get_cl_fd()) == -1)
+		if (::close(it->get_cl_fd()) == -1)
 			std::perror("err close()");
 		++it;
 	}
 }
 
-void	Webserv::_close_all_listen_sockets()
+// TODO : not sure for zombie cgi-proccess, need to verif
+void	Webserv::_close_cgi_pipe_rfd(int fd)
 {
+	std::vector<cgi_pipe_rfd>::iterator	it = _cgi_pipe_rfds.begin();
+	std::vector<cgi_pipe_rfd>::iterator	it_end = _cgi_pipe_rfds.end();
+	while (it != it_end)
+	{
+		if (*it == fd)
+		{
+			std::cerr << "close cgi-fd " << fd << "\n";
+			if (::close(fd) == -1)
+				std::perror("err close()");
+			_cgi_pipe_rfds.erase(it);
+			break;
+		}
+		++it;
+	}
+}
+
+void	Webserv::_close_all_cgi_pipe_rfd()
+{
+	std::vector<cgi_pipe_rfd>::iterator	it = _cgi_pipe_rfds.begin();
+	std::vector<cgi_pipe_rfd>::iterator	it_end = _cgi_pipe_rfds.end();
+	while (it != it_end)
+	{
+		std::cerr << "close cgi-fd " << it->fd << "\n";
+		if (::close(it->fd) == -1)
+			std::perror("err close()");
+		++it;
+	}
+	_cgi_pipe_rfds.clear();
+}
+
+void	Webserv::_close_all_listen_sockets()
+{ // TODO : change like clients (clear in place of pop_back)
 	while (!_listen_sockets.empty())
 	{
 		// _epoll_update(_listen_sockets.back().fd, 0, EPOLL_CTL_DEL); // normalement superflu, DEBUG

srcs/webserv/response.cpp

@@ -3,7 +3,7 @@
 
 enum send_return
 {
-	SEND_IN_PROGRESS, // unused
+	SEND_IN_PROGRESS,
 	SEND_COMPLETE,
 	SEND_CLOSE,
 };
@@ -40,9 +40,26 @@ int	Webserv::_send_response(Client *client)
 
 	std::cerr << "send()\n";
 
+	if (client->response.empty())
+	{
 		_append_base_headers(client);
 		if (!client->status)
+		{
 			_construct_response(client);
+			if (client->cgi_pipe_rfd)
+				return SEND_IN_PROGRESS;
+		}
+	}
+	else if (client->cgi_pipe_rfd)
+	{
+		// /*DEBUG*/ std::cout << "\n" B_PURPLE "[response]:" RESET "\n"; ::print_special(client->response); std::cout << B_PURPLE "-----------" RESET "\n\n";
+		// /*DEBUG*/ std::cout << "\n" B_PURPLE "[script output]:" RESET "\n"; ::print_special(script_output); std::cout << B_PURPLE "-----------" RESET "\n\n";
+		_check_script_output(client, client->cgi_output); // FD_CGI : adjust for client->cgi_output;
+		if (client->status < 400)
+			client->response += client->cgi_output;
+		// /*DEBUG*/ std::cout << "\n" B_PURPLE "[response + output]:" RESET "\n"; ::print_special(client->response); std::cout << B_PURPLE "-----------" RESET "\n\n";		
+	}
+
 	_insert_status_line(client);
 	if (client->status >= 400)
 		_error_html_response(client);
@@ -89,14 +106,7 @@ void	Webserv::_construct_response(Client *client)
 	path = _replace_url_root(client, client->get_rq_abs_path());
 	if (_is_cgi(client, path))
 	{
-		script_output = _exec_cgi(client);
+		_exec_cgi(client);
-//*DEBUG*/ std::cout << "\n" B_PURPLE "[response]:" RESET "\n"; ::print_special(client->response); std::cout << B_PURPLE "-----------" RESET "\n\n";
-//*DEBUG*/ std::cout << "\n" B_PURPLE "[script output]:" RESET "\n"; ::print_special(script_output); std::cout << B_PURPLE "-----------" RESET "\n\n";
-		_check_script_output(client, script_output);
-		if (client->status < 400)
-			client->response += script_output;
-//*DEBUG*/ std::cout << "\n" B_PURPLE "[response + output]:" RESET "\n"; ::print_special(client->response); std::cout << B_PURPLE "-----------" RESET "\n\n";
-
 		return;
 	}
 	_process_method(client, path);

srcs/webserv/run_loop.cpp

@@ -4,6 +4,96 @@
 #define MAX_EVENTS 42 // arbitrary
 #define TIMEOUT 3000
 
+// TODO: temp to move in own file
+bool	operator==(const cgi_pipe_rfd& lhs, int fd)
+	{ return lhs.fd == fd; }
+
+bool	operator==(int fd, const cgi_pipe_rfd& rhs)
+	{ return fd == rhs.fd; }
+
+#define BUFSIZE 8192 // (8Ko)
+#define STATUS_500 std::string("Status: 500" CRLF CRLF);
+void	Webserv::_read_cgi_output(cgi_pipe_rfd &cgi_fd)
+{
+	char	buf[BUFSIZE];
+	ssize_t	ret;
+	pid_t	wait_ret;
+
+	ret = ::read(cgi_fd.fd, buf, BUFSIZE);
+	std::cerr << "cgi read ret = " << ret << "\n";
+	if (ret == -1)
+	{
+		std::perror("err read(cgi_fd)");
+		cgi_fd.client->cgi_output = STATUS_500;
+	}
+	else if (ret == 0)
+	{
+		(void)0;
+	}
+	else
+	{
+		cgi_fd.client->cgi_output.append(buf, ret);
+	}
+
+
+	wait_ret = ::waitpid(cgi_fd.cgi_pid, NULL, WNOHANG);
+	std::cerr << "cgi waitpid ret = " << wait_ret << "\n";
+	if (wait_ret == 0 && ret == -1)
+	{
+		_epoll_update(cgi_fd.client->get_cl_fd(), EPOLLOUT, EPOLL_CTL_ADD);
+		_close_cgi_pipe_rfd(cgi_fd.fd);
+		// TODO: kill the child :)
+	}
+	else if (wait_ret == cgi_fd.cgi_pid)
+	{
+		_epoll_update(cgi_fd.client->get_cl_fd(), EPOLLOUT, EPOLL_CTL_ADD);
+		_close_cgi_pipe_rfd(cgi_fd.fd);
+	}
+	return;
+}
+
+void	Webserv::_handle_epoll_error_cgi_fd(uint32_t events, std::vector<cgi_pipe_rfd>::iterator it)
+{
+	(void)events;
+	std::cerr << "cgi EPOLLERR" << "\n";
+	
+	pid_t	wait_ret;
+	wait_ret = ::waitpid(it->cgi_pid, NULL, WNOHANG);
+	std::cerr << "cgi EPOLLHUP waitpid ret = " << wait_ret << "\n";
+	if (wait_ret == 0)
+	{
+		_epoll_update(it->client->get_cl_fd(), EPOLLOUT, EPOLL_CTL_ADD);
+		_close_cgi_pipe_rfd(it->fd);
+		// TODO: kill the child :)
+	}
+	else if (wait_ret == it->cgi_pid)
+	{
+		_epoll_update(it->client->get_cl_fd(), EPOLLOUT, EPOLL_CTL_ADD);
+		_close_cgi_pipe_rfd(it->fd);
+	}	
+}
+
+void	Webserv::_cgi_epollhup(uint32_t events, std::vector<cgi_pipe_rfd>::iterator it)
+{
+	(void)events;
+	std::cerr << "cgi EPOLLHUP" << "\n";
+
+	pid_t	wait_ret;
+	wait_ret = ::waitpid(it->cgi_pid, NULL, WNOHANG);
+	std::cerr << "cgi EPOLLHUP waitpid ret = " << wait_ret << "\n";
+	if (wait_ret == 0)
+	{
+		_epoll_update(it->client->get_cl_fd(), EPOLLOUT, EPOLL_CTL_ADD);
+		_close_cgi_pipe_rfd(it->fd);
+		// TODO: kill the child :)
+	}
+	else if (wait_ret == it->cgi_pid)
+	{
+		_epoll_update(it->client->get_cl_fd(), EPOLLOUT, EPOLL_CTL_ADD);
+		_close_cgi_pipe_rfd(it->fd);
+	}	
+}
+
 void	Webserv::run()
 {
 	std::cerr << "Server started\n";
@@ -11,7 +101,8 @@ void	Webserv::run()
 	int	nfds;
 	int	i;
 	int	count_loop = 0;
-	std::vector<listen_socket>::iterator	it_socket;
+	std::vector<listen_socket>::iterator	it_lsocket;
+	std::vector<cgi_pipe_rfd>::iterator		it_cgi_fd;
 
 	g_run = true;
 	while (g_run)
@@ -33,13 +124,23 @@ void	Webserv::run()
 		while (i < nfds)
 		{
 			try {
-				it_socket = std::find(_listen_sockets.begin(), _listen_sockets.end(), events[i].data.fd);
+				it_lsocket = std::find(_listen_sockets.begin(), _listen_sockets.end(), events[i].data.fd);
-				if (it_socket != _listen_sockets.end())
+				it_cgi_fd = std::find(_cgi_pipe_rfds.begin(), _cgi_pipe_rfds.end(), events[i].data.fd); // Could be moved in the loop to avoid useless find() call
+				if (it_lsocket != _listen_sockets.end())
 				{
 					if (events[i].events & EPOLLERR || events[i].events & EPOLLHUP)
-						_handle_epoll_error_lsocket(events[i].events, it_socket);
+						_handle_epoll_error_lsocket(events[i].events, it_lsocket);
 					else if (events[i].events & EPOLLIN)
-						_accept_connection(*it_socket);
+						_accept_connection(*it_lsocket);
+				}
+				else if (it_cgi_fd != _cgi_pipe_rfds.end())
+				{
+					if (events[i].events & EPOLLERR)
+						_handle_epoll_error_cgi_fd(events[i].events, it_cgi_fd);
+					else if (events[i].events & EPOLLHUP)
+						_cgi_epollhup(events[i].events, it_cgi_fd);
+					else if (events[i].events & EPOLLIN)
+						_read_cgi_output(*it_cgi_fd);
 				}
 				else
 				{