/* mpm_worker.c - 아파치 Apache MPM 중 하나인 Worker MPM 입니다.  */

/* Copyright 2001-2005 The Apache Software Foundation or its licensors, as
 * applicable.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
  아래 코드들은 Apache MPM 중 Worker MPM 을 거의 가져온 코드들입니다.  아래의
  코드들은 모두 Apache 라이센스를 따릅니다.
 */

/*
  아래 코드는 Apache MPM 중 Worker MPM 을 가져온 코드들로써, Apache 코드들의 경우,
  ROVM 부분의 네트워크 쪽을 그대로 붙이기에는 코드가 너무 방대해지고, 다소 복잡해
  지기 때문에, 아래와 같이 최소한으로 간소화하여 정리하였다.

  하지만 최대한 Apache Worker MPM 전체 구조를 따를려고 하였다.

  (수정 부분)
  Apache Worker MPM 의 경우, 많은 전역 변수를 사용하도록 설계되어 있지만, 이
  코드는 전역 변수를 사용하지 않습니다.  (물론 thread-safe 하게 짜야 할 경우가
  없지만 말입니다..)
*/
#include <sys/types.h>
#include <unistd.h>
#include <stdlib.h>

#include "types.h"
#include "mpool.h"
#include "tktree.h"
#include "clstree.h"
#include "sha1.h"
#include "listen.h"
#include "rovm.h"

#include "thread.h"
#include "thread_mutex.h"
#include "thread_cond.h"

#include "mpm_common.h"
#include "mpm_worker_fdqueue.h"
#include "mpm_worker_pod.h"
#include "mpm_worker.h"

#include "connection.h"
#include "request.h"

#include "log.h"
#include "utils.h"

static rc_pool_t *global_pchild;

/* 아래 변수들은 signal 에 의해서 설정되는 값이기 때문에, thread-safe 하게
   묶을 수가 없어서 아래와 같이 전역 변수로 설정을 하였다.

   volatile just in case */
static int volatile shutdown_pending;
static int volatile restart_pending;
static int volatile is_graceful;
static volatile int child_fatal;
static int sick_child_detected;

/* The LISTENER_SIGNAL signal will be sent from the main thread to the
 * listener thread to wake it up for graceful termination (what a child
 * process from an old generation does when the admin does "apachectl
 * graceful").  This signal will be blocked in all threads of a child
 * process except for the listener thread.
 */
#define LISTENER_SIGNAL     SIGHUP

static void
sig_term (int sig)
{
  shutdown_pending = 1;
}

static void
restart (int sig)
{
  restart_pending = 1;
}

static void
set_signals (struct rovm *r)
{
#ifndef NO_USE_SIGACTION
  struct sigaction sa;
#endif
  
  if (!CONF_ONE_PROCESS (ROVM_CONF (r))) 
    rc_fatal_signal_setup (r);
  
#ifndef NO_USE_SIGACTION
  sigemptyset(&sa.sa_mask);
  sa.sa_flags = 0;
  
  sa.sa_handler = sig_term;
  if (sigaction(SIGTERM, &sa, NULL) < 0)
    rovm_log (r, ROVMLOG_WARNING, ROVMLOG_MARK, "sigaction(SIGTERM)");
#ifdef SIGINT
  if (sigaction(SIGINT, &sa, NULL) < 0)
    rovm_log (r, ROVMLOG_WARNING, ROVMLOG_MARK, "sigaction(SIGINT)");
#endif
#ifdef SIGXCPU
  sa.sa_handler = SIG_DFL;
  if (sigaction(SIGXCPU, &sa, NULL) < 0)
    rovm_log (r, ROVMLOG_WARNING, ROVMLOG_MARK, "sigaction(SIGXCPU)");
#endif
#ifdef SIGXFSZ
  sa.sa_handler = SIG_DFL;
  if (sigaction(SIGXFSZ, &sa, NULL) < 0)
    rovm_log (r, ROVMLOG_WARNING, ROVMLOG_MARK, "sigaction(SIGXFSZ)");
#endif
#ifdef SIGPIPE
  sa.sa_handler = SIG_IGN;
  if (sigaction(SIGPIPE, &sa, NULL) < 0)
    rovm_log (r, ROVMLOG_WARNING, ROVMLOG_MARK, "sigaction(SIGPIPE)");
#endif
  
  /* we want to ignore HUPs and AP_SIG_GRACEFUL while we're busy
   * processing one */
  sigaddset(&sa.sa_mask, SIGHUP);
  sa.sa_handler = restart;
  if (sigaction(SIGHUP, &sa, NULL) < 0)
    rovm_log (r, ROVMLOG_WARNING, ROVMLOG_MARK, "sigaction(SIGHUP)");
#else
  if (!CONF_ONE_PROCESS (ROVM_CONF (r))) 
    {
#ifdef SIGXCPU
      rc_signal (SIGXCPU, SIG_DFL);
#endif /* SIGXCPU */
#ifdef SIGXFSZ
      rc_signal (SIGXFSZ, SIG_DFL);
#endif /* SIGXFSZ */
    }
  
  rc_signal (SIGTERM, sig_term);
#ifdef SIGHUP
  rc_signal (SIGHUP, restart);
#endif /* SIGHUP */
#ifdef SIGPIPE
  rc_signal (SIGPIPE, SIG_IGN);
#endif /* SIGPIPE */
  
#endif
}


static void
close_worker_sockets (thread_starter *ts)
{
  int i;
  struct rovm *r = (struct rovm *) ts->arg;

  for (i = 0; i < CONF_THREADS_PER_CHILD (ROVM_CONF (r)); i++) 
    {
      if (ts->worker_sockets[i]) 
        {
          rc_socket_close (ts->worker_sockets[i]);
          ts->worker_sockets[i] = NULL;
        }
    }
}

static void clean_child_exit (int code)
{
  if (global_pchild) 
    mp_destroy (global_pchild);

  exit(code);
}

static void 
join_start_thread (rc_thread_t *start_thread_id)
{
  rc_status_t rv, thread_rv;
  
  rv = rc_thread_join (&thread_rv, start_thread_id);
  if (rv != RC_SUCCESS) 
    {
      rovm_log (NULL, ROVMLOG_CRIT, ROVMLOG_MARK,
		"rc_thread_join: unable to join the start "
		"thread");
    }
}

static void
wakeup_listener (thread_starter *ts)
{
/* The LISTENER_SIGNAL signal will be sent from the main thread to the
 * listener thread to wake it up for graceful termination (what a child
 * process from an old generation does when the admin does "apachectl
 * graceful").  This signal will be blocked in all threads of a child
 * process except for the listener thread.
 */
#define LISTENER_SIGNAL     SIGHUP

  ts->listener_may_exit = 1;

  if (!ts->listener_os_thread) 
    {
      /* XXX there is an obscure path that this doesn't handle perfectly:
       *     right after listener thread is created but before
       *     listener_os_thread is set, the first worker thread hits an
       *     error and starts graceful termination
       */
      return;
    }
  /*
   * we should just be able to "kill(ap_my_pid, LISTENER_SIGNAL)" on all
   * platforms and wake up the listener thread since it is the only thread
   * with SIGHUP unblocked, but that doesn't work on Linux
   */
#ifdef HAVE_PTHREAD_KILL
  pthread_kill(*ts->listener_os_thread, LISTENER_SIGNAL);
#else
  kill (ts->my_pid, LISTENER_SIGNAL);
#endif
}

#define ST_INIT              0
#define ST_GRACEFUL          1
#define ST_UNGRACEFUL        2

static void
signal_threads (thread_starter *ts, int mode)
{
  if (ts->terminate_mode == mode) 
    return;

  ts->terminate_mode = mode;
  
  /* in case we weren't called from the listener thread, wake up the
   * listener thread
   */
  wakeup_listener (ts);
  
  /* for ungraceful termination, let the workers exit now;
   * for graceful termination, the listener thread will notify the
   * workers to exit once it has stopped accepting new connections
   */
  if (mode == ST_UNGRACEFUL) 
    {
      ts->workers_may_exit = 1;
      rc_queue_interrupt_all (ts->worker_queue);
      rc_queue_info_term (ts->worker_queue_info);
      close_worker_sockets (ts);
    }
}

static void
join_workers (thread_starter *ts, 
	      rc_thread_t *listener, 
	      rc_thread_t **threads)
{
  int i;
  rc_status_t rv, thread_rv;
  
  if (listener) 
    {
      int iter;
      
      /* deal with a rare timing window which affects waking up the
       * listener thread...  if the signal sent to the listener thread
       * is delivered between the time it verifies that the
       * listener_may_exit flag is clear and the time it enters a
       * blocking syscall, the signal didn't do any good...  work around
       * that by sleeping briefly and sending it again
       */
      iter = 0;
      while (iter < 10 &&
#ifdef HAVE_PTHREAD_KILL
	     pthread_kill (*ts->listener_os_thread, 0)
#else
	     kill (ts->my_pid, 0)
#endif
	     == 0) 
	{
	  /* listener not dead yet */
	  rc_sleep (rc_time_make (0, 500000));
	  wakeup_listener (ts);
	  ++iter;
        }
      if (iter >= 10) 
	rovm_log (NULL, ROVMLOG_DEBUG, ROVMLOG_MARK, "the listener thread didn't exit");
      else 
	{
	  rv = rc_thread_join (&thread_rv, listener);
	  if (rv != RC_SUCCESS) 
	    {
	      rovm_log (NULL, ROVMLOG_CRIT, ROVMLOG_MARK,
			"rc_thread_join: unable to join listener thread");
            }
        }
    }
  
  for (i = 0; i < CONF_THREADS_PER_CHILD (ROVM_CONF ((struct rovm *) ts->arg)); i++) 
    {
      if (threads[i])
	{ /* if we ever created this thread */
	  rv = rc_thread_join (&thread_rv, threads[i]);
	  if (rv != RC_SUCCESS)
	    {
	      rovm_log (NULL, ROVMLOG_CRIT, ROVMLOG_MARK,
			"rc_thread_join: unable to join worker "
			"thread %d",
			i);
            }
        }
    }
}

static void 
unblock_signal(int sig)
{
  sigset_t sig_mask;
  
  sigemptyset (&sig_mask);
  sigaddset (&sig_mask, sig);
#if defined(SIGPROCMASK_SETS_THREAD_MASK)
  sigprocmask (SIG_UNBLOCK, &sig_mask, NULL);
#else
  pthread_sigmask (SIG_UNBLOCK, &sig_mask, NULL);
#endif
}

static void
dummy_signal_handler (int sig)
{
  /* XXX If specifying SIG_IGN is guaranteed to unblock a syscall,
         then we don't need this goofy function.  */
}

static void
rc_close_listeners (struct rovm *r)
{
  rc_listen_rec *lr;

  for (lr = ROVM_LISTENERS (r); lr; lr = lr->next) 
    {
      rc_socket_close (lr->sd);
      lr->active = 0;
    }
}

static void *
listener_thread (rc_thread_t *thd, void *arg)
{
  void *csd = NULL;
  int have_idle_worker = 0;
  rc_status_t rv;
  rc_pool_t *ptrans = NULL;            /* Pool for per-transaction stuff */
  proc_info *ti = (proc_info *) arg;
  thread_starter *ts = (thread_starter *) ti->ts;
  struct rovm *r = (struct rovm *) ts->arg;
  rc_listen_rec *lr;

  /* Unblock the signal used to wake this thread up, and set a handler for
     it.  */
  unblock_signal (LISTENER_SIGNAL);
  rc_signal (LISTENER_SIGNAL, dummy_signal_handler);

  /*
    현재 listener_thread 는 multi-listening 을 지원하고 있지 않다.  그래에 그에 대한
    poll 기능이 apache 에는 존재하지만, 여기에서는 존재하지 않는다.
   */

  while (1)
    {
      if (ts->listener_may_exit)
        break;
      
      if (!have_idle_worker) 
        {
          /* the following pops a recycled ptrans pool off a stack
           * if there is one, in addition to reserving a worker thread
           */
          rv = rc_queue_info_wait_for_idler (ts->worker_queue_info, &ptrans);
          if (RC_STATUS_IS_EOF (rv))
            break; /* we've been signaled to die now */
          else if (rv != RC_SUCCESS) 
            {
              rovm_log (NULL, ROVMLOG_EMERG, ROVMLOG_MARK,
                        "rc_queue_info_wait failed. Attempting to "
                        " shutdown process gracefully.");
              signal_threads (ts, ST_GRACEFUL);
              break;
            }
          have_idle_worker = 1;
        }

      if (!ROVM_LISTENERS (r)->next)
        /* Only one listener, so skip the poll */
        lr = ROVM_LISTENERS (r);
      else
        {
          rovm_log (NULL, ROVMLOG_EMERG, ROVMLOG_MARK,
                    "Not supported multi listening yet.");
          signal_threads (ts, ST_GRACEFUL);
          break;
        }
      
      if (!ts->listener_may_exit)
        {
          if (ptrans == NULL) 
            {
              /* we can't use a recycled transaction pool this time.
               * create a new transaction pool */
              rc_allocator_t *allocator;
              
              rc_allocator_create (&allocator);
              mp_create_ex (&ptrans, PROCREC_PCONF (ROVM_PROCESS (r)), NULL, allocator);
              rc_allocator_owner_set (allocator, ptrans);
            }
          mp_tag (ptrans, "transaction");
          rv = lr->accept_func (&csd, lr, ptrans);
          if (rv == RC_EGENERAL) 
            {
              /* E[NM]FILE, ENOMEM, etc */
              ts->resource_shortage = 1;
              signal_threads (ts, ST_GRACEFUL);
            }
          if (csd != NULL) 
            {
              rv = rc_queue_push (ts->worker_queue, csd, ptrans);
              if (rv) 
                {
                  /* trash the connection; we couldn't queue the connected
                   * socket to a worker
                   */
                  rc_socket_close(csd);
                  rovm_log (NULL, ROVMLOG_CRIT, ROVMLOG_MARK, "ap_queue_push failed");
                }
              else 
                have_idle_worker = 0;
            }
        }
    }

  rc_close_listeners (r);
  rc_queue_term (ts->worker_queue);

  /* wake up the main thread */
  kill (ts->my_pid, SIGTERM);

  rc_thread_exit (thd, RC_SUCCESS);
  
  return NULL;
}

/*****************************************************************
  Child process main loop.

  이 함수가 호출이 되었다는 것은 아래와 같은 것을 말한다고 할 수 있을 것이다.

    1) Listener Thread 와 Worker Thread 간 연동이 잘되고 있다.
    2) Socket 값을 얻었기 때문에, 이를 이용하여 계속 사용자가 보낸 값을
       읽을 수 있다.
    3) 본격적인 사용자가 보내는 buffer 를 처리할 시간이다.
    4) 메모리 POOL 인자 P 를 이용하면 처리가 끝난 시점에 내용은 모두 
       지워질 것이다.  그리고 다음 연결시 재사용될 것이다.
*/

static void
process_socket (thread_starter *ts, rc_pool_t *p, rc_socket_t *sock)
{
  conn_rec *current_conn;

  current_conn = rc_create_connection (ts, p, sock);
  if (current_conn)
    {
      rc_process_connection (current_conn);
      rc_close_connection (current_conn);
    }
}

static void * 
worker_thread (rc_thread_t *thd, void *arg)
{
  proc_info *ti = (proc_info *) arg;
  thread_starter *ts = ti->ts;
  int thread_slot = ti->tid;
  int is_idle = 0;
  rc_status_t rv;
  rc_socket_t *csd = NULL;
  rc_pool_t *last_ptrans = NULL;
  rc_pool_t *ptrans;                /* Pool for per-transaction stuff */

  while (!ts->workers_may_exit) 
    {
      if (!is_idle) 
        {
          rv = rc_queue_info_set_idle (ts->worker_queue_info, last_ptrans);
          last_ptrans = NULL;
          if (rv != RC_SUCCESS) 
            {
              rovm_log (NULL, ROVMLOG_EMERG, ROVMLOG_MARK,
                        "rc_queue_info_set_idle failed. Attempting to "
                        "shutdown process gracefully.");
              signal_threads (ts, ST_GRACEFUL);
              break;
            }
          is_idle = 1;
        }

worker_pop:
      if (ts->workers_may_exit)
        break;

      rv = rc_queue_pop (ts->worker_queue, &csd, &ptrans);
      
      if (rv != RC_SUCCESS) 
        {
          /* We get RC_EOF during a graceful shutdown once all the connections
           * accepted by this server process have been handled.
           */
          if (RC_STATUS_IS_EOF (rv)) 
            break;

          /* We get RC_EINTR whenever ap_queue_pop() has been interrupted
           * from an explicit call to ap_queue_interrupt_all(). This allows
           * us to unblock threads stuck in ap_queue_pop() when a shutdown
           * is pending.
           *
           * If workers_may_exit is set and this is ungraceful termination/
           * restart, we are bound to get an error on some systems (e.g.,
           * AIX, which sanity-checks mutex operations) since the queue
           * may have already been cleaned up.  Don't log the "error" if
           * workers_may_exit is set.
           */
          else if (RC_STATUS_IS_EINTR (rv))
            goto worker_pop;
          /* We got some other error. */
          else if (!ts->workers_may_exit)
            rovm_log (NULL, ROVMLOG_CRIT, ROVMLOG_MARK, "ap_queue_pop failed");

          continue;
        }

      is_idle = 0;
      ts->worker_sockets[thread_slot] = csd;
      process_socket (ts, ptrans, csd);
      ts->worker_sockets[thread_slot] = NULL;
      mp_clear (ptrans);
      last_ptrans = ptrans;
    }

  rc_thread_exit (thd, RC_SUCCESS);
  return NULL;
}

static void
create_listener_thread (thread_starter *ts)
{
  int my_child_num = ts->child_num_arg;
  rc_threadattr_t *thread_attr = ts->threadattr;
  proc_info *my_info;
  rc_status_t rv;

  my_info = (proc_info *) malloc (sizeof (proc_info));
  my_info->pid = my_child_num;
  my_info->tid = -1; /* listener thread doesn't have a thread slot */
  my_info->sd = 0;
  my_info->ts = ts;

  rv = rc_thread_create (&ts->listener, thread_attr, listener_thread, my_info, ts->p);
  if (rv != RC_SUCCESS) 
    {
      rovm_log (NULL, ROVMLOG_ALERT, ROVMLOG_MARK,
		"rc_thread_create: unable to create listener thread");
      clean_child_exit (RCEXIT_CHILDSICK);
    }

  rc_os_thread_get (&ts->listener_os_thread, ts->listener);
}

/**
   하나의 프로세스에 대해 CONF_THREADS_PER_CHILD 에 해당하는 worker thread 갯수와
   하나의 listener thread 를 open 한다.
 */
static void * 
start_threads (rc_thread_t *thd, void *arg)
{
  rc_status_t rv;
  proc_info *my_info;
  int i, listener_started = 0, threads_created = 0;
  int loops = 0, prev_threads_created = 0;
  thread_starter *ts = (thread_starter *) arg;
  int child_num_arg = ts->child_num_arg;
  int my_child_num = child_num_arg;
  rc_thread_t **threads = ts->threads;
  rc_threadattr_t *thread_attr = ts->threadattr;
  rc_pool_t *pchild = ts->p;
  struct rovm *r = (struct rovm *) ts->arg;

  /* We must create the fd queues before we start up the listener
   * and worker threads. */
  ts->worker_queue = mp_calloc (pchild, sizeof (*ts->worker_queue));
  rv = rc_queue_init (ts->worker_queue, CONF_THREADS_PER_CHILD (ROVM_CONF (r)), pchild);
  if (rv != RC_SUCCESS) 
    {
      rovm_log (NULL, ROVMLOG_ALERT, ROVMLOG_MARK, "rc_queue_init () failed");
      clean_child_exit (RCEXIT_CHILDFATAL);
    }

  rv = rc_queue_info_create (&ts->worker_queue_info, pchild, CONF_THREADS_PER_CHILD (ROVM_CONF (r)));
  if (rv != RC_SUCCESS) 
    {
      rovm_log (NULL, ROVMLOG_ALERT, ROVMLOG_MARK, "ap_queue_info_create() failed");
      clean_child_exit (RCEXIT_CHILDFATAL);
    }

  ts->worker_sockets = mp_calloc (pchild, CONF_THREADS_PER_CHILD (ROVM_CONF (r)) * sizeof(rc_socket_t *));
  
  while (1) 
    {
      /* threads_per_child does not include the listener thread */
      for (i = 0; i < CONF_THREADS_PER_CHILD (ROVM_CONF (r)); i++)
	{
	  my_info = (proc_info *) malloc (sizeof (proc_info));
	  if (my_info == NULL) 
	    {
	      rovm_log (NULL, ROVMLOG_ALERT, ROVMLOG_MARK, "malloc: out of memory");
	      clean_child_exit (RCEXIT_CHILDFATAL);
            }
	  my_info->pid = my_child_num;
	  my_info->tid = i;
	  my_info->sd = 0;
          my_info->ts = ts;

	  rv = rc_thread_create (&threads[i], thread_attr, worker_thread, my_info, pchild);
	  if (rv != RC_SUCCESS)
	    {
	      rovm_log (NULL, ROVMLOG_ALERT, ROVMLOG_MARK, 
			"rc_thread_create: unable to create worker thread");
	      clean_child_exit (RCEXIT_CHILDSICK);
            }
	  threads_created++;
        }

      /* Start the listener only when there are workers available */
      if (!listener_started && threads_created) 
	{
	  create_listener_thread (ts);
	  listener_started = 1;
	}

      if (threads_created == CONF_THREADS_PER_CHILD (ROVM_CONF (r))) 
	break;

      /* wait for previous generation to clean up an entry */
      rc_sleep (rc_time_from_sec (1));
      ++loops;

      if (loops % 120 == 0) 
	{ /* every couple of minutes */
	  if (prev_threads_created == threads_created) 
	    {
	      rovm_log (NULL, ROVMLOG_DEBUG, ROVMLOG_MARK,
			"child %d isn't taking over "
			"slots very quickly (%d of %d)",
			ts->my_pid, threads_created, CONF_THREADS_PER_CHILD (ROVM_CONF (r)));
	    }
	  prev_threads_created = threads_created;
	}
    }
  
  rc_thread_exit (thd, RC_SUCCESS);

  return NULL;  
}

/**
   `One Process' 모드로 실행 중일 경우, SIGTERM 과 SIGINT 외의 Signal 을
   블럭하는데 사용되는 callback 함수입니다.

   @param signum 받은 Signal 번호
 */
static int 
check_signal (int signum)
{
  switch (signum) 
    {
    case SIGTERM:
    case SIGINT:
      return 1;
    }
  return 0;
}

/**
   (다수의 fork 를 사용할 경우) Children 프로세스이며, 이 과정에서 listener thread 와
   worker thread 를 생성하게 된다.

   @param r		ROVM 구조체
 */
void
child_main (r)
     struct rovm *r;
{
  rc_status_t rv;
  thread_starter *ts;
  rc_threadattr_t *thread_attr;
  rc_thread_t *start_thread_id, **threads;
  rc_pool_t *pchild;

  mp_create (&pchild, PROCREC_PCONF (ROVM_PROCESS (r)));
  global_pchild = pchild;

  threads = (rc_thread_t **) rc_calloc (1, sizeof (rc_thread_t *) * CONF_THREADS_PER_CHILD (ROVM_CONF (r)));
  if (!threads)
    {
      rovm_log (NULL, ROVMLOG_ERR, ROVMLOG_MARK, "malloc: out of memory");
      clean_child_exit (RCEXIT_CHILDFATAL);
    }
  
  rc_threadattr_create (&thread_attr, pchild);
  /* 0 는 PTHREAD_CREATE_JOINABLE 를 의미한다.  */
  rc_threadattr_detach_set (thread_attr, 0);

  ts = (thread_starter *) mp_alloc (pchild, sizeof (*ts));
  ts->threads = threads;
  ts->listener = NULL;
  ts->child_num_arg = -1;
  ts->threadattr = thread_attr;
  ts->p = pchild;
  ts->my_pid = getpid ();
  ts->arg = (void *) r;
  ts->listener_may_exit = 0;
  ts->workers_may_exit = 0;
  ts->resource_shortage = 0;
  ts->terminate_mode = ST_INIT;

  rv = rc_thread_create (&start_thread_id, thread_attr, start_threads, ts, pchild);
  if (rv != RC_SUCCESS) 
    {
      rovm_log (NULL, ROVMLOG_ERR, ROVMLOG_MARK,
		"rc_thread_create: unable to create worker thread");
      clean_child_exit (RCEXIT_CHILDSICK);
    }

  if (CONF_ONE_PROCESS (ROVM_CONF (r)))
    {
      /* Block until we get a terminating signal. */
      rc_signal_thread (check_signal);
      /* make sure the start thread has finished; signal_threads()
         and join_workers() depend on that  */
      /* XXX join_start_thread() won't be awakened if one of our
             threads encounters a critical error and attempts to
             shutdown this child  */
      join_start_thread (start_thread_id);
      signal_threads (ts, ST_UNGRACEFUL);
      /* A terminating signal was received. Now join each of the workers to clean them up.
         If the worker already exited, then the join frees their resources and returns.
         If the worker hasn't exited, then this blocks until they have (then cleans up).  */
      join_workers (ts, ts->listener, threads);
    }
  else
    {
      /* remove SIGTERM from the set of blocked signals...  if one of
         * the other threads in the process needs to take us down
         * (e.g., for MaxRequestsPerChild) it will send us SIGTERM
         */
      unblock_signal (SIGTERM);
      rc_signal (SIGTERM, dummy_signal_handler);

      while (1)
        {
          rv = rc_mpm_pod_check (ROVM_POD (r));
          if (rv == RC_POD_NORESTART)
            {
              /* see if termination was triggered while we slept */
              switch (ts->terminate_mode) 
                {
                case ST_GRACEFUL:
                  rv = RC_POD_GRACEFUL;
                  break;
                case ST_UNGRACEFUL:
                  rv = RC_POD_RESTART;
                  break;
                }
            }
          if (rv == RC_POD_GRACEFUL || rv == RC_POD_RESTART)
            {
              /* make sure the start thread has finished;
               * signal_threads() and join_workers depend on that
               */
              join_start_thread (start_thread_id);
              signal_threads (ts, rv == RC_POD_GRACEFUL ? ST_GRACEFUL : ST_UNGRACEFUL);
              break;
            }
        }

      /* A terminating signal was received. Now join each of the
       * workers to clean them up.
       *   If the worker already exited, then the join frees
       *   their resources and returns.
       *   If the worker hasn't exited, then this blocks until
       *   they have (then cleans up).
       */
      join_workers (ts, ts->listener, threads);
    }

  rc_free (threads);

  clean_child_exit (ts->resource_shortage ? RCEXIT_CHILDSICK : 0);
}

static void
just_die (int sig)
{
  clean_child_exit (0);
}

void
sb_set_parent_pid (struct rovm *r, int slot, pid_t pid)
{
  SB_PARENT_IDX (ROVM_SCOREBOARD (r), slot)->pid = pid;
}

pid_t
sb_get_parent_pid (struct rovm *r, int slot)
{
  return SB_PARENT_IDX (ROVM_SCOREBOARD (r), slot)->pid;
}

static int
make_child (r, slot)
     struct rovm *r;
     int slot;
{
  int pid;

  if (slot + 1 > CONF_MAX_DAEMONS_LIMIT (ROVM_CONF (r)))
    CONF_MAX_DAEMONS_LIMIT (ROVM_CONF (r)) = slot + 1;

  if (CONF_ONE_PROCESS (ROVM_CONF (r)))
    {
      set_signals (r);
      sb_set_parent_pid (r, slot, getpid ());
      child_main (r, slot);
    }

  if ((pid = fork ()) == -1) 
    {
      rovm_log (NULL, ROVMLOG_ERR, ROVMLOG_MARK, 
                "fork: Unable to fork new process");
      
      /* fork didn't succeed. */
      
      /* In case system resources are maxxed out, we don't want
         Apache running away with the CPU trying to fork over and
         over and over again. */
      rc_sleep (rc_time_from_sec (10));
      
      return -1;
    }

  if (!pid)
    {
      rc_signal (SIGTERM, just_die);
      child_main (r);
      
      clean_child_exit (0);
    }

  if (sb_get_parent_pid (r, slot))
    {
      rovm_log (NULL, ROVMLOG_DEBUG, ROVMLOG_MARK, 
                "taking over scoreboard slot from %" RC_PID_T_FMT "",
                sb_get_parent_pid (r, slot));
      rovm_log (NULL, ROVMLOG_DEBUG, ROVMLOG_MARK, "TODO");
    }

  sb_set_parent_pid (r, slot, pid);

  return 0;
}

/* start up a bunch of children */
static void
startup_children (r, number_to_start)
     struct rovm *r;
     int number_to_start;
{
  int i;
  
  for (i = 0; number_to_start && i < CONF_DAEMONS_LIMIT (ROVM_CONF (r)); ++i)
    {
      if (make_child (r, i) < 0)
        {
          break;
        }
      --number_to_start;
    }
}

static void
perform_idle_server_maintenance (r)
     struct rovm *r;
{
  if (restart_pending || shutdown_pending)
    rc_mpm_pod_signal (ROVM_POD (r), TRUE);
}

int 
find_child_by_pid (struct rovm *r, rc_proc_t *pid)
{
  int i;
  int max_daemons_limit = CONF_DAEMONS_TO_START (ROVM_CONF (r));

  for (i = 0; i < max_daemons_limit; ++i) 
    {
      if (SB_PARENT_IDX (ROVM_SCOREBOARD (r), i)->pid == pid->pid) 
        return i;
    }

  return -1;
}

/**
   MPM 에서 main loop 를 수행할 예정이었던 함수이다. ㅡ.ㅡ;  원래 목적은 apache 와 같이
   fork 를 통해서 여러 자식 process 를 생성한 후, 많은 thread 를 한번에 생성할 예정이었는데,
   현재는 fork 를 하지 않기 때문에, 함수가 굉장히 간단해 졌다.  하지만 앞으로 업데이트가
   될 예정인 함수이다.

   @param r     ROVM 구조체
 */
void
server_main_loop (r, remaining_children_to_start)
     struct rovm *r;
     int remaining_children_to_start;
{
  int status, processed_status, child_slot;
  rc_exit_why_e exitwhy;
  rc_proc_t pid;

  while (!restart_pending && !shutdown_pending)
    {
      rc_wait_or_timeout (&exitwhy, &status, &pid, PROCREC_PCONF (ROVM_PROCESS (r)));

      if (pid.pid != -1)
        {
          processed_status = rc_process_child_status (&pid, exitwhy, status);
          if (processed_status == RCEXIT_CHILDFATAL)
            {
              shutdown_pending = 1;
              child_fatal = 1;
              return;
            }
          else if (processed_status == RCEXIT_CHILDSICK) 
            {
              /* tell perform_idle_server_maintenance to check into this
               * on the next timer pop
               */
              sick_child_detected = 1;
            }

          /* 여기에 도달했다는 것은 child 가 죽었다는 것을 뜻합니다.  */
          child_slot = find_child_by_pid (r, &pid);
          if (child_slot >= 0)
            {
              sb_set_parent_pid (r, child_slot, 0);
              if (remaining_children_to_start
                  && child_slot < CONF_DAEMONS_LIMIT (ROVM_CONF (r)))
                {
                  make_child (r, child_slot);
                  remaining_children_to_start--;
                }
              else
                fprintf (stderr, "hmm");
            }
          else if (is_graceful) 
            {
              /* Great, we've probably just lost a slot in the
                 scoreboard.  Somehow we don't know about this child.  */
              rovm_log (NULL, ROVMLOG_WARNING, ROVMLOG_MARK, 
                        "long lost child came home! (pid %ld)", (long) pid.pid);
            }
             
          continue;
        }
      else if (remaining_children_to_start)
        {
          /* we hit a 1 second timeout in which none of the previous
           * generation of children needed to be reaped... so assume
           * they're all done, and pick up the slack if any is left.
           */
          startup_children (r, remaining_children_to_start);
          remaining_children_to_start = 0;
          /* In any event we really shouldn't do the code below because
           * few of the servers we just started are in the IDLE state
           * yet, so we'd mistakenly create an extra server.
           */
          continue;
        }

      perform_idle_server_maintenance (r);
    }
  
  return;
}

/**
   mpm_run () 함수가 실행되기 전에 미리 설정해 둘 필요가 있는 부분에 대해서
   이 함수에서 실행이 됩니다.

   @param r     ROVM 구조체
 */
int
mpm_pre_config (r)
     struct rovm *r;
{
  static int restart_num = 0;
  rc_status_t rv;

  if (restart_num++ == 0) 
    {
      is_graceful = 0;

      if (!CONF_ONE_PROCESS (ROVM_CONF (r)))
        {
          rv = rc_proc_detach (CONF_NO_DETACH (ROVM_CONF (r))
                               ? RC_PROC_DETACH_FOREGROUND 
                               : RC_PROC_DETACH_DAEMONIZE);
          if (rv != RC_SUCCESS)
            {
              rovm_log (NULL, ROVMLOG_CRIT, ROVMLOG_MARK, "rc_proc_detach failed");
              return -1;
            }
        }
    }

  return 0;
}

/**
   MPM 의 entry 포인터.

   @param r     ROVM 구조체

   @return 성공시 값을 0 을 반환하며, 그렇지 않을 경우, -1 을 반환한다.
 */
int
mpm_run (r)
     struct rovm *r;
{
  int remaining_children_to_start;
  rc_status_t rv;

  /* Multi-Process 를 이용하영 서비스를 시작할 경우는, POD 를 사용하기 때문에,
     첫 부분에서 설정을 해 줍니다.  */
  if (!CONF_ONE_PROCESS (ROVM_CONF (r)))
    {
      if ((rv = rc_mpm_pod_open (PROCREC_PCONF (ROVM_PROCESS (r)), &ROVM_POD (r), r)))
        {
          rovm_log (NULL, ROVMLOG_CRIT, ROVMLOG_MARK, "Could not open pipe-of-death.");
          return -1;
        }
    }

  set_signals (r);

  remaining_children_to_start = CONF_DAEMONS_TO_START (ROVM_CONF (r));
  if (remaining_children_to_start > CONF_DAEMONS_LIMIT (ROVM_CONF (r)))
    remaining_children_to_start = CONF_DAEMONS_LIMIT (ROVM_CONF (r));

  if (!is_graceful)
    {
      startup_children (r, remaining_children_to_start);
      remaining_children_to_start = 0;
    }

  restart_pending = shutdown_pending = 0;

  server_main_loop (r, remaining_children_to_start);

  if (shutdown_pending)
    {
      /* Time to shut down:
         Kill child processes, tell them to call child_exit, etc...  */
      rc_mpm_pod_killpg (ROVM_POD (r), CONF_DAEMONS_LIMIT (ROVM_CONF (r)), FALSE);
      rc_reclaim_child_processes (r, 1);

      if (!child_fatal)
        rovm_log (NULL, ROVMLOG_NOTICE, ROVMLOG_MARK, "caught SIGTERM, shutting down");
      else
        rovm_log (NULL, ROVMLOG_NOTICE, ROVMLOG_MARK, "TODO: child_fatal");

      return 1;
    }

  /* 여기에 도달했다는 것은 restart 를 의미합니다.  */

  return 0;
}
