/**
 * threadpool.c
 *
 * This file will contain your implementation of a threadpool.
 */

#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <pthread.h>
#include <sys/time.h>

#include "threadpool.h"

#define DEBUG


#define timersub(a, b, result) do { \
(result)->tv_sec = (a)->tv_sec - (b)->tv_sec; \
(result)->tv_usec = (a)->tv_usec - (b)->tv_usec; \
if ((result)->tv_usec < 0) { \
--(result)->tv_sec; \
(result)->tv_usec += 1000000; \
} \
} while (0) 


/*----------------------------------------------------------------------
  Utility function: "elapsed-time fprintf".  fprintf, but prepends the 
    time difference between the first argument and now.
*/
void etfprintf(struct timeval then, ...) {
  va_list         args;
  struct timeval  now;
  struct timeval  diff;
  FILE           *out;
  char           *fmt;
  
  va_start(args, then);
  out = va_arg(args, FILE *);
  fmt = va_arg(args, char *);
  
  gettimeofday(&now, NULL);
  timersub(&now, &then, &diff);
  fprintf(out, "%03d.%06d:", diff.tv_sec, diff.tv_usec);
  vfprintf(out, fmt, args);
  fflush(NULL);
  va_end(args);
}

typedef enum {
  ALL_RUN, ALL_EXIT
} poolstate_t;

/*----------------------------------------------------------------------
  _threadpool is the internal threadpool structure that is cast to type 
    "threadpool" before it given out to callers.
  All accesses to a _threadpool (both read and write) must be protected 
    by that pool's mutex.  
  Worker threads wait on a "job_posted" signal; when one is received the 
    receiving thread (which has the mutex):
      * checks the pool state. If it is ALL_EXIT the thread decrements 
        the pool's 'live' count, yields the mutex and exits; otherwise 
        continues:
      * while the job field is non-null:
        * copy the job and arg fields into the thread, 
        * NULL out the pool's job and arg fields
        * signal "job_taken" 
        * yield the mutex and run the job on the arg.
        * grab the mutex
      * when a null job field is found, wait on "job_posted"
  The dispatcher thread dispatches a job as follows:
      * grab the pool's mutex
      * while the job field is not NULL: 
        * signal "job_posted"
        * wait on "job_taken"
      * copy job and args to pool, signal "job_posted", yield the mutex
*/
typedef struct _threadpool_st {
  struct timeval  created;    // When the threadpool was created.
  pthread_t      *threadArray;      // The threads themselves.

  pthread_mutex_t mutex;      // protects all vars declared below.


  //Initialization Condition Variable
  pthread_cond_t  thread_alive; //new worker thread :"I'm alive, ok to start someone else"

  //STEADY STATE Condition Variables
  pthread_cond_t  job_posted; // dispatcher: "Hey guys, there's a job!"
  pthread_cond_t  job_taken;  // a worker: "Got it!"

  //Destruction Condition Variable
  pthread_cond_t  thread_exiting; //worker thread: "I saw ALL_EXIT, over and out"

  
  poolstate_t     state;      // Threads check this before getting job.
  int             numThreadsInPool;      // Number of entries in threadArray.
  int             live;       // Number of live threads in pool (when
                              //   pool is being destroyed, live<=numThreadsInPool)
  int             dispatchCallsWaiting; //Number of threads waiting on job_taken until their 
                                        //jobs can be taken
  

  dispatch_fn     job;        // Job needing a thread to execute it
  void           *arg;        // Args for the job
} _threadpool;


/*----------------------------------------------------------------------
  Define the life of a working thread.
*/
void * do_work(void * owning_pool) {

  int myid;  //My thread Id

  // When we get a posted job, we copy it into these local vars.
  dispatch_fn  myjob;
  void        *myarg;  
  

  // Convert pointer to owning pool to proper type.
  _threadpool *pool = (_threadpool *) owning_pool;

#ifdef DEBUG
  etfprintf(pool->created, stderr, 
      " >>> Thread[%d] starting, grabbing mutex.\n", myid);
#endif
  
  // Grab mutex so we can begin waiting for a job
  if (0 != pthread_mutex_lock(&pool->mutex)) {
    perror("\nMutex lock failed!:");
    exit(EXIT_FAILURE);
  }

  // Remember my creation sequence number
  myid = pool->live;

#ifdef DEBUG
    etfprintf(pool->created, stderr,
        " <<< Thread[%d] signal thread_alive\n", myid);
#endif

  //JNM signal thread_alive
  pthread_cond_signal(&pool->thread_alive);

  
  // Main loop: wait for job posting, do job(s) ... forever
  for( ; ; ) {
    
#ifdef DEBUG
    etfprintf(pool->created, stderr,
        " <<< Thread[%d] waiting for signal.\n", myid);
#endif

    if (NULL == pool->job){  
      pthread_cond_wait(&pool->job_posted, &pool->mutex);
    
#ifdef DEBUG
      etfprintf(pool->created, stderr,
        " >>> Thread[%d] received signal.\n", myid);
#endif
  
      // We've just woken up and we have the mutex.  Check pool's state
      if (ALL_EXIT == pool->state){
	break;
      }
    }

    //could instead do until NULL == pool->job
    while (NULL != pool->job) {
      // while we find work to do
      myjob = pool->job;
      myarg = pool->arg;
      pool->job = NULL;
      pool->arg = NULL;
      pthread_cond_signal(&pool->job_taken);

#ifdef DEBUG
      etfprintf(pool->created, stderr,
          " <<< Thread[%d] yielding mutex, taking job.\n", myid);
#endif
  
      // Yield mutex so other jobs can be posted
      if (0 != pthread_mutex_unlock(&pool->mutex)) {
        perror("\n\nMutex unlock failed!:");
        exit(EXIT_FAILURE);
      }
      
      // Run the job we've taken
      myjob(myarg);
      
#ifdef DEBUG
      etfprintf(pool->created, stderr,
          " >>> Thread[%d] job done, grabbing mutex.\n", myid);
#endif
  
      // Grab mutex so we can grab posted job, or (if no job is posted)
      //   begin waiting for next posting.
      if (0 != pthread_mutex_lock(&pool->mutex)) {
        perror("\n\nMutex lock failed!:");
        exit(EXIT_FAILURE);
      }
    }
  }
      
  // If we get here, we broke from loop because state is ALL_EXIT.
  --pool->live;
  
#ifdef DEBUG
    etfprintf(pool->created, stderr, 
        " <<< Thread[%d] exiting (signalling 'job_taken').\n", myid);
#endif
  
  // This signals the destroyer that one thread has exited, so it can keep on destroying.
    pthread_cond_signal(&pool->thread_exiting);
  
  if (0 != pthread_mutex_unlock(&pool->mutex)) {
    perror("\n\nMutex unlock failed!:");
    exit(EXIT_FAILURE);
  }
  
  return NULL;
}  

/*----------------------------------------------------------------------
  Create a thread pool.
*/
threadpool create_threadpool(int num_threads_in_pool) {
  _threadpool *pool;  // pool we create and hand back
  int thisThread;              // work var

  // sanity check the argument
  if ((num_threads_in_pool <= 0) || (num_threads_in_pool > MAXT_IN_POOL))
    return NULL;

  // create the _threadpool struct
  pool = (_threadpool *) malloc(sizeof(_threadpool));
  if (pool == NULL) {
    fprintf(stderr, "\n\nOut of memory creating a new threadpool!\n");
    return NULL;
  }

  // initialize everything but the threadArray and live thread count
  pthread_mutex_init(&(pool->mutex), NULL);
  pthread_cond_init(&(pool->job_posted), NULL);
  pthread_cond_init(&(pool->job_taken), NULL);
  pthread_cond_init(&(pool->thread_alive), NULL);
  pthread_cond_init(&(pool->thread_exiting), NULL);
  pool->numThreadsInPool = num_threads_in_pool;
  pool->state = ALL_RUN;
  pool->job = NULL;
  pool->arg = NULL;
  gettimeofday(&pool->created, NULL);
  pool->dispatchCallsWaiting = 0;

  // create the array of threads within the pool
  pool->threadArray = 
      (pthread_t *) malloc (pool->numThreadsInPool * sizeof(pthread_t));
  if (NULL == pool->threadArray) {
    fprintf(stderr, "\n\nOut of memory allocating thread array!\n");
    free(pool);
    pool = NULL;
    return NULL;
  }

  // Grab the mutex
  if (0 != pthread_mutex_lock(&pool->mutex)) {
    perror("Mutex lock failed (!!):");
    exit(-1);
  }

  // bring each thread to life (update counters in loop so threads can
  //   access pool->live to find out their ID#
  for (thisThread = 0; thisThread < pool->numThreadsInPool; ++thisThread) {
    if (0 != pthread_create(pool->threadArray + thisThread, NULL, do_work, (void *) pool)) {
      unsigned int j;      
      perror("\n\nThread creation failed:");

      fprintf(stderr, "Forcibly canceling threads\n");
      for (j = 0; j < thisThread; ++j) {
	if (0 !=  pthread_cancel(*(pool->threadArray + j))){
	  exit(EXIT_FAILURE);
	}
      }

      free(pool);
      return NULL;
    }
    pthread_cond_wait(&pool->thread_alive, &pool->mutex);
    pool->live = thisThread+1;
    pthread_detach(pool->threadArray[thisThread]);  // automatic cleanup when thread exits.
  }
  
#ifdef DEBUG
  etfprintf(pool->created, stderr,
      " <<< Threadpool created with %d threads.\n", num_threads_in_pool);
#endif

  //we've see all threads created so we don't need this condition variable anymore
  if (0 != pthread_cond_destroy(&pool->thread_alive)) {
    perror("\nCondition Variable 'thread_alive' destruction failed!:");
    exit(EXIT_FAILURE);
  }

  if (0 != pthread_mutex_unlock(&pool->mutex)) {
    perror("\n\nMutex unlock failed!:");
    exit(EXIT_FAILURE);
  }  
  return (threadpool) pool;



}


/*----------------------------------------------------------------------
  Dispatch a thread
*/
void dispatch(threadpool from_me, dispatch_fn dispatch_to_here,
	      void *arg) {
  _threadpool *pool = (_threadpool *) from_me;

#ifdef DEBUG
  etfprintf(pool->created, stderr,
      " >>> Dispatcher: grabbing mutex.\n");
#endif
  
  // Grab the mutex
  if (0 != pthread_mutex_lock(&pool->mutex)) {
    perror("Mutex lock failed (!!):");
    exit(-1);
  }

  if (ALL_EXIT == pool->state){
    return ;
  }
  // If there's already a job posted, signal workers and wait until
  //   one of them takes it.
  while (NULL != pool->job) {

    pool->dispatchCallsWaiting++;
#ifdef DEBUG
    etfprintf(pool->created, stderr,
        " <<< Dispatcher: prev job untaken, signaling 'posted', %d threads waiting on 'job_taken'.\n",
        pool->dispatchCallsWaiting);
#endif
    pthread_cond_signal(&pool->job_posted);


    pthread_cond_wait(&pool->job_taken, &pool->mutex);
    pool->dispatchCallsWaiting--;

    //I was woken up not because my job is being
    //taken but because the thread pool is being
    //destroyed
    if (ALL_EXIT == pool->state){
      pthread_cond_signal(&pool->thread_exiting);
      return ;
    }
  }
  
  // Finally, there's room to post a job. Do so and signal workers.
#ifdef DEBUG
    etfprintf(pool->created, stderr,
        " <<< Dispatcher: posting job, signaling 'posted', yielding mutex\n");
#endif
  pool->job = dispatch_to_here;
  pool->arg = arg;
  pthread_cond_signal(&pool->job_posted);
  
  // Yield mutex so a worker can pick up the job
  if (0 != pthread_mutex_unlock(&pool->mutex)) {
    perror("\n\nMutex unlock failed!:");
    exit(EXIT_FAILURE);
  }
  
  return; 
}

/*----------------------------------------------------------------------
  Destroy a thread pool.  If there is a job still waiting for a thread
    to execute it, tough.  We set the ALL_EXIT flag anyways.
*/
void destroy_threadpool(threadpool destroyme) {
  _threadpool *pool = (_threadpool *) destroyme;

  // Cause all threads to exit. Because they were detached when created,
  //   the underlying memory for each is automatically reclaimed.
#ifdef DEBUG
  etfprintf(pool->created, stderr,
      " >>> Destroyer: grabbing mutex, setting state to ALL_EXIT. Live = %d\n",
      pool->live);
#endif
  
  // Grab the mutex
  if (0 != pthread_mutex_lock(&pool->mutex)) {
    perror("Mutex lock failed (!!):");
    exit(-1);
  }

  pool->state = ALL_EXIT;
  
  while (pool->live > 0) {
#ifdef DEBUG
    etfprintf(pool->created, stderr,
        " <<< Destroyer: signalling 'job_posted', waiting on 'job_taken'.\n");
#endif
    // get workers to check in ...
    pthread_cond_signal(&pool->job_posted);
    // ... and wake up when they check out.
    pthread_cond_wait(&pool->thread_exiting, &pool->mutex);
#ifdef DEBUG
    etfprintf(pool->created, stderr,
        " >>> Destroyer: received 'thread_exiting'. Live = %d\n", pool->live);
#endif
  }

  //Free any threads waiting in dispatch - their jobs will not
  //be performed but they will return to their callers
  while (pool->dispatchCallsWaiting > 0){

#ifdef DEBUG
    etfprintf(pool->created, stderr,
        " <<< Destroyer: signalling 'job_taken', waiting on 'thread_exiting'.\n");
#endif

    //get dispatch threads to return ...
    pthread_cond_signal(&pool->job_taken);

    //and wakeup with then check out
    pthread_cond_wait(&pool->thread_exiting, &pool->mutex);

#ifdef DEBUG
    etfprintf(pool->created, stderr,
        " >>> Destroyer: received 'thread_exiting'. Dispatch calls still waiting = %d\n", 
	pool->dispatchCallsWaiting);
#endif

  }
  

  
  // Null-out entries in pool's thread array; free array.
  memset(pool->threadArray, 0, pool->numThreadsInPool * sizeof(pthread_t));
  free(pool->threadArray);
  
  // Destroy the mutex and condition variables in the pool.
#ifdef DEBUG
  etfprintf(pool->created, stderr,
      " <<< Destroyer: releasing mutex prior to destroying it.\n");
#endif
  
  if (0 != pthread_mutex_unlock(&pool->mutex)) {
    perror("\n\nMutex unlock failed!:");
    exit(EXIT_FAILURE);
  }
  
#ifdef DEBUG
  etfprintf(pool->created, stderr,
      " --- Destroyer: destroying mutex.\n");
#endif
  
  if (0 != pthread_mutex_destroy(&pool->mutex)) {
    perror("\nMutex destruction failed!:");
    exit(EXIT_FAILURE);
  }

#ifdef DEBUG
  etfprintf(pool->created, stderr,
      " --- Destroyer: destroying conditional variables.\n");
#endif
  
  if (0 != pthread_cond_destroy(&pool->job_posted)) {
    perror("\nCondition Variable 'job_posted' destruction failed!:");
    exit(EXIT_FAILURE);
  }
  
  if (0 != pthread_cond_destroy(&pool->job_taken)) {
    perror("\nCondition Variable 'job_taken' destruction failed!:");
    exit(EXIT_FAILURE);
  }

  if (0 != pthread_cond_destroy(&pool->thread_exiting)) {
    perror("\nCondition Variable 'thread_exiting' destruction failed!:");
    exit(EXIT_FAILURE);
  }
  
  // Zero out all bytes of the pool
  memset(pool, 0, sizeof(_threadpool));
  
  // Free the pool and null out the pointer to it
  free(pool);
  pool = NULL;
  destroyme = NULL;
}

