// version 0.5 modifed by ian a. mason
// may 26 @ u.n.e
//
// reads and writes blocks rather than 
// the whole big matrix 
#include <stdlib.h>
#include <pvm3.h>
#include <stdio.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/types.h>
#include <time.h>
#include <limits.h>
#include "mmlib.h"

#define VERSION     "mm.0.5"  
                    //make this automatic in beta
#define ENCODING    PvmDataDefault 
                    //PvmDataRaw
#define MAXNTIDS    100
#define MAXROW      100
#define ATAG        2
#define BTAG        3
#define VERBOSE     0
#define PERMS       0666


int parse_args(int, char*[], int*, int*, int *, int []);

int main(int argc, char* argv[]){
    int ntask, info, mytid, mygid;
    int i, j, m, blksize, matrix_size; 
    int row, col, up, down;
    int child[MAXNTIDS-1];
    int myrow[MAXROW];
    /* A = P[0] B = P[1] C = P[2]              */
    char   P[3][PATH_MAX];
    /* A = F[0] B = F[1] C = F[2]              */          
    int    F[3];
    /* A = M[0] B = M[1] C = M[2]  TMP = M[3]  */
    int *M[4];        

    //real time for the time being
    int start_time = time(NULL); 
    
    if(parse_args(argc, argv, &m, &blksize, &ntask, F) < 0) exit(0);
    
    mytid = pvm_mytid();

    //tinkerville
    pvm_setopt(PvmRoute, PvmRouteDirect);

    if (mytid < 0){
      pvm_perror(argv[0]);
      return -1; 
    }

    mygid = pvm_joingroup("mmult");

    if (mygid < 0){
      pvm_perror(argv[0]); 
      pvm_exit(); 
      return -1; 
    }

    if ((mygid == 0) && (ntask > 1)) {
      info = pvm_spawn(VERSION, 
                       &argv[1], 
                       PvmTaskDefault, 
                       (char*)0,
                       ntask-1, 
                       child);

      if(info != ntask-1){
        pvm_lvgroup("mmult"); 
        pvm_exit(); 
        return -1; } 
      //what about the orphans?
    }

    // barrier
    info = pvm_barrier("mmult",ntask);
    if (info < 0) pvm_perror(argv[0]);

    for (i = 0; i < m; i++)
      myrow[i] = pvm_gettid("mmult", 
                            (mygid/m)*m + i);

    for(i = 0; i < 4; i++)
      M[i] = (int*)calloc(sizeof(int), 
                            blksize*blksize);


    if (!(M[0] && M[1] && M[2] && M[3])) {
      fprintf(stderr, 
              "%s: out of memory!\n", 
              argv[0]);
      for(i = 0; i < 4; i++)free(M[i]);
      pvm_lvgroup("mmult"); 
      pvm_exit();
      return -1; }
    
    row = mygid/m; 
    col = mygid % m;

    for(i = 0; i < 3; i++){
      sprintf(P[i], 
              "%s.%d.%d", 
              argv[i+1], 
              row, 
              col);
      F[i] = open(P[i], 
                  ( i < 2 ? 
                    O_RDONLY : 
                    O_WRONLY) ); 
    }
      
    matrix_size = m * blksize;

    for(j = 0; j < 2; j++)
      get_block_row(F[j], 
                    blksize, 
                    0, 
                    0, 
                    0, 
                    blksize*blksize, 
                    M[j]);


    up = pvm_gettid("mmult", 
                    ((row)?
                     (row-1):
                     (m-1))*m+col);

    down = pvm_gettid("mmult", 
                      ((row == (m-1))?
                       col:
                       (row+1)*m+col));


    for (i = 0; i < m; i++) {
      if (col == (row + i)%m) {
        pvm_initsend(ENCODING);
        pvm_pkint(M[0], blksize*blksize, 1);
        pvm_mcast(myrow, m, (i+1)*ATAG);
        block_mult(M[2],M[0],M[1],blksize);  
      } else {
        pvm_recv(pvm_gettid("mmult", 
                            row*m + (row +i)%m), 
                            (i+1)*ATAG);
        pvm_upkint(M[3], blksize*blksize, 1);
        block_mult(M[2],M[3],M[1],blksize); 
      }
      pvm_initsend(ENCODING);
      pvm_pkint(M[1], blksize*blksize, 1);
      pvm_send(up, (i+1)*BTAG);
      pvm_recv(down, (i+1)*BTAG);
      pvm_upkint(M[1], blksize*blksize, 1);  
    }

    info = pvm_barrier("mmult",ntask);
    if (info < 0) pvm_perror(argv[0]);

    for (i = 0 ; i < blksize*blksize; i++)
      if (M[0][i] != M[2][i])
        printf("Error a[%d] (%d) != c[%d] (%d) \n", 
               i, 
               M[0][i],
               i,
               M[2][i]);

    //more verbose code exists in verbose_code

    for(i = 0; i < 2; i++)
      close(F[i]);

    

    set_block_row(F[2], 
                  blksize, 
                  0, 
                  0, 
                  0, 
                  blksize*blksize, 
                  M[2]);

    printf("%s %d %d task %d done successfully after %ld seconds.\n", 
           VERSION, 
           m, 
           blksize, 
           mytid, 
           time(NULL) - start_time);

    for(i = 0; i < 4; i++)
      free(M[i]);

    close(F[2]);

    pvm_lvgroup("mmult");
    pvm_exit();
    return 0; 
      
}


int parse_args(int argc, char *argv[], int *m, int *blksize, int *ntask, int F[]){
  int i;
  if ((argc != 6) || 
      ((F[0] = open(argv[1], O_RDONLY)) == -1) ||
      ((F[1] = open(argv[2], O_RDONLY)) == -1) ||
      ((F[2] = open(argv[3], O_WRONLY)) == -1) ||
      ((*m   = atoi(argv[4])) <= 0)             ||
      ((*blksize   = atoi(argv[5])) <= 0)){

    fprintf(stderr, 
            "Usage: %s matrixA matrixB matrixC m blk\n", argv[0]);
    return(-1); };

  if(*m > MAXROW){
    fprintf(stderr, "m = %d not valid.\n", *m);
    return(-1); };

  *ntask = (*m)*(*m);

  if ((*ntask < 1) || (*ntask >= MAXNTIDS)) {
    fprintf(stderr, "ntask in parse_args = %d not valid.\n", *ntask);
    pvm_exit();
    return(-1); };

  for(i = 0; i < 3; i++)close(F[i]);

  return(0); 
}