/****************************************************************************** 
 * file:    clusterinfo.c
 * purpose: Determine cluster info i.e., number of machines and processes
 *          running on each of them.
 *
 *******************************************************************************/
#if HAVE_CONFIG_H
#   include "config.h"
#endif

#include <assert.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>

#include "clusterinfo.h"
#include "message.h"

/* NO_SHMEM enables to simulate cluster environment on a single workstation.
 * Must define NO_SHMMAX_SEARCH in shmem.c to prevent depleting shared memory
 * due to a gready shmem request by the master process on cluster node 0. */ 
#define armci_enable_alpha_hack() 1

#define ARMCI_TAG 30000
#define DEBUG 0
#define MAX_HOSTNAME 80
#define CHECK_NODE_NAMES 
#define CLUSTER 1 /* until I remove all non CLUSTER code */

/* print info on how many cluster nodes detected */
#define PRINT_CLUSTER_INFO 1

static const char *network_protocol="DMAPP";

/* stores cluster configuration.
 * Initialized before user threads are created and then read-only */
armci_clus_t *armci_clus_info;
int armci_nclus;
int armci_clus_me;
int armci_master;
int armci_clus_first;
int armci_clus_last;

#ifdef HITACHI
#include <hmpp/nalloc.h>
# define GETHOSTNAME sr_gethostname
ndes_t _armci_group;

static int sr_gethostname(char *name, int len)
{
    int no;
    pid_t ppid;

    if(hmpp_nself (&_armci_group,&no,&ppid,0,NULL) <0)
        return -1;

    if(len<6)armci_die("len too small",len);
    if(no>1024)armci_die("expected node id <1024",no);
    sprintf(name,"n%d",no);
    return 0;
}
#elif defined(SGIALTIX)
# define GETHOSTNAME altix_gethostname
static int altix_gethostname(char *name, int len) {
    sprintf(name,"altix");
    return 0;
}
#elif defined(CRAY_XT) /* && !defined(PORTALS) */
#define GETHOSTNAME cnos_gethostname
static int cnos_gethostname(char *name, int len)
{
    int size,rank;
    size=PMI_Get_rank(&rank);
    sprintf(name,"%d",rank);
}
#else
# define GETHOSTNAME gethostname
#endif

static char* merge_names(char *name)
{
    int jump = 1, rem, to, from;
    int lenmes, lenbuf, curlen, totbuflen= armci_nproc*HOSTNAME_LEN;
    int len = strlen(name);
    char *work = malloc(totbuflen);

    if(!work)armci_die("armci: merge_names: malloc failed: ",totbuflen);

    strcpy(work, name); 
    curlen = len+1;

    /* prefix tree merges names in the order of process numbering in log(P)time
     * result = name_1//name_2//...//name_P-1
     */
    do {
        jump *= 2; rem = armci_me%jump;
        if(rem){
            to = armci_me - rem;
            armci_msg_snd(ARMCI_TAG, work, curlen, to);
            break;
        }else{
            from = armci_me + jump/2;
            if(from < armci_nproc){
                lenbuf = totbuflen - curlen;
                armci_msg_rcv(ARMCI_TAG, work+curlen, lenbuf, &lenmes, from);
                curlen += lenmes;
            }
        }
    }while (jump < armci_nproc);
    return(work);
}


static void process_hostlist(char *names)
{
#ifdef CLUSTER

    int i, cluster=0;
    char *s,*master;
    int len, root=0;

    /******** inspect list of machine names to determine locality ********/
    if (armci_me==0){

        /* first find out how many cluster nodes we got */
        armci_nclus =1; s=master=names; 
        for(i=1; i < armci_nproc; i++){
            s += strlen(s)+1;
            if(strcmp(s,master)){
                /* we found a new machine name on the list */
                master = s;
                armci_nclus++;
                /*fprintf(stderr,"new name %s len =%d\n",master, strlen(master));*/

            }
        }

        /* allocate memory */ 
        armci_clus_info = (armci_clus_t*)malloc(armci_nclus*sizeof(armci_clus_t));
        if(!armci_clus_info)
            armci_die("malloc failed for clusinfo",armci_nclus);

        /* fill the data structure  -- go through the list again */ 
        s=names;
        master="*-"; /* impossible hostname */
        cluster =0;
        for(i=0; i < armci_nproc; i++){
            if(strcmp(s,master)){
                /* we found a new machine name on the list */
                master = s;
                armci_clus_info[cluster].nslave=1;
                armci_clus_info[cluster].master=i;
                strcpy(armci_clus_info[cluster].hostname, master); 

#ifdef    CHECK_NODE_NAMES
                /* need consecutive task id allocated on the same node
                 * the current test only compares hostnames against first cluster */
                if(cluster) 
                    if(!strcmp(master,armci_clus_info[0].hostname)){
                        /* we have seen that hostname before */
                        fprintf(stderr, "ARMCI supports block process mapping only\n");
                        armci_die("Cannot run: improper task to host mapping!",0); 
                    }
#endif
                cluster++;

            }
            else{
                /* the process is still on the same host */
                armci_clus_info[cluster-1].nslave++;
            }
            s += strlen(s)+1;
        }

        if(armci_nclus != cluster)
            armci_die("inconsistency processing clusterinfo",armci_nclus);

    }
    /******** process 0 got all data                             ********/

    /* now broadcast locality info struct to all processes 
     * two steps are needed because of the unknown length of hostname list
     */
    len = sizeof(int);
    armci_msg_brdcst(&armci_nclus, len, root);

    if(armci_me){
        /* allocate memory */ 
        armci_clus_info = (armci_clus_t*)malloc(armci_nclus*sizeof(armci_clus_t));
        if(!armci_clus_info)
            armci_die("malloc failed for clusinfo",armci_nclus);
    }

    len = sizeof(armci_clus_t)*armci_nclus;
    armci_msg_brdcst(armci_clus_info, len, root);

    /******** all processes 0 got all data                         ********/

    /* now determine current cluster node id by comparing me to master */
    armci_clus_me = armci_nclus-1;
    for(i =0; i< armci_nclus-1; i++)
        if(armci_me < armci_clus_info[i+1].master){
            armci_clus_me=i;
            break;
        }
#else

    armci_clus_me=0;
    armci_nclus=1;
    armci_clus_info = (armci_clus_t*)malloc(armci_nclus*sizeof(armci_clus_t));
    if(!armci_clus_info)
        armci_die("malloc failed for clusinfo",armci_nclus);
    strcpy(armci_clus_info[0].hostname, names); 
    armci_clus_info[0].master=0;
    armci_clus_info[0].nslave=armci_nproc;
#endif

    /* Starting process ID on my node */
    armci_clus_first = armci_clus_info[armci_clus_me].master;

    /* Last process ID on my node */
    armci_clus_last = armci_clus_first +
        armci_clus_info[armci_clus_me].nslave-1;

}


/*\ Substring Replacement: replace needle with nail in a haystack
  \*/
static char *substr_replace(char *haystack, char *needle, char *nail)
{
    char *tmp, *pos, *first;
    size_t len=strlen(needle), nlen=strlen(nail),bytes;
    size_t left;

    pos = strstr(haystack,needle);
    if (pos ==NULL) return NULL;
    first= tmp = calloc(strlen(haystack)+nlen-len+1+1,1);
    if(first==NULL) return(NULL);
    bytes = pos - haystack;
    while(bytes){ *tmp = *haystack; tmp++; haystack++; bytes--;}
    while(nlen) { *tmp = *nail; tmp++; nail++; nlen--;}
    haystack += len;
    left = strlen(haystack);
    while(left>0){*tmp = *haystack; tmp++; haystack++; left --;}
    *tmp='\0';
    return(first);
}


/*\ ARMCI_HOSTNAME_REPLACE contains "needle/nail" string to derive new hostname
  \*/
static char *new_hostname(char *host)
{
    char *tmp, *needle, *nail;
    if((tmp =getenv("ARMCI_HOSTNAME_REPLACE"))){
        needle = strdup(tmp);
        if(needle== NULL) return NULL;
        nail = strchr(needle,'/');
        if(nail == NULL) return NULL;
        *nail = '\0';
        nail++;
        if(nail == (needle+1)){
            char* tmp1 = calloc(strlen(host)+strlen(nail)+1,1);
            if(tmp1 == NULL) return NULL;
            strcpy(tmp1,host);
            strcat(tmp1,nail);
            return tmp1;
        }
        return substr_replace(host,needle,nail);
    } else return NULL;
}


static void print_clus_info()
{
    int i;

    if(PRINT_CLUSTER_INFO && armci_nclus > 1 && armci_me ==0){
        printf("ARMCI configured for %d cluster nodes. Network protocol is '%s'.\n",
                armci_nclus, network_protocol);
        fflush(stdout);
    }

    if(armci_me==0 && DEBUG) for(i=0;i<armci_nclus;i++)
        printf("%s cluster:%d nodes:%d master=%d\n",armci_clus_info[i].hostname,i,
                armci_clus_info[i].nslave,armci_clus_info[i].master);
}

void armci_init_clusinfo()
{
    char name[MAX_HOSTNAME], *merged;
    int  len, limit, rc;
    char *tmp;
#if ARMCI_ENABLE_GPC_CALLS
    char *enval;
#endif

    if((tmp =getenv("ARMCI_HOSTNAME")) == NULL){
        limit = MAX_HOSTNAME-1;
        rc = GETHOSTNAME(name, limit);
        if(rc < 0)armci_die("armci: gethostname failed",rc);
        tmp = new_hostname(name);
    }
    if(tmp != NULL){
        if(strlen(tmp) >= MAX_HOSTNAME)
            armci_die("armci: hostname too long",strlen(tmp));
        strcpy(name,tmp);
        printf("%d using %s hostname\n",armci_me, name);
        fflush(stdout);
    }
    len =  strlen(name);
#if ARMCI_ENABLE_GPC_CALLS
    /*a simple way to run as many servers as compute processes*/ 
    enval = getenv("ARMCI_NSERV_EQ_NPROC");
    if(enval != NULL){
        sprintf(name+len,"n%d",getpid());
        len =  strlen(name);
        printf("\n%s\n",name);
    }
#endif


#ifdef HOSTNAME_TRUNCATE
    {    
        /* in some cases (e.g.,SP) when name is used to determine
         * cluster structure but not to establish communication
         * we can truncate hostnames to save memory */
        int i;
        limit = HOSTNAME_LEN-1;
        for(i=0; i<len; i++){
            if(name[i] =='.')break; /*we are not truncating 1st part of hostname*/
            if(i==limit)armci_die("Please increase HOSTNAME_LEN in ARMCI >",i+1);
        }
        if(len>limit)name[limit]='\0';
        len =limit;
    }
#else
    if(len >= HOSTNAME_LEN-1)
        armci_die("armci: gethostname overrun name string length",len);
#endif

#ifdef NO_SHMEM
    if(armci_enable_alpha_hack()) {
        name[len]='0'+armci_me;
        name[len+1]='\0';
        len++;
    }
#endif

    if(DEBUG)
        fprintf(stderr,"%d: %s len=%d\n",armci_me, name,(int)strlen(name));

#ifdef CLUSTER
    merged = merge_names(name); /* create hostname list */
    process_hostlist(merged);        /* compute cluster info */
    free(merged);
#else
    process_hostlist(name);        /* compute cluster info */
#endif

    armci_master = armci_clus_info[armci_clus_me].master;

#ifdef NO_SHMEM
    if(armci_enable_alpha_hack()) {
        int i;
        for(i=0;i<armci_nclus;i++){
            int len=strlen(armci_clus_info[i].hostname);
            /*     fprintf(stderr,"----hostlen=%d\n",len);*/
            armci_clus_info[i].hostname[len-1]='\0';
        }
    }
#endif

    print_clus_info();

}


/*\ find cluster node id the specified process is on
  \*/
int armci_clus_id(int p)
{
    int from, to, found, c;

    if (p <0 || p >= armci_nproc)
        armci_die("armci_clus_id: out of range",p);

    if (p < armci_clus_first){ 
        from = 0; 
        to = armci_clus_me;
    }
    else {
        from = armci_clus_me; 
        to = armci_nclus;
    }

    found = to - 1;

    /* Binary search algorithm to be implemented,
     * sequential search for now */
    for(c = from; c < to - 1; c++)
        if (p < armci_clus_info[c+1].master) {
            found = c;
            break;
        }

    return found;
}


/*\ return number of processes in the domain represented by id; id<0 means my node
  \*/
int armci_domain_nprocs(armci_domain_t domain, int id)
{
    if(id >= armci_nclus) 
        armci_die2("armci domain error",id,armci_nclus);
    /* This is an error condition */
    if(id < 0) {
        fprintf(stderr,"[%d] Returned domain is invalid\n", armci_me);
        id = armci_clus_me;
    }
    return armci_clus_info[id].nslave;
}

/* return number of nodes in diven domain */
int armci_domain_count(armci_domain_t domain)
{
    return armci_nclus;
}

/* return domain ID of the specified process */
int armci_domain_id(armci_domain_t domain, int glob_proc_id)
{
    int id = glob_proc_id;

    if(id <0 || id >= armci_nproc)  {
        armci_die2("armci domain error",id,armci_nproc);
    }

    return armci_clus_id(glob_proc_id);
}

/* return global ID of a process loc_proc_id in domain identified by id
 *  armci_domain_nproc(id)< loc_proc_id >=0
 */

int armci_domain_glob_proc_id(armci_domain_t domain, int id, int loc_proc_id)
{
    if(id <0 || id >= armci_nclus) 
        armci_die2("armci domain error",id,armci_nclus);

    if(loc_proc_id<0 || loc_proc_id>= armci_clus_info[id].nslave)
        armci_die2("armci domain proc error",
                loc_proc_id,armci_clus_info[id].nslave);

    return (armci_clus_info[id].master + loc_proc_id);
}

/* return ID of domain that the calling process belongs to
*/
int armci_domain_my_id(armci_domain_t domain)
{
    return armci_clus_me;
}

/* Check whether the oricess is in the same domain */
int armci_domain_same_id (armci_domain_t domain, int proc)
{
#if 0
    int rc = SAMECLUSNODE(proc);
    return rc;
#else
    assert(0);
#endif
}
