ipyrad/assemble/util.py
def get_threaded_view(ipyclient, split=True):
    """ gets optimum threaded view of ids given the host setup """
    ## engine ids
    ## e.g., [0, 1, 2, 3, 4, 5, 6, 7, 8]
    eids = ipyclient.ids

    ## get host names
    ## e.g., ['a', 'a', 'b', 'b', 'a', 'c', 'c', 'c', 'c']
    dview = ipyclient.direct_view()
    hosts = dview.apply_sync(socket.gethostname)

    ## group ids into a dict by their hostnames
    ## e.g., {a: [0, 1, 4], b: [2, 3], c: [5, 6, 7, 8]}
    hostdict = defaultdict(list)
    for host, eid in zip(hosts, eids):
        hostdict[host].append(eid)

    ## Now split threads on the same host into separate proc if there are many
    hostdictkeys = hostdict.keys()
    for key in hostdictkeys:
        gids = hostdict[key]
        maxt = len(gids)
        if len(gids) >= 4:
            maxt = 2
        ## if 4 nodes and 4 ppn, put one sample per host
        if (len(gids) == 4) and (len(hosts) >= 4):
            maxt = 4
        if len(gids) >= 6:
            maxt = 3
        if len(gids) >= 8:
            maxt = 4
        if len(gids) >= 16:
            maxt = 4
        ## split ids into groups of maxt
        threaded = [gids[i:i+maxt] for i in xrange(0, len(gids), maxt)]
        lth = len(threaded)
        ## if anything was split (lth>1) update hostdict with new proc
        if lth > 1:
            hostdict.pop(key)
            for hostid in range(lth):
                hostdict[str(key)+"_"+str(hostid)] = threaded[hostid]

    ## make sure split numbering is correct
    #threaded = hostdict.values()
    #assert len(ipyclient.ids) <= len(list(itertools.chain(*threaded)))
    LOGGER.info("threaded_view: %s", dict(hostdict))
    return hostdict
def get_threaded_view(ipyclient, split=True):
    """ gets optimum threaded view of ids given the host setup """
    ## engine ids
    ## e.g., [0, 1, 2, 3, 4, 5, 6, 7, 8]
    eids = ipyclient.ids

    ## get host names
    ## e.g., ['a', 'a', 'b', 'b', 'a', 'c', 'c', 'c', 'c']
    dview = ipyclient.direct_view()
    hosts = dview.apply_sync(socket.gethostname)

    ## group ids into a dict by their hostnames
    ## e.g., {a: [0, 1, 4], b: [2, 3], c: [5, 6, 7, 8]}
    hostdict = defaultdict(list)
    for host, eid in zip(hosts, eids):
        hostdict[host].append(eid)

    ## Now split threads on the same host into separate proc if there are many
    hostdictkeys = hostdict.keys()
    for key in hostdictkeys:
        gids = hostdict[key]
        maxt = len(gids)
        if len(gids) >= 4:
            maxt = 2
        ## if 4 nodes and 4 ppn, put one sample per host
        if (len(gids) == 4) and (len(hosts) >= 4):
            maxt = 4
        if len(gids) >= 6:
            maxt = 3
        if len(gids) >= 8:
            maxt = 4
        if len(gids) >= 16:
            maxt = 4
        ## split ids into groups of maxt
        threaded = [gids[i:i+maxt] for i in xrange(0, len(gids), maxt)]
        lth = len(threaded)
        ## if anything was split (lth>1) update hostdict with new proc
        if lth > 1:
            hostdict.pop(key)
            for hostid in range(lth):
                hostdict[str(key)+"_"+str(hostid)] = threaded[hostid]

    ## make sure split numbering is correct
    #threaded = hostdict.values()
    #assert len(ipyclient.ids) <= len(list(itertools.chain(*threaded)))
    LOGGER.info("threaded_view: %s", dict(hostdict))
    return hostdict
def get_threaded_view ( ipyclient , split = True ) : ## engine ids ## e.g., [0, 1, 2, 3, 4, 5, 6, 7, 8] eids = ipyclient . ids ## get host names ## e.g., ['a', 'a', 'b', 'b', 'a', 'c', 'c', 'c', 'c'] dview = ipyclient . direct_view ( ) hosts = dview . apply_sync ( socket . gethostname ) ## group ids into a dict by their hostnames ## e.g., {a: [0, 1, 4], b: [2, 3], c: [5, 6, 7, 8]} hostdict = defaultdict ( list ) for host , eid in zip ( hosts , eids ) : hostdict [ host ] . append ( eid ) ## Now split threads on the same host into separate proc if there are many hostdictkeys = hostdict . keys ( ) for key in hostdictkeys : gids = hostdict [ key ] maxt = len ( gids ) if len ( gids ) >= 4 : maxt = 2 ## if 4 nodes and 4 ppn, put one sample per host if ( len ( gids ) == 4 ) and ( len ( hosts ) >= 4 ) : maxt = 4 if len ( gids ) >= 6 : maxt = 3 if len ( gids ) >= 8 : maxt = 4 if len ( gids ) >= 16 : maxt = 4 ## split ids into groups of maxt threaded = [ gids [ i : i + maxt ] for i in xrange ( 0 , len ( gids ) , maxt ) ] lth = len ( threaded ) ## if anything was split (lth>1) update hostdict with new proc if lth > 1 : hostdict . pop ( key ) for hostid in range ( lth ) : hostdict [ str ( key ) + "_" + str ( hostid ) ] = threaded [ hostid ] ## make sure split numbering is correct #threaded = hostdict.values() #assert len(ipyclient.ids) <= len(list(itertools.chain(*threaded))) LOGGER . info ( "threaded_view: %s" , dict ( hostdict ) ) return hostdict
gets optimum threaded view of ids given the host setup
gets optimum threaded view of ids given the host setup