ipyrad/assemble/write_outfiles.py
def make_vcf(data, samples, ipyclient, full=0):
    """
    Write the full VCF for loci passing filtering. Other vcf formats are
    possible, like SNPs-only, or with filtered loci included but the filter
    explicitly labeled. These are not yet supported, however.
    """
    ## start vcf progress bar
    start = time.time()
    printstr = " building vcf file     | {} | s7 |"
    LOGGER.info("Writing .vcf file")
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(20, 0, printstr.format(elapsed), spacer=data._spacer)

    ## create outputs for v and V, gzip V to be friendly
    data.outfiles.vcf = os.path.join(data.dirs.outfiles, data.name+".vcf")
    if full:
        data.outfiles.VCF = os.path.join(data.dirs.outfiles, data.name+".vcf.gz")

    ## get some db info
    with h5py.File(data.clust_database, 'r') as io5:
        ## will iterate optim loci at a time
        optim = io5["seqs"].attrs["chunksize"][0]
        nloci = io5["seqs"].shape[0]
        ## get name and snp padding
        anames = io5["seqs"].attrs["samples"]
        snames = [i.name for i in samples]
        names = [i for i in anames if i in snames]

    ## get names index
    sidx = np.array([i in snames for i in anames])

    ## client for sending jobs to parallel engines, for this step we'll limit
    ## to half of the available cpus if
    lbview = ipyclient.load_balanced_view()

    ## send jobs in chunks
    vasyncs = {}
    total = 0
    for chunk in xrange(0, nloci, optim):
        vasyncs[chunk] = lbview.apply(vcfchunk, *(data, optim, sidx, chunk, full))
        total += 1

    ## tmp files get left behind and intensive processes are left running when a
    ## a job is killed/interrupted during vcf build, so we try/except wrap.
    try:
        while 1:
            keys = [i for (i, j) in vasyncs.items() if j.ready()]
            ## check for failures
            for job in keys:
                if not vasyncs[job].successful():
                    ## raise exception
                    err = " error in vcf build chunk {}: {}"\
                          .format(job, vasyncs[job].result())
                    LOGGER.error(err)
                    raise IPyradWarningExit(err)
                else:
                    ## free up memory
                    del vasyncs[job]

            finished = total - len(vasyncs) #sum([i.ready() for i in vasyncs.values()])
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            progressbar(total, finished, printstr.format(elapsed), spacer=data._spacer)
            time.sleep(0.5)
            if not vasyncs:
                break
        print("")

    except Exception as inst:
        ## make sure all future jobs are aborted
        keys = [i for (i, j) in vasyncs.items() if not j.ready()]
        try:
            for job in keys:
                #vasyncs[job].abort()
                vasyncs[job].cancel()
        except Exception:
            pass
        ## make sure all tmp files are destroyed
        vcfchunks = glob.glob(os.path.join(data.dirs.outfiles, "*.vcf.[0-9]*"))
        h5chunks = glob.glob(os.path.join(data.dirs.outfiles, ".tmp.[0-9]*.h5"))
        for dfile in vcfchunks+h5chunks:
            os.remove(dfile)
        ## reraise the error
        raise inst


    ## writing full vcf file to disk
    start = time.time()
    printstr = " writing vcf file      | {} | s7 |"
    res = lbview.apply(concat_vcf, *(data, names, full))
    ogchunks = len(glob.glob(data.outfiles.vcf+".*"))
    while 1:
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        curchunks = len(glob.glob(data.outfiles.vcf+".*"))
        progressbar(ogchunks, ogchunks-curchunks, printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if res.ready():
            break
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(1, 1, printstr.format(elapsed), spacer=data._spacer)
    print("")
def make_vcf(data, samples, ipyclient, full=0):
    """
    Write the full VCF for loci passing filtering. Other vcf formats are
    possible, like SNPs-only, or with filtered loci included but the filter
    explicitly labeled. These are not yet supported, however.
    """
    ## start vcf progress bar
    start = time.time()
    printstr = " building vcf file     | {} | s7 |"
    LOGGER.info("Writing .vcf file")
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(20, 0, printstr.format(elapsed), spacer=data._spacer)

    ## create outputs for v and V, gzip V to be friendly
    data.outfiles.vcf = os.path.join(data.dirs.outfiles, data.name+".vcf")
    if full:
        data.outfiles.VCF = os.path.join(data.dirs.outfiles, data.name+".vcf.gz")

    ## get some db info
    with h5py.File(data.clust_database, 'r') as io5:
        ## will iterate optim loci at a time
        optim = io5["seqs"].attrs["chunksize"][0]
        nloci = io5["seqs"].shape[0]
        ## get name and snp padding
        anames = io5["seqs"].attrs["samples"]
        snames = [i.name for i in samples]
        names = [i for i in anames if i in snames]

    ## get names index
    sidx = np.array([i in snames for i in anames])

    ## client for sending jobs to parallel engines, for this step we'll limit
    ## to half of the available cpus if
    lbview = ipyclient.load_balanced_view()

    ## send jobs in chunks
    vasyncs = {}
    total = 0
    for chunk in xrange(0, nloci, optim):
        vasyncs[chunk] = lbview.apply(vcfchunk, *(data, optim, sidx, chunk, full))
        total += 1

    ## tmp files get left behind and intensive processes are left running when a
    ## a job is killed/interrupted during vcf build, so we try/except wrap.
    try:
        while 1:
            keys = [i for (i, j) in vasyncs.items() if j.ready()]
            ## check for failures
            for job in keys:
                if not vasyncs[job].successful():
                    ## raise exception
                    err = " error in vcf build chunk {}: {}"\
                          .format(job, vasyncs[job].result())
                    LOGGER.error(err)
                    raise IPyradWarningExit(err)
                else:
                    ## free up memory
                    del vasyncs[job]

            finished = total - len(vasyncs) #sum([i.ready() for i in vasyncs.values()])
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            progressbar(total, finished, printstr.format(elapsed), spacer=data._spacer)
            time.sleep(0.5)
            if not vasyncs:
                break
        print("")

    except Exception as inst:
        ## make sure all future jobs are aborted
        keys = [i for (i, j) in vasyncs.items() if not j.ready()]
        try:
            for job in keys:
                #vasyncs[job].abort()
                vasyncs[job].cancel()
        except Exception:
            pass
        ## make sure all tmp files are destroyed
        vcfchunks = glob.glob(os.path.join(data.dirs.outfiles, "*.vcf.[0-9]*"))
        h5chunks = glob.glob(os.path.join(data.dirs.outfiles, ".tmp.[0-9]*.h5"))
        for dfile in vcfchunks+h5chunks:
            os.remove(dfile)
        ## reraise the error
        raise inst


    ## writing full vcf file to disk
    start = time.time()
    printstr = " writing vcf file      | {} | s7 |"
    res = lbview.apply(concat_vcf, *(data, names, full))
    ogchunks = len(glob.glob(data.outfiles.vcf+".*"))
    while 1:
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        curchunks = len(glob.glob(data.outfiles.vcf+".*"))
        progressbar(ogchunks, ogchunks-curchunks, printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if res.ready():
            break
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(1, 1, printstr.format(elapsed), spacer=data._spacer)
    print("")
def make_vcf ( data , samples , ipyclient , full = 0 ) : ## start vcf progress bar start = time . time ( ) printstr = " building vcf file     | {} | s7 |" LOGGER . info ( "Writing .vcf file" ) elapsed = datetime . timedelta ( seconds = int ( time . time ( ) - start ) ) progressbar ( 20 , 0 , printstr . format ( elapsed ) , spacer = data . _spacer ) ## create outputs for v and V, gzip V to be friendly data . outfiles . vcf = os . path . join ( data . dirs . outfiles , data . name + ".vcf" ) if full : data . outfiles . VCF = os . path . join ( data . dirs . outfiles , data . name + ".vcf.gz" ) ## get some db info with h5py . File ( data . clust_database , 'r' ) as io5 : ## will iterate optim loci at a time optim = io5 [ "seqs" ] . attrs [ "chunksize" ] [ 0 ] nloci = io5 [ "seqs" ] . shape [ 0 ] ## get name and snp padding anames = io5 [ "seqs" ] . attrs [ "samples" ] snames = [ i . name for i in samples ] names = [ i for i in anames if i in snames ] ## get names index sidx = np . array ( [ i in snames for i in anames ] ) ## client for sending jobs to parallel engines, for this step we'll limit ## to half of the available cpus if lbview = ipyclient . load_balanced_view ( ) ## send jobs in chunks vasyncs = { } total = 0 for chunk in xrange ( 0 , nloci , optim ) : vasyncs [ chunk ] = lbview . apply ( vcfchunk , * ( data , optim , sidx , chunk , full ) ) total += 1 ## tmp files get left behind and intensive processes are left running when a ## a job is killed/interrupted during vcf build, so we try/except wrap. try : while 1 : keys = [ i for ( i , j ) in vasyncs . items ( ) if j . ready ( ) ] ## check for failures for job in keys : if not vasyncs [ job ] . successful ( ) : ## raise exception err = " error in vcf build chunk {}: {}" . format ( job , vasyncs [ job ] . result ( ) ) LOGGER . error ( err ) raise IPyradWarningExit ( err ) else : ## free up memory del vasyncs [ job ] finished = total - len ( vasyncs ) #sum([i.ready() for i in vasyncs.values()]) elapsed = datetime . timedelta ( seconds = int ( time . time ( ) - start ) ) progressbar ( total , finished , printstr . format ( elapsed ) , spacer = data . _spacer ) time . sleep ( 0.5 ) if not vasyncs : break print ( "" ) except Exception as inst : ## make sure all future jobs are aborted keys = [ i for ( i , j ) in vasyncs . items ( ) if not j . ready ( ) ] try : for job in keys : #vasyncs[job].abort() vasyncs [ job ] . cancel ( ) except Exception : pass ## make sure all tmp files are destroyed vcfchunks = glob . glob ( os . path . join ( data . dirs . outfiles , "*.vcf.[0-9]*" ) ) h5chunks = glob . glob ( os . path . join ( data . dirs . outfiles , ".tmp.[0-9]*.h5" ) ) for dfile in vcfchunks + h5chunks : os . remove ( dfile ) ## reraise the error raise inst ## writing full vcf file to disk start = time . time ( ) printstr = " writing vcf file      | {} | s7 |" res = lbview . apply ( concat_vcf , * ( data , names , full ) ) ogchunks = len ( glob . glob ( data . outfiles . vcf + ".*" ) ) while 1 : elapsed = datetime . timedelta ( seconds = int ( time . time ( ) - start ) ) curchunks = len ( glob . glob ( data . outfiles . vcf + ".*" ) ) progressbar ( ogchunks , ogchunks - curchunks , printstr . format ( elapsed ) , spacer = data . _spacer ) time . sleep ( 0.1 ) if res . ready ( ) : break elapsed = datetime . timedelta ( seconds = int ( time . time ( ) - start ) ) progressbar ( 1 , 1 , printstr . format ( elapsed ) , spacer = data . _spacer ) print ( "" )
Write the full VCF for loci passing filtering. Other vcf formats are
    possible, like SNPs-only, or with filtered loci included but the filter
    explicitly labeled. These are not yet supported, however.
Write the full VCF for loci passing filtering . Other vcf formats are possible like SNPs - only or with filtered loci included but the filter explicitly labeled . These are not yet supported however .