ipyrad/analysis/tetrad2.py
def _store_N_samples(self, start, ipyclient, quiet=False):
        """ 
        Find all quartets of samples and store in a large array
        A chunk size is assigned for sampling from the array of quartets
        based on the number of cpus available. This should be relatively 
        large so that we don't spend a lot of time doing I/O, but small 
        enough that jobs finish often for checkpointing.
        """

        breaks = 2
        if self.params.nquartets < 5000:
            breaks = 1
        if self.params.nquartets > 100000:
            breaks = 8
        if self.params.nquartets > 500000:
            breaks = 16
        if self.params.nquartets > 5000000:
            breaks = 32

        ## chunk up the data
        ncpus = len(ipyclient)    
        self._chunksize = (self.params.nquartets // (breaks * ncpus) \
                        + (self.params.nquartets % (breaks * ncpus)))

        ## create h5 OUT empty arrays
        ## 'quartets' stores the inferred quartet relationship (1 x 4)
        ## This can get huge, so we need to choose the dtype wisely. 
        ## the values are simply the index of the taxa, so uint16 is good.
        with h5py.File(self.database.output, 'w') as io5:
            io5.create_dataset("quartets", 
                               (self.params.nquartets, 4), 
                               dtype=np.uint16, 
                               chunks=(self._chunksize, 4))
            ## group for bootstrap invariant matrices ((16, 16), uint32)
            ## these store the actual matrix counts. dtype uint32 can store
            ## up to 4294967295. More than enough. uint16 max is 65535.
            ## the 0 boot is the original seqarray.
            io5.create_group("invariants")

        ## append to h5 IN array (which has the seqarray, bootsarr, maparr)
        ## and fill it with all of the quartet sets we will ever sample.
        ## the samplign method will vary depending on whether this is random, 
        ## all, or equal splits (in a separate but similar function). 
        with h5py.File(self.database.input, 'a') as io5:
            try:
                io5.create_dataset("quartets", 
                               (self.params.nquartets, 4), 
                               dtype=np.uint16, 
                               chunks=(self._chunksize, 4),
                               compression='gzip')
            except RuntimeError:
                raise IPyradWarningExit(
                    "database file already exists for this analysis, "
                  + "you must run with the force flag to overwrite")
            
        ## submit store job to write into self.database.input
        if self.params.method == "all":
            async = ipyclient[0].apply(store_all, self)
        elif self.params.method == "random":
            async = ipyclient[0].apply(store_random, self)
        elif self.params.method == "equal":
            async = ipyclient[0].apply(store_equal, self) 

        ## progress bar 
        printstr = "generating q-sets | {} | "
        prog = 0        
        while 1:
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            if not quiet:
                if async.stdout:
                    prog = int(async.stdout.strip().split()[-1])
                progressbar(self.params.nquartets, prog,
                            printstr.format(elapsed), spacer="")
            if not async.ready():
                time.sleep(0.1)
            else:
                break

        if not async.successful():
            raise IPyradWarningExit(async.result())
        if not quiet:
            print("")
def _store_N_samples(self, start, ipyclient, quiet=False):
        """ 
        Find all quartets of samples and store in a large array
        A chunk size is assigned for sampling from the array of quartets
        based on the number of cpus available. This should be relatively 
        large so that we don't spend a lot of time doing I/O, but small 
        enough that jobs finish often for checkpointing.
        """

        breaks = 2
        if self.params.nquartets < 5000:
            breaks = 1
        if self.params.nquartets > 100000:
            breaks = 8
        if self.params.nquartets > 500000:
            breaks = 16
        if self.params.nquartets > 5000000:
            breaks = 32

        ## chunk up the data
        ncpus = len(ipyclient)    
        self._chunksize = (self.params.nquartets // (breaks * ncpus) \
                        + (self.params.nquartets % (breaks * ncpus)))

        ## create h5 OUT empty arrays
        ## 'quartets' stores the inferred quartet relationship (1 x 4)
        ## This can get huge, so we need to choose the dtype wisely. 
        ## the values are simply the index of the taxa, so uint16 is good.
        with h5py.File(self.database.output, 'w') as io5:
            io5.create_dataset("quartets", 
                               (self.params.nquartets, 4), 
                               dtype=np.uint16, 
                               chunks=(self._chunksize, 4))
            ## group for bootstrap invariant matrices ((16, 16), uint32)
            ## these store the actual matrix counts. dtype uint32 can store
            ## up to 4294967295. More than enough. uint16 max is 65535.
            ## the 0 boot is the original seqarray.
            io5.create_group("invariants")

        ## append to h5 IN array (which has the seqarray, bootsarr, maparr)
        ## and fill it with all of the quartet sets we will ever sample.
        ## the samplign method will vary depending on whether this is random, 
        ## all, or equal splits (in a separate but similar function). 
        with h5py.File(self.database.input, 'a') as io5:
            try:
                io5.create_dataset("quartets", 
                               (self.params.nquartets, 4), 
                               dtype=np.uint16, 
                               chunks=(self._chunksize, 4),
                               compression='gzip')
            except RuntimeError:
                raise IPyradWarningExit(
                    "database file already exists for this analysis, "
                  + "you must run with the force flag to overwrite")
            
        ## submit store job to write into self.database.input
        if self.params.method == "all":
            async = ipyclient[0].apply(store_all, self)
        elif self.params.method == "random":
            async = ipyclient[0].apply(store_random, self)
        elif self.params.method == "equal":
            async = ipyclient[0].apply(store_equal, self) 

        ## progress bar 
        printstr = "generating q-sets | {} | "
        prog = 0        
        while 1:
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            if not quiet:
                if async.stdout:
                    prog = int(async.stdout.strip().split()[-1])
                progressbar(self.params.nquartets, prog,
                            printstr.format(elapsed), spacer="")
            if not async.ready():
                time.sleep(0.1)
            else:
                break

        if not async.successful():
            raise IPyradWarningExit(async.result())
        if not quiet:
            print("")
def _store_N_samples ( self , start , ipyclient , quiet = False ) : breaks = 2 if self . params . nquartets < 5000 : breaks = 1 if self . params . nquartets > 100000 : breaks = 8 if self . params . nquartets > 500000 : breaks = 16 if self . params . nquartets > 5000000 : breaks = 32 ## chunk up the data ncpus = len ( ipyclient ) self . _chunksize = ( self . params . nquartets // ( breaks * ncpus ) + ( self . params . nquartets % ( breaks * ncpus ) ) ) ## create h5 OUT empty arrays ## 'quartets' stores the inferred quartet relationship (1 x 4) ## This can get huge, so we need to choose the dtype wisely.  ## the values are simply the index of the taxa, so uint16 is good. with h5py . File ( self . database . output , 'w' ) as io5 : io5 . create_dataset ( "quartets" , ( self . params . nquartets , 4 ) , dtype = np . uint16 , chunks = ( self . _chunksize , 4 ) ) ## group for bootstrap invariant matrices ((16, 16), uint32) ## these store the actual matrix counts. dtype uint32 can store ## up to 4294967295. More than enough. uint16 max is 65535. ## the 0 boot is the original seqarray. io5 . create_group ( "invariants" ) ## append to h5 IN array (which has the seqarray, bootsarr, maparr) ## and fill it with all of the quartet sets we will ever sample. ## the samplign method will vary depending on whether this is random,  ## all, or equal splits (in a separate but similar function).  with h5py . File ( self . database . input , 'a' ) as io5 : try : io5 . create_dataset ( "quartets" , ( self . params . nquartets , 4 ) , dtype = np . uint16 , chunks = ( self . _chunksize , 4 ) , compression = 'gzip' ) except RuntimeError : raise IPyradWarningExit ( "database file already exists for this analysis, " + "you must run with the force flag to overwrite" ) ## submit store job to write into self.database.input if self . params . method == "all" : async = ipyclient [ 0 ] . apply ( store_all , self ) elif self . params . method == "random" : async = ipyclient [ 0 ] . apply ( store_random , self ) elif self . params . method == "equal" : async = ipyclient [ 0 ] . apply ( store_equal , self ) ## progress bar  printstr = "generating q-sets | {} | " prog = 0 while 1 : elapsed = datetime . timedelta ( seconds = int ( time . time ( ) - start ) ) if not quiet : if async . stdout : prog = int ( async . stdout . strip ( ) . split ( ) [ - 1 ] ) progressbar ( self . params . nquartets , prog , printstr . format ( elapsed ) , spacer = "" ) if not async . ready ( ) : time . sleep ( 0.1 ) else : break if not async . successful ( ) : raise IPyradWarningExit ( async . result ( ) ) if not quiet : print ( "" )
Find all quartets of samples and store in a large array
        A chunk size is assigned for sampling from the array of quartets
        based on the number of cpus available. This should be relatively 
        large so that we don't spend a lot of time doing I/O, but small 
        enough that jobs finish often for checkpointing.
Find all quartets of samples and store in a large array A chunk size is assigned for sampling from the array of quartets based on the number of cpus available . This should be relatively large so that we don t spend a lot of time doing I / O but small enough that jobs finish often for checkpointing .