astrobase/varclass/rfclass.py
def collect_nonperiodic_features(
        featuresdir,
        magcol,
        outfile,
        pklglob='varfeatures-*.pkl',
        featurestouse=NONPERIODIC_FEATURES_TO_COLLECT,
        maxobjects=None,
        labeldict=None,
        labeltype='binary',
):
    '''This collects variability features into arrays for use with the classifer.

    Parameters
    ----------

    featuresdir : str
        This is the directory where all the varfeatures pickles are. Use
        `pklglob` to specify the glob to search for. The `varfeatures` pickles
        contain objectids, a light curve magcol, and features as dict
        key-vals. The :py:mod:`astrobase.lcproc.lcvfeatures` module can be used
        to produce these.

    magcol : str
        This is the key in each varfeatures pickle corresponding to the magcol
        of the light curve the variability features were extracted from.

    outfile : str
        This is the filename of the output pickle that will be written
        containing a dict of all the features extracted into np.arrays.

    pklglob : str
        This is the UNIX file glob to use to search for varfeatures pickle files
        in `featuresdir`.

    featurestouse : list of str
        Each varfeatures pickle can contain any combination of non-periodic,
        stellar, and periodic features; these must have the same names as
        elements in the list of strings provided in `featurestouse`.  This tries
        to get all the features listed in NONPERIODIC_FEATURES_TO_COLLECT by
        default. If `featurestouse` is provided as a list, gets only the
        features listed in this kwarg instead.

    maxobjects : int or None
        The controls how many pickles from the featuresdir to process. If None,
        will process all varfeatures pickles.

    labeldict : dict or None
        If this is provided, it must be a dict with the following key:val list::

            '<objectid>':<label value>

        for each objectid collected from the varfeatures pickles. This will turn
        the collected information into a training set for classifiers.

        Example: to carry out non-periodic variable feature collection of fake
        LCS prepared by :py:mod:`astrobase.fakelcs.generation`, use the value
        of the 'isvariable' dict elem from the `fakelcs-info.pkl` here, like
        so::

            labeldict={x:y for x,y in zip(fakelcinfo['objectid'],
                                          fakelcinfo['isvariable'])}

    labeltype : {'binary', 'classes'}
        This is either 'binary' or 'classes' for binary/multi-class
        classification respectively.

    Returns
    -------

    dict
        This returns a dict with all of the features collected into np.arrays,
        ready to use as input to a scikit-learn classifier.

    '''

    # list of input pickles generated by varfeatures in lcproc.py
    pklist = glob.glob(os.path.join(featuresdir, pklglob))

    if maxobjects:
        pklist = pklist[:maxobjects]


    # fancy progress bar with tqdm if present
    if TQDM:
        listiterator = tqdm(pklist)
    else:
        listiterator = pklist

    # go through all the varfeatures arrays

    feature_dict = {'objectids':[],'magcol':magcol, 'availablefeatures':[]}

    LOGINFO('collecting features for magcol: %s' % magcol)

    for pkl in listiterator:

        with open(pkl,'rb') as infd:
            varf = pickle.load(infd)

        # update the objectid list
        objectid = varf['objectid']
        if objectid not in feature_dict['objectids']:
            feature_dict['objectids'].append(objectid)

        thisfeatures = varf[magcol]

        if featurestouse and len(featurestouse) > 0:
            featurestoget = featurestouse
        else:
            featurestoget = NONPERIODIC_FEATURES_TO_COLLECT

        # collect all the features for this magcol/objectid combination
        for feature in featurestoget:

            # update the global feature list if necessary
            if ((feature not in feature_dict['availablefeatures']) and
                (feature in thisfeatures)):

                feature_dict['availablefeatures'].append(feature)
                feature_dict[feature] = []

            if feature in thisfeatures:

                feature_dict[feature].append(
                    thisfeatures[feature]
                )

    # now that we've collected all the objects and their features, turn the list
    # into arrays, and then concatenate them
    for feat in feature_dict['availablefeatures']:
        feature_dict[feat] = np.array(feature_dict[feat])

    feature_dict['objectids'] = np.array(feature_dict['objectids'])

    feature_array = np.column_stack([feature_dict[feat] for feat in
                                     feature_dict['availablefeatures']])
    feature_dict['features_array'] = feature_array


    # if there's a labeldict available, use it to generate a label array. this
    # feature collection is now a training set.
    if isinstance(labeldict, dict):

        labelarray = np.zeros(feature_dict['objectids'].size, dtype=np.int64)

        # populate the labels for each object in the training set
        for ind, objectid in enumerate(feature_dict['objectids']):

            if objectid in labeldict:

                # if this is a binary classifier training set, convert bools to
                # ones and zeros
                if labeltype == 'binary':

                    if labeldict[objectid]:
                        labelarray[ind] = 1

                # otherwise, use the actual class label integer
                elif labeltype == 'classes':
                    labelarray[ind] = labeldict[objectid]

        feature_dict['labels_array'] = labelarray


    feature_dict['kwargs'] = {'pklglob':pklglob,
                              'featurestouse':featurestouse,
                              'maxobjects':maxobjects,
                              'labeltype':labeltype}

    # write the info to the output pickle
    with open(outfile,'wb') as outfd:
        pickle.dump(feature_dict, outfd, pickle.HIGHEST_PROTOCOL)

    # return the feature_dict
    return feature_dict
def collect_nonperiodic_features(
        featuresdir,
        magcol,
        outfile,
        pklglob='varfeatures-*.pkl',
        featurestouse=NONPERIODIC_FEATURES_TO_COLLECT,
        maxobjects=None,
        labeldict=None,
        labeltype='binary',
):
    '''This collects variability features into arrays for use with the classifer.

    Parameters
    ----------

    featuresdir : str
        This is the directory where all the varfeatures pickles are. Use
        `pklglob` to specify the glob to search for. The `varfeatures` pickles
        contain objectids, a light curve magcol, and features as dict
        key-vals. The :py:mod:`astrobase.lcproc.lcvfeatures` module can be used
        to produce these.

    magcol : str
        This is the key in each varfeatures pickle corresponding to the magcol
        of the light curve the variability features were extracted from.

    outfile : str
        This is the filename of the output pickle that will be written
        containing a dict of all the features extracted into np.arrays.

    pklglob : str
        This is the UNIX file glob to use to search for varfeatures pickle files
        in `featuresdir`.

    featurestouse : list of str
        Each varfeatures pickle can contain any combination of non-periodic,
        stellar, and periodic features; these must have the same names as
        elements in the list of strings provided in `featurestouse`.  This tries
        to get all the features listed in NONPERIODIC_FEATURES_TO_COLLECT by
        default. If `featurestouse` is provided as a list, gets only the
        features listed in this kwarg instead.

    maxobjects : int or None
        The controls how many pickles from the featuresdir to process. If None,
        will process all varfeatures pickles.

    labeldict : dict or None
        If this is provided, it must be a dict with the following key:val list::

            '<objectid>':<label value>

        for each objectid collected from the varfeatures pickles. This will turn
        the collected information into a training set for classifiers.

        Example: to carry out non-periodic variable feature collection of fake
        LCS prepared by :py:mod:`astrobase.fakelcs.generation`, use the value
        of the 'isvariable' dict elem from the `fakelcs-info.pkl` here, like
        so::

            labeldict={x:y for x,y in zip(fakelcinfo['objectid'],
                                          fakelcinfo['isvariable'])}

    labeltype : {'binary', 'classes'}
        This is either 'binary' or 'classes' for binary/multi-class
        classification respectively.

    Returns
    -------

    dict
        This returns a dict with all of the features collected into np.arrays,
        ready to use as input to a scikit-learn classifier.

    '''

    # list of input pickles generated by varfeatures in lcproc.py
    pklist = glob.glob(os.path.join(featuresdir, pklglob))

    if maxobjects:
        pklist = pklist[:maxobjects]


    # fancy progress bar with tqdm if present
    if TQDM:
        listiterator = tqdm(pklist)
    else:
        listiterator = pklist

    # go through all the varfeatures arrays

    feature_dict = {'objectids':[],'magcol':magcol, 'availablefeatures':[]}

    LOGINFO('collecting features for magcol: %s' % magcol)

    for pkl in listiterator:

        with open(pkl,'rb') as infd:
            varf = pickle.load(infd)

        # update the objectid list
        objectid = varf['objectid']
        if objectid not in feature_dict['objectids']:
            feature_dict['objectids'].append(objectid)

        thisfeatures = varf[magcol]

        if featurestouse and len(featurestouse) > 0:
            featurestoget = featurestouse
        else:
            featurestoget = NONPERIODIC_FEATURES_TO_COLLECT

        # collect all the features for this magcol/objectid combination
        for feature in featurestoget:

            # update the global feature list if necessary
            if ((feature not in feature_dict['availablefeatures']) and
                (feature in thisfeatures)):

                feature_dict['availablefeatures'].append(feature)
                feature_dict[feature] = []

            if feature in thisfeatures:

                feature_dict[feature].append(
                    thisfeatures[feature]
                )

    # now that we've collected all the objects and their features, turn the list
    # into arrays, and then concatenate them
    for feat in feature_dict['availablefeatures']:
        feature_dict[feat] = np.array(feature_dict[feat])

    feature_dict['objectids'] = np.array(feature_dict['objectids'])

    feature_array = np.column_stack([feature_dict[feat] for feat in
                                     feature_dict['availablefeatures']])
    feature_dict['features_array'] = feature_array


    # if there's a labeldict available, use it to generate a label array. this
    # feature collection is now a training set.
    if isinstance(labeldict, dict):

        labelarray = np.zeros(feature_dict['objectids'].size, dtype=np.int64)

        # populate the labels for each object in the training set
        for ind, objectid in enumerate(feature_dict['objectids']):

            if objectid in labeldict:

                # if this is a binary classifier training set, convert bools to
                # ones and zeros
                if labeltype == 'binary':

                    if labeldict[objectid]:
                        labelarray[ind] = 1

                # otherwise, use the actual class label integer
                elif labeltype == 'classes':
                    labelarray[ind] = labeldict[objectid]

        feature_dict['labels_array'] = labelarray


    feature_dict['kwargs'] = {'pklglob':pklglob,
                              'featurestouse':featurestouse,
                              'maxobjects':maxobjects,
                              'labeltype':labeltype}

    # write the info to the output pickle
    with open(outfile,'wb') as outfd:
        pickle.dump(feature_dict, outfd, pickle.HIGHEST_PROTOCOL)

    # return the feature_dict
    return feature_dict
def collect_nonperiodic_features ( featuresdir , magcol , outfile , pklglob = 'varfeatures-*.pkl' , featurestouse = NONPERIODIC_FEATURES_TO_COLLECT , maxobjects = None , labeldict = None , labeltype = 'binary' , ) : # list of input pickles generated by varfeatures in lcproc.py pklist = glob . glob ( os . path . join ( featuresdir , pklglob ) ) if maxobjects : pklist = pklist [ : maxobjects ] # fancy progress bar with tqdm if present if TQDM : listiterator = tqdm ( pklist ) else : listiterator = pklist # go through all the varfeatures arrays feature_dict = { 'objectids' : [ ] , 'magcol' : magcol , 'availablefeatures' : [ ] } LOGINFO ( 'collecting features for magcol: %s' % magcol ) for pkl in listiterator : with open ( pkl , 'rb' ) as infd : varf = pickle . load ( infd ) # update the objectid list objectid = varf [ 'objectid' ] if objectid not in feature_dict [ 'objectids' ] : feature_dict [ 'objectids' ] . append ( objectid ) thisfeatures = varf [ magcol ] if featurestouse and len ( featurestouse ) > 0 : featurestoget = featurestouse else : featurestoget = NONPERIODIC_FEATURES_TO_COLLECT # collect all the features for this magcol/objectid combination for feature in featurestoget : # update the global feature list if necessary if ( ( feature not in feature_dict [ 'availablefeatures' ] ) and ( feature in thisfeatures ) ) : feature_dict [ 'availablefeatures' ] . append ( feature ) feature_dict [ feature ] = [ ] if feature in thisfeatures : feature_dict [ feature ] . append ( thisfeatures [ feature ] ) # now that we've collected all the objects and their features, turn the list # into arrays, and then concatenate them for feat in feature_dict [ 'availablefeatures' ] : feature_dict [ feat ] = np . array ( feature_dict [ feat ] ) feature_dict [ 'objectids' ] = np . array ( feature_dict [ 'objectids' ] ) feature_array = np . column_stack ( [ feature_dict [ feat ] for feat in feature_dict [ 'availablefeatures' ] ] ) feature_dict [ 'features_array' ] = feature_array # if there's a labeldict available, use it to generate a label array. this # feature collection is now a training set. if isinstance ( labeldict , dict ) : labelarray = np . zeros ( feature_dict [ 'objectids' ] . size , dtype = np . int64 ) # populate the labels for each object in the training set for ind , objectid in enumerate ( feature_dict [ 'objectids' ] ) : if objectid in labeldict : # if this is a binary classifier training set, convert bools to # ones and zeros if labeltype == 'binary' : if labeldict [ objectid ] : labelarray [ ind ] = 1 # otherwise, use the actual class label integer elif labeltype == 'classes' : labelarray [ ind ] = labeldict [ objectid ] feature_dict [ 'labels_array' ] = labelarray feature_dict [ 'kwargs' ] = { 'pklglob' : pklglob , 'featurestouse' : featurestouse , 'maxobjects' : maxobjects , 'labeltype' : labeltype } # write the info to the output pickle with open ( outfile , 'wb' ) as outfd : pickle . dump ( feature_dict , outfd , pickle . HIGHEST_PROTOCOL ) # return the feature_dict return feature_dict
This collects variability features into arrays for use with the classifer.

    Parameters
    ----------

    featuresdir : str
        This is the directory where all the varfeatures pickles are. Use
        `pklglob` to specify the glob to search for. The `varfeatures` pickles
        contain objectids, a light curve magcol, and features as dict
        key-vals. The :py:mod:`astrobase.lcproc.lcvfeatures` module can be used
        to produce these.

    magcol : str
        This is the key in each varfeatures pickle corresponding to the magcol
        of the light curve the variability features were extracted from.

    outfile : str
        This is the filename of the output pickle that will be written
        containing a dict of all the features extracted into np.arrays.

    pklglob : str
        This is the UNIX file glob to use to search for varfeatures pickle files
        in `featuresdir`.

    featurestouse : list of str
        Each varfeatures pickle can contain any combination of non-periodic,
        stellar, and periodic features; these must have the same names as
        elements in the list of strings provided in `featurestouse`.  This tries
        to get all the features listed in NONPERIODIC_FEATURES_TO_COLLECT by
        default. If `featurestouse` is provided as a list, gets only the
        features listed in this kwarg instead.

    maxobjects : int or None
        The controls how many pickles from the featuresdir to process. If None,
        will process all varfeatures pickles.

    labeldict : dict or None
        If this is provided, it must be a dict with the following key:val list::

            '<objectid>':<label value>

        for each objectid collected from the varfeatures pickles. This will turn
        the collected information into a training set for classifiers.

        Example: to carry out non-periodic variable feature collection of fake
        LCS prepared by :py:mod:`astrobase.fakelcs.generation`, use the value
        of the 'isvariable' dict elem from the `fakelcs-info.pkl` here, like
        so::

            labeldict={x:y for x,y in zip(fakelcinfo['objectid'],
                                          fakelcinfo['isvariable'])}

    labeltype : {'binary', 'classes'}
        This is either 'binary' or 'classes' for binary/multi-class
        classification respectively.

    Returns
    -------

    dict
        This returns a dict with all of the features collected into np.arrays,
        ready to use as input to a scikit-learn classifier.
This collects variability features into arrays for use with the classifer .