astrobase/checkplot/pkl.py
def checkplot_pickle_update(
        currentcp,
        updatedcp,
        outfile=None,
        outgzip=False,
        pickleprotocol=None,
        verbose=True
):
    '''This updates the current checkplotdict with updated values provided.


    Parameters
    ----------

    currentcp : dict or str
        This is either a checkplotdict produced by `checkplot_pickle` above or a
        checkplot pickle file produced by the same function. This checkplot will
        be updated from the `updatedcp` checkplot.

    updatedcp : dict or str
        This is either a checkplotdict produced by `checkplot_pickle` above or a
        checkplot pickle file produced by the same function. This checkplot will
        be the source of the update to the  `currentcp` checkplot.

    outfile : str or None
        The name of the output checkplot pickle file. The function will output
        the new checkplot gzipped pickle file to `outfile` if outfile is a
        filename. If `currentcp` is a file and `outfile`, this will be set to
        that filename, so the function updates it in place.

    outgzip : bool
        This controls whether to gzip the output pickle. It turns out that this
        is the slowest bit in the output process, so if you're after speed, best
        not to use this. This is False by default since it turns out that gzip
        actually doesn't save that much space (29 MB vs. 35 MB for the average
        checkplot pickle).

    pickleprotocol : int or None
        This sets the pickle file protocol to use when writing the pickle:

        If None, will choose a protocol using the following rules:

        - 4 -> default in Python >= 3.4 - fast but incompatible with Python 2
        - 3 -> default in Python 3.0-3.3 - mildly fast
        - 2 -> default in Python 2 - very slow, but compatible with Python 2/3

        The default protocol kwarg is None, this will make an automatic choice
        for pickle protocol that's best suited for the version of Python in
        use. Note that this will make pickles generated by Py3 incompatible with
        Py2.

    verbose : bool
        If True, will indicate progress and warn about problems.

    Returns
    -------

    str
        The path to the updated checkplot pickle file. If `outfile` was None and
        `currentcp` was a filename, this will return `currentcp` to indicate
        that the checkplot pickle file was updated in place.

    '''

    # break out python 2.7 and > 3 nonsense
    if sys.version_info[:2] > (3,2):

        # generate the outfile filename
        if not outfile and isinstance(currentcp,str):
            plotfpath = currentcp
        elif outfile:
            plotfpath = outfile
        elif isinstance(currentcp, dict) and currentcp['objectid']:
            if outgzip:
                plotfpath = 'checkplot-%s.pkl.gz' % currentcp['objectid']
            else:
                plotfpath = 'checkplot-%s.pkl' % currentcp['objectid']
        else:
            # we'll get this later below
            plotfpath = None

        if (isinstance(currentcp, str) and os.path.exists(currentcp)):
            cp_current = _read_checkplot_picklefile(currentcp)
        elif isinstance(currentcp, dict):
            cp_current = currentcp
        else:
            LOGERROR('currentcp: %s of type %s is not a '
                     'valid checkplot filename (or does not exist), or a dict' %
                     (os.path.abspath(currentcp), type(currentcp)))
            return None

        if (isinstance(updatedcp, str) and os.path.exists(updatedcp)):
            cp_updated = _read_checkplot_picklefile(updatedcp)
        elif isinstance(updatedcp, dict):
            cp_updated = updatedcp
        else:
            LOGERROR('updatedcp: %s of type %s is not a '
                     'valid checkplot filename (or does not exist), or a dict' %
                     (os.path.abspath(updatedcp), type(updatedcp)))
            return None

    # check for unicode in python 2.7
    else:

        # generate the outfile filename
        if (not outfile and
            (isinstance(currentcp, str) or isinstance(currentcp, unicode))):
            plotfpath = currentcp
        elif outfile:
            plotfpath = outfile
        elif isinstance(currentcp, dict) and currentcp['objectid']:
            if outgzip:
                plotfpath = 'checkplot-%s.pkl.gz' % currentcp['objectid']
            else:
                plotfpath = 'checkplot-%s.pkl' % currentcp['objectid']
        else:
            # we'll get this later below
            plotfpath = None

        # get the current checkplotdict
        if ((isinstance(currentcp, str) or isinstance(currentcp, unicode)) and
            os.path.exists(currentcp)):
            cp_current = _read_checkplot_picklefile(currentcp)
        elif isinstance(currentcp,dict):
            cp_current = currentcp
        else:
            LOGERROR('currentcp: %s of type %s is not a '
                     'valid checkplot filename (or does not exist), or a dict' %
                     (os.path.abspath(currentcp), type(currentcp)))
            return None

        # get the updated checkplotdict
        if ((isinstance(updatedcp, str) or isinstance(updatedcp, unicode)) and
            os.path.exists(updatedcp)):
            cp_updated = _read_checkplot_picklefile(updatedcp)
        elif isinstance(updatedcp, dict):
            cp_updated = updatedcp
        else:
            LOGERROR('updatedcp: %s of type %s is not a '
                     'valid checkplot filename (or does not exist), or a dict' %
                     (os.path.abspath(updatedcp), type(updatedcp)))
            return None

    # do the update using python's dict update mechanism
    # this requires updated to be in the same checkplotdict format as current
    # all keys in current will now be from updated
    cp_current.update(cp_updated)

    # figure out the plotfpath if we haven't by now
    if not plotfpath and outgzip:
        plotfpath = 'checkplot-%s.pkl.gz' % cp_current['objectid']
    elif (not plotfpath) and (not outgzip):
        plotfpath = 'checkplot-%s.pkl' % cp_current['objectid']

    # make sure we write the correct postfix
    if plotfpath.endswith('.gz'):
        outgzip = True

    # write the new checkplotdict
    return _write_checkplot_picklefile(cp_current,
                                       outfile=plotfpath,
                                       outgzip=outgzip,
                                       protocol=pickleprotocol)
def checkplot_pickle_update(
        currentcp,
        updatedcp,
        outfile=None,
        outgzip=False,
        pickleprotocol=None,
        verbose=True
):
    '''This updates the current checkplotdict with updated values provided.


    Parameters
    ----------

    currentcp : dict or str
        This is either a checkplotdict produced by `checkplot_pickle` above or a
        checkplot pickle file produced by the same function. This checkplot will
        be updated from the `updatedcp` checkplot.

    updatedcp : dict or str
        This is either a checkplotdict produced by `checkplot_pickle` above or a
        checkplot pickle file produced by the same function. This checkplot will
        be the source of the update to the  `currentcp` checkplot.

    outfile : str or None
        The name of the output checkplot pickle file. The function will output
        the new checkplot gzipped pickle file to `outfile` if outfile is a
        filename. If `currentcp` is a file and `outfile`, this will be set to
        that filename, so the function updates it in place.

    outgzip : bool
        This controls whether to gzip the output pickle. It turns out that this
        is the slowest bit in the output process, so if you're after speed, best
        not to use this. This is False by default since it turns out that gzip
        actually doesn't save that much space (29 MB vs. 35 MB for the average
        checkplot pickle).

    pickleprotocol : int or None
        This sets the pickle file protocol to use when writing the pickle:

        If None, will choose a protocol using the following rules:

        - 4 -> default in Python >= 3.4 - fast but incompatible with Python 2
        - 3 -> default in Python 3.0-3.3 - mildly fast
        - 2 -> default in Python 2 - very slow, but compatible with Python 2/3

        The default protocol kwarg is None, this will make an automatic choice
        for pickle protocol that's best suited for the version of Python in
        use. Note that this will make pickles generated by Py3 incompatible with
        Py2.

    verbose : bool
        If True, will indicate progress and warn about problems.

    Returns
    -------

    str
        The path to the updated checkplot pickle file. If `outfile` was None and
        `currentcp` was a filename, this will return `currentcp` to indicate
        that the checkplot pickle file was updated in place.

    '''

    # break out python 2.7 and > 3 nonsense
    if sys.version_info[:2] > (3,2):

        # generate the outfile filename
        if not outfile and isinstance(currentcp,str):
            plotfpath = currentcp
        elif outfile:
            plotfpath = outfile
        elif isinstance(currentcp, dict) and currentcp['objectid']:
            if outgzip:
                plotfpath = 'checkplot-%s.pkl.gz' % currentcp['objectid']
            else:
                plotfpath = 'checkplot-%s.pkl' % currentcp['objectid']
        else:
            # we'll get this later below
            plotfpath = None

        if (isinstance(currentcp, str) and os.path.exists(currentcp)):
            cp_current = _read_checkplot_picklefile(currentcp)
        elif isinstance(currentcp, dict):
            cp_current = currentcp
        else:
            LOGERROR('currentcp: %s of type %s is not a '
                     'valid checkplot filename (or does not exist), or a dict' %
                     (os.path.abspath(currentcp), type(currentcp)))
            return None

        if (isinstance(updatedcp, str) and os.path.exists(updatedcp)):
            cp_updated = _read_checkplot_picklefile(updatedcp)
        elif isinstance(updatedcp, dict):
            cp_updated = updatedcp
        else:
            LOGERROR('updatedcp: %s of type %s is not a '
                     'valid checkplot filename (or does not exist), or a dict' %
                     (os.path.abspath(updatedcp), type(updatedcp)))
            return None

    # check for unicode in python 2.7
    else:

        # generate the outfile filename
        if (not outfile and
            (isinstance(currentcp, str) or isinstance(currentcp, unicode))):
            plotfpath = currentcp
        elif outfile:
            plotfpath = outfile
        elif isinstance(currentcp, dict) and currentcp['objectid']:
            if outgzip:
                plotfpath = 'checkplot-%s.pkl.gz' % currentcp['objectid']
            else:
                plotfpath = 'checkplot-%s.pkl' % currentcp['objectid']
        else:
            # we'll get this later below
            plotfpath = None

        # get the current checkplotdict
        if ((isinstance(currentcp, str) or isinstance(currentcp, unicode)) and
            os.path.exists(currentcp)):
            cp_current = _read_checkplot_picklefile(currentcp)
        elif isinstance(currentcp,dict):
            cp_current = currentcp
        else:
            LOGERROR('currentcp: %s of type %s is not a '
                     'valid checkplot filename (or does not exist), or a dict' %
                     (os.path.abspath(currentcp), type(currentcp)))
            return None

        # get the updated checkplotdict
        if ((isinstance(updatedcp, str) or isinstance(updatedcp, unicode)) and
            os.path.exists(updatedcp)):
            cp_updated = _read_checkplot_picklefile(updatedcp)
        elif isinstance(updatedcp, dict):
            cp_updated = updatedcp
        else:
            LOGERROR('updatedcp: %s of type %s is not a '
                     'valid checkplot filename (or does not exist), or a dict' %
                     (os.path.abspath(updatedcp), type(updatedcp)))
            return None

    # do the update using python's dict update mechanism
    # this requires updated to be in the same checkplotdict format as current
    # all keys in current will now be from updated
    cp_current.update(cp_updated)

    # figure out the plotfpath if we haven't by now
    if not plotfpath and outgzip:
        plotfpath = 'checkplot-%s.pkl.gz' % cp_current['objectid']
    elif (not plotfpath) and (not outgzip):
        plotfpath = 'checkplot-%s.pkl' % cp_current['objectid']

    # make sure we write the correct postfix
    if plotfpath.endswith('.gz'):
        outgzip = True

    # write the new checkplotdict
    return _write_checkplot_picklefile(cp_current,
                                       outfile=plotfpath,
                                       outgzip=outgzip,
                                       protocol=pickleprotocol)
def checkplot_pickle_update ( currentcp , updatedcp , outfile = None , outgzip = False , pickleprotocol = None , verbose = True ) : # break out python 2.7 and > 3 nonsense if sys . version_info [ : 2 ] > ( 3 , 2 ) : # generate the outfile filename if not outfile and isinstance ( currentcp , str ) : plotfpath = currentcp elif outfile : plotfpath = outfile elif isinstance ( currentcp , dict ) and currentcp [ 'objectid' ] : if outgzip : plotfpath = 'checkplot-%s.pkl.gz' % currentcp [ 'objectid' ] else : plotfpath = 'checkplot-%s.pkl' % currentcp [ 'objectid' ] else : # we'll get this later below plotfpath = None if ( isinstance ( currentcp , str ) and os . path . exists ( currentcp ) ) : cp_current = _read_checkplot_picklefile ( currentcp ) elif isinstance ( currentcp , dict ) : cp_current = currentcp else : LOGERROR ( 'currentcp: %s of type %s is not a ' 'valid checkplot filename (or does not exist), or a dict' % ( os . path . abspath ( currentcp ) , type ( currentcp ) ) ) return None if ( isinstance ( updatedcp , str ) and os . path . exists ( updatedcp ) ) : cp_updated = _read_checkplot_picklefile ( updatedcp ) elif isinstance ( updatedcp , dict ) : cp_updated = updatedcp else : LOGERROR ( 'updatedcp: %s of type %s is not a ' 'valid checkplot filename (or does not exist), or a dict' % ( os . path . abspath ( updatedcp ) , type ( updatedcp ) ) ) return None # check for unicode in python 2.7 else : # generate the outfile filename if ( not outfile and ( isinstance ( currentcp , str ) or isinstance ( currentcp , unicode ) ) ) : plotfpath = currentcp elif outfile : plotfpath = outfile elif isinstance ( currentcp , dict ) and currentcp [ 'objectid' ] : if outgzip : plotfpath = 'checkplot-%s.pkl.gz' % currentcp [ 'objectid' ] else : plotfpath = 'checkplot-%s.pkl' % currentcp [ 'objectid' ] else : # we'll get this later below plotfpath = None # get the current checkplotdict if ( ( isinstance ( currentcp , str ) or isinstance ( currentcp , unicode ) ) and os . path . exists ( currentcp ) ) : cp_current = _read_checkplot_picklefile ( currentcp ) elif isinstance ( currentcp , dict ) : cp_current = currentcp else : LOGERROR ( 'currentcp: %s of type %s is not a ' 'valid checkplot filename (or does not exist), or a dict' % ( os . path . abspath ( currentcp ) , type ( currentcp ) ) ) return None # get the updated checkplotdict if ( ( isinstance ( updatedcp , str ) or isinstance ( updatedcp , unicode ) ) and os . path . exists ( updatedcp ) ) : cp_updated = _read_checkplot_picklefile ( updatedcp ) elif isinstance ( updatedcp , dict ) : cp_updated = updatedcp else : LOGERROR ( 'updatedcp: %s of type %s is not a ' 'valid checkplot filename (or does not exist), or a dict' % ( os . path . abspath ( updatedcp ) , type ( updatedcp ) ) ) return None # do the update using python's dict update mechanism # this requires updated to be in the same checkplotdict format as current # all keys in current will now be from updated cp_current . update ( cp_updated ) # figure out the plotfpath if we haven't by now if not plotfpath and outgzip : plotfpath = 'checkplot-%s.pkl.gz' % cp_current [ 'objectid' ] elif ( not plotfpath ) and ( not outgzip ) : plotfpath = 'checkplot-%s.pkl' % cp_current [ 'objectid' ] # make sure we write the correct postfix if plotfpath . endswith ( '.gz' ) : outgzip = True # write the new checkplotdict return _write_checkplot_picklefile ( cp_current , outfile = plotfpath , outgzip = outgzip , protocol = pickleprotocol )
This updates the current checkplotdict with updated values provided.


    Parameters
    ----------

    currentcp : dict or str
        This is either a checkplotdict produced by `checkplot_pickle` above or a
        checkplot pickle file produced by the same function. This checkplot will
        be updated from the `updatedcp` checkplot.

    updatedcp : dict or str
        This is either a checkplotdict produced by `checkplot_pickle` above or a
        checkplot pickle file produced by the same function. This checkplot will
        be the source of the update to the  `currentcp` checkplot.

    outfile : str or None
        The name of the output checkplot pickle file. The function will output
        the new checkplot gzipped pickle file to `outfile` if outfile is a
        filename. If `currentcp` is a file and `outfile`, this will be set to
        that filename, so the function updates it in place.

    outgzip : bool
        This controls whether to gzip the output pickle. It turns out that this
        is the slowest bit in the output process, so if you're after speed, best
        not to use this. This is False by default since it turns out that gzip
        actually doesn't save that much space (29 MB vs. 35 MB for the average
        checkplot pickle).

    pickleprotocol : int or None
        This sets the pickle file protocol to use when writing the pickle:

        If None, will choose a protocol using the following rules:

        - 4 -> default in Python >= 3.4 - fast but incompatible with Python 2
        - 3 -> default in Python 3.0-3.3 - mildly fast
        - 2 -> default in Python 2 - very slow, but compatible with Python 2/3

        The default protocol kwarg is None, this will make an automatic choice
        for pickle protocol that's best suited for the version of Python in
        use. Note that this will make pickles generated by Py3 incompatible with
        Py2.

    verbose : bool
        If True, will indicate progress and warn about problems.

    Returns
    -------

    str
        The path to the updated checkplot pickle file. If `outfile` was None and
        `currentcp` was a filename, this will return `currentcp` to indicate
        that the checkplot pickle file was updated in place.
This updates the current checkplotdict with updated values provided .