astrobase/services/dust.py
def extinction_query(lon, lat,
                     coordtype='equatorial',
                     sizedeg=5.0,
                     forcefetch=False,
                     cachedir='~/.astrobase/dust-cache',
                     verbose=True,
                     timeout=10.0,
                     jitter=5.0):
    '''This queries the 2MASS DUST service to find the extinction parameters
    for the given `lon`, `lat`.

    Parameters
    ----------

    lon,lat: float
        These are decimal right ascension and declination if `coordtype =
        'equatorial'`. These are are decimal Galactic longitude and latitude if
        `coordtype = 'galactic'`.

    coordtype : {'equatorial','galactic'}
        Sets the type of coordinates passed in as `lon`, `lat`.

    sizedeg : float
        This is the width of the image returned by the DUST service. This can
        usually be left as-is if you're interested in the extinction only.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our request.

    jitter : float
        This is used to control the scale of the random wait in seconds before
        starting the query. Useful in parallelized situations.

    Returns
    -------

    dict
        A dict of the following form is returned::

            {'Amag':{dict of extinction A_v values for several mag systems},
             'table': array containing the full extinction table,
             'tablefile': the path to the full extinction table file on disk,
             'provenance': 'cached' or 'new download',
             'request': string repr of the request made to 2MASS DUST}

    '''

    dustparams = DUST_PARAMS.copy()

    # convert the lon, lat to the required format
    # and generate the param dict
    if coordtype == 'equatorial':
        locstr = '%.3f %.3f Equ J2000' % (lon, lat)
    elif coordtype == 'galactic':
        locstr = '%.3f %.3f gal' % (lon, lat)
    else:
        LOGERROR('unknown coordinate type: %s' % coordtype)
        return None

    dustparams['locstr'] = locstr
    dustparams['regSize'] = '%.3f' % sizedeg

    # see if the cachedir exists
    if '~' in cachedir:
        cachedir = os.path.expanduser(cachedir)
    if not os.path.exists(cachedir):
        os.makedirs(cachedir)

    # generate the cachekey and cache filename
    cachekey = '%s - %.1f' % (locstr, sizedeg)
    cachekey = hashlib.sha256(cachekey.encode()).hexdigest()
    cachefname = os.path.join(cachedir, '%s.txt' % cachekey)
    provenance = 'cache'

    # if this does not exist in cache or if we're forcefetching, do the query
    if forcefetch or (not os.path.exists(cachefname)):

        time.sleep(random.randint(1,jitter))
        provenance = 'new download'

        try:

            if verbose:
                LOGINFO('submitting 2MASS DUST request for '
                        'lon = %.3f, lat = %.3f, type = %s, size = %.1f' %
                        (lon, lat, coordtype, sizedeg))

            req = requests.get(DUST_URL, dustparams, timeout=timeout)
            req.raise_for_status()

            resp = req.text

            # see if we got an extinction table URL in the response
            tableurl = DUST_REGEX.search(resp)

            # if we did, download it to the cache directory
            if tableurl:

                tableurl = tableurl.group(0)

                req2 = requests.get(tableurl, timeout=timeout)

                # write the table to the cache directory
                with open(cachefname,'wb') as outfd:
                    outfd.write(req2.content)

                tablefname = cachefname

            else:
                LOGERROR('could not get extinction parameters for '
                         '%s (%.3f, %.3f) with size = %.1f' % (coordtype,
                                                               lon,lat,sizedeg))
                LOGERROR('error from DUST service follows:\n%s' % resp)
                return None

        except requests.exceptions.Timeout as e:
            LOGERROR('DUST request timed out for '
                     '%s (%.3f, %.3f) with size = %.1f' % (coordtype,
                                                           lon,lat,sizedeg))
            return None

        except Exception as e:
            LOGEXCEPTION('DUST request failed for '
                         '%s (%.3f, %.3f) with size = %.1f' % (coordtype,
                                                               lon,lat,sizedeg))
            return None

    # if this result is available in the cache, get it from there
    else:

        if verbose:
            LOGINFO('getting cached 2MASS DUST result for '
                    'lon = %.3f, lat = %.3f, coordtype = %s, size = %.1f' %
                    (lon, lat, coordtype, sizedeg))

        tablefname = cachefname

    #
    # now we should have the extinction table in some form
    #
    # read and parse the extinction table using astropy.Table
    extinction_table = Table.read(tablefname, format='ascii.ipac')

    # get the columns we need
    filters = np.array(extinction_table['Filter_name'])
    a_sf11_byfilter = np.array(extinction_table['A_SandF'])
    a_sfd98_byfilter = np.array(extinction_table['A_SFD'])

    # generate the output dict
    extdict = {'Amag':{x:{'sf11':y, 'sfd98':z} for
                       x,y,z in zip(filters,a_sf11_byfilter,a_sfd98_byfilter)},
               'table':np.array(extinction_table),
               'tablefile':os.path.abspath(cachefname),
               'provenance':provenance,
               'request':'%s (%.3f, %.3f) with size = %.1f' % (coordtype,
                                                               lon,lat,
                                                               sizedeg)}

    return extdict
def extinction_query(lon, lat,
                     coordtype='equatorial',
                     sizedeg=5.0,
                     forcefetch=False,
                     cachedir='~/.astrobase/dust-cache',
                     verbose=True,
                     timeout=10.0,
                     jitter=5.0):
    '''This queries the 2MASS DUST service to find the extinction parameters
    for the given `lon`, `lat`.

    Parameters
    ----------

    lon,lat: float
        These are decimal right ascension and declination if `coordtype =
        'equatorial'`. These are are decimal Galactic longitude and latitude if
        `coordtype = 'galactic'`.

    coordtype : {'equatorial','galactic'}
        Sets the type of coordinates passed in as `lon`, `lat`.

    sizedeg : float
        This is the width of the image returned by the DUST service. This can
        usually be left as-is if you're interested in the extinction only.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our request.

    jitter : float
        This is used to control the scale of the random wait in seconds before
        starting the query. Useful in parallelized situations.

    Returns
    -------

    dict
        A dict of the following form is returned::

            {'Amag':{dict of extinction A_v values for several mag systems},
             'table': array containing the full extinction table,
             'tablefile': the path to the full extinction table file on disk,
             'provenance': 'cached' or 'new download',
             'request': string repr of the request made to 2MASS DUST}

    '''

    dustparams = DUST_PARAMS.copy()

    # convert the lon, lat to the required format
    # and generate the param dict
    if coordtype == 'equatorial':
        locstr = '%.3f %.3f Equ J2000' % (lon, lat)
    elif coordtype == 'galactic':
        locstr = '%.3f %.3f gal' % (lon, lat)
    else:
        LOGERROR('unknown coordinate type: %s' % coordtype)
        return None

    dustparams['locstr'] = locstr
    dustparams['regSize'] = '%.3f' % sizedeg

    # see if the cachedir exists
    if '~' in cachedir:
        cachedir = os.path.expanduser(cachedir)
    if not os.path.exists(cachedir):
        os.makedirs(cachedir)

    # generate the cachekey and cache filename
    cachekey = '%s - %.1f' % (locstr, sizedeg)
    cachekey = hashlib.sha256(cachekey.encode()).hexdigest()
    cachefname = os.path.join(cachedir, '%s.txt' % cachekey)
    provenance = 'cache'

    # if this does not exist in cache or if we're forcefetching, do the query
    if forcefetch or (not os.path.exists(cachefname)):

        time.sleep(random.randint(1,jitter))
        provenance = 'new download'

        try:

            if verbose:
                LOGINFO('submitting 2MASS DUST request for '
                        'lon = %.3f, lat = %.3f, type = %s, size = %.1f' %
                        (lon, lat, coordtype, sizedeg))

            req = requests.get(DUST_URL, dustparams, timeout=timeout)
            req.raise_for_status()

            resp = req.text

            # see if we got an extinction table URL in the response
            tableurl = DUST_REGEX.search(resp)

            # if we did, download it to the cache directory
            if tableurl:

                tableurl = tableurl.group(0)

                req2 = requests.get(tableurl, timeout=timeout)

                # write the table to the cache directory
                with open(cachefname,'wb') as outfd:
                    outfd.write(req2.content)

                tablefname = cachefname

            else:
                LOGERROR('could not get extinction parameters for '
                         '%s (%.3f, %.3f) with size = %.1f' % (coordtype,
                                                               lon,lat,sizedeg))
                LOGERROR('error from DUST service follows:\n%s' % resp)
                return None

        except requests.exceptions.Timeout as e:
            LOGERROR('DUST request timed out for '
                     '%s (%.3f, %.3f) with size = %.1f' % (coordtype,
                                                           lon,lat,sizedeg))
            return None

        except Exception as e:
            LOGEXCEPTION('DUST request failed for '
                         '%s (%.3f, %.3f) with size = %.1f' % (coordtype,
                                                               lon,lat,sizedeg))
            return None

    # if this result is available in the cache, get it from there
    else:

        if verbose:
            LOGINFO('getting cached 2MASS DUST result for '
                    'lon = %.3f, lat = %.3f, coordtype = %s, size = %.1f' %
                    (lon, lat, coordtype, sizedeg))

        tablefname = cachefname

    #
    # now we should have the extinction table in some form
    #
    # read and parse the extinction table using astropy.Table
    extinction_table = Table.read(tablefname, format='ascii.ipac')

    # get the columns we need
    filters = np.array(extinction_table['Filter_name'])
    a_sf11_byfilter = np.array(extinction_table['A_SandF'])
    a_sfd98_byfilter = np.array(extinction_table['A_SFD'])

    # generate the output dict
    extdict = {'Amag':{x:{'sf11':y, 'sfd98':z} for
                       x,y,z in zip(filters,a_sf11_byfilter,a_sfd98_byfilter)},
               'table':np.array(extinction_table),
               'tablefile':os.path.abspath(cachefname),
               'provenance':provenance,
               'request':'%s (%.3f, %.3f) with size = %.1f' % (coordtype,
                                                               lon,lat,
                                                               sizedeg)}

    return extdict
def extinction_query ( lon , lat , coordtype = 'equatorial' , sizedeg = 5.0 , forcefetch = False , cachedir = '~/.astrobase/dust-cache' , verbose = True , timeout = 10.0 , jitter = 5.0 ) : dustparams = DUST_PARAMS . copy ( ) # convert the lon, lat to the required format # and generate the param dict if coordtype == 'equatorial' : locstr = '%.3f %.3f Equ J2000' % ( lon , lat ) elif coordtype == 'galactic' : locstr = '%.3f %.3f gal' % ( lon , lat ) else : LOGERROR ( 'unknown coordinate type: %s' % coordtype ) return None dustparams [ 'locstr' ] = locstr dustparams [ 'regSize' ] = '%.3f' % sizedeg # see if the cachedir exists if '~' in cachedir : cachedir = os . path . expanduser ( cachedir ) if not os . path . exists ( cachedir ) : os . makedirs ( cachedir ) # generate the cachekey and cache filename cachekey = '%s - %.1f' % ( locstr , sizedeg ) cachekey = hashlib . sha256 ( cachekey . encode ( ) ) . hexdigest ( ) cachefname = os . path . join ( cachedir , '%s.txt' % cachekey ) provenance = 'cache' # if this does not exist in cache or if we're forcefetching, do the query if forcefetch or ( not os . path . exists ( cachefname ) ) : time . sleep ( random . randint ( 1 , jitter ) ) provenance = 'new download' try : if verbose : LOGINFO ( 'submitting 2MASS DUST request for ' 'lon = %.3f, lat = %.3f, type = %s, size = %.1f' % ( lon , lat , coordtype , sizedeg ) ) req = requests . get ( DUST_URL , dustparams , timeout = timeout ) req . raise_for_status ( ) resp = req . text # see if we got an extinction table URL in the response tableurl = DUST_REGEX . search ( resp ) # if we did, download it to the cache directory if tableurl : tableurl = tableurl . group ( 0 ) req2 = requests . get ( tableurl , timeout = timeout ) # write the table to the cache directory with open ( cachefname , 'wb' ) as outfd : outfd . write ( req2 . content ) tablefname = cachefname else : LOGERROR ( 'could not get extinction parameters for ' '%s (%.3f, %.3f) with size = %.1f' % ( coordtype , lon , lat , sizedeg ) ) LOGERROR ( 'error from DUST service follows:\n%s' % resp ) return None except requests . exceptions . Timeout as e : LOGERROR ( 'DUST request timed out for ' '%s (%.3f, %.3f) with size = %.1f' % ( coordtype , lon , lat , sizedeg ) ) return None except Exception as e : LOGEXCEPTION ( 'DUST request failed for ' '%s (%.3f, %.3f) with size = %.1f' % ( coordtype , lon , lat , sizedeg ) ) return None # if this result is available in the cache, get it from there else : if verbose : LOGINFO ( 'getting cached 2MASS DUST result for ' 'lon = %.3f, lat = %.3f, coordtype = %s, size = %.1f' % ( lon , lat , coordtype , sizedeg ) ) tablefname = cachefname # # now we should have the extinction table in some form # # read and parse the extinction table using astropy.Table extinction_table = Table . read ( tablefname , format = 'ascii.ipac' ) # get the columns we need filters = np . array ( extinction_table [ 'Filter_name' ] ) a_sf11_byfilter = np . array ( extinction_table [ 'A_SandF' ] ) a_sfd98_byfilter = np . array ( extinction_table [ 'A_SFD' ] ) # generate the output dict extdict = { 'Amag' : { x : { 'sf11' : y , 'sfd98' : z } for x , y , z in zip ( filters , a_sf11_byfilter , a_sfd98_byfilter ) } , 'table' : np . array ( extinction_table ) , 'tablefile' : os . path . abspath ( cachefname ) , 'provenance' : provenance , 'request' : '%s (%.3f, %.3f) with size = %.1f' % ( coordtype , lon , lat , sizedeg ) } return extdict
This queries the 2MASS DUST service to find the extinction parameters
    for the given `lon`, `lat`.

    Parameters
    ----------

    lon,lat: float
        These are decimal right ascension and declination if `coordtype =
        'equatorial'`. These are are decimal Galactic longitude and latitude if
        `coordtype = 'galactic'`.

    coordtype : {'equatorial','galactic'}
        Sets the type of coordinates passed in as `lon`, `lat`.

    sizedeg : float
        This is the width of the image returned by the DUST service. This can
        usually be left as-is if you're interested in the extinction only.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our request.

    jitter : float
        This is used to control the scale of the random wait in seconds before
        starting the query. Useful in parallelized situations.

    Returns
    -------

    dict
        A dict of the following form is returned::

            {'Amag':{dict of extinction A_v values for several mag systems},
             'table': array containing the full extinction table,
             'tablefile': the path to the full extinction table file on disk,
             'provenance': 'cached' or 'new download',
             'request': string repr of the request made to 2MASS DUST}
This queries the 2MASS DUST service to find the extinction parameters for the given lon lat .