dsub/lib/dsub_util.py
def simple_pattern_exists_in_gcs(file_pattern, credentials=None):
  """True iff an object exists matching the input GCS pattern.

  The GCS pattern must be a full object reference or a "simple pattern" that
  conforms to the dsub input and output parameter restrictions:

    * No support for **, ? wildcards or [] character ranges
    * Wildcards may only appear in the file name

  Args:
    file_pattern: eg. 'gs://foo/ba*'
    credentials: Optional credential to be used to load the file from gcs.

  Raises:
    ValueError: if file_pattern breaks the rules.

  Returns:
    True iff a file exists that matches that pattern.
  """
  if '*' not in file_pattern:
    return _file_exists_in_gcs(file_pattern, credentials)
  if not file_pattern.startswith('gs://'):
    raise ValueError('file name must start with gs://')
  gcs_service = _get_storage_service(credentials)
  bucket_name, prefix = file_pattern[len('gs://'):].split('/', 1)
  if '*' in bucket_name:
    raise ValueError('Wildcards may not appear in the bucket name')
  # There is a '*' in prefix because we checked there's one in file_pattern
  # and there isn't one in bucket_name. Hence it must be in prefix.
  assert '*' in prefix
  prefix_no_wildcard = prefix[:prefix.index('*')]
  request = gcs_service.objects().list(
      bucket=bucket_name, prefix=prefix_no_wildcard)
  response = request.execute()
  if 'items' not in response:
    return False
  items_list = [i['name'] for i in response['items']]
  return any(fnmatch.fnmatch(i, prefix) for i in items_list)
def simple_pattern_exists_in_gcs(file_pattern, credentials=None):
  """True iff an object exists matching the input GCS pattern.

  The GCS pattern must be a full object reference or a "simple pattern" that
  conforms to the dsub input and output parameter restrictions:

    * No support for **, ? wildcards or [] character ranges
    * Wildcards may only appear in the file name

  Args:
    file_pattern: eg. 'gs://foo/ba*'
    credentials: Optional credential to be used to load the file from gcs.

  Raises:
    ValueError: if file_pattern breaks the rules.

  Returns:
    True iff a file exists that matches that pattern.
  """
  if '*' not in file_pattern:
    return _file_exists_in_gcs(file_pattern, credentials)
  if not file_pattern.startswith('gs://'):
    raise ValueError('file name must start with gs://')
  gcs_service = _get_storage_service(credentials)
  bucket_name, prefix = file_pattern[len('gs://'):].split('/', 1)
  if '*' in bucket_name:
    raise ValueError('Wildcards may not appear in the bucket name')
  # There is a '*' in prefix because we checked there's one in file_pattern
  # and there isn't one in bucket_name. Hence it must be in prefix.
  assert '*' in prefix
  prefix_no_wildcard = prefix[:prefix.index('*')]
  request = gcs_service.objects().list(
      bucket=bucket_name, prefix=prefix_no_wildcard)
  response = request.execute()
  if 'items' not in response:
    return False
  items_list = [i['name'] for i in response['items']]
  return any(fnmatch.fnmatch(i, prefix) for i in items_list)
def simple_pattern_exists_in_gcs ( file_pattern , credentials = None ) : if '*' not in file_pattern : return _file_exists_in_gcs ( file_pattern , credentials ) if not file_pattern . startswith ( 'gs://' ) : raise ValueError ( 'file name must start with gs://' ) gcs_service = _get_storage_service ( credentials ) bucket_name , prefix = file_pattern [ len ( 'gs://' ) : ] . split ( '/' , 1 ) if '*' in bucket_name : raise ValueError ( 'Wildcards may not appear in the bucket name' ) # There is a '*' in prefix because we checked there's one in file_pattern # and there isn't one in bucket_name. Hence it must be in prefix. assert '*' in prefix prefix_no_wildcard = prefix [ : prefix . index ( '*' ) ] request = gcs_service . objects ( ) . list ( bucket = bucket_name , prefix = prefix_no_wildcard ) response = request . execute ( ) if 'items' not in response : return False items_list = [ i [ 'name' ] for i in response [ 'items' ] ] return any ( fnmatch . fnmatch ( i , prefix ) for i in items_list )
True iff an object exists matching the input GCS pattern.

  The GCS pattern must be a full object reference or a "simple pattern" that
  conforms to the dsub input and output parameter restrictions:

    * No support for **, ? wildcards or [] character ranges
    * Wildcards may only appear in the file name

  Args:
    file_pattern: eg. 'gs://foo/ba*'
    credentials: Optional credential to be used to load the file from gcs.

  Raises:
    ValueError: if file_pattern breaks the rules.

  Returns:
    True iff a file exists that matches that pattern.
True iff an object exists matching the input GCS pattern .