core/src/main/java/com/twitter/elephantbird/mapreduce/input/MultiInputFormat.java
private static Format determineFileFormat(InputSplit split,
                                            Configuration conf)
                                            throws IOException {
    FileSplit fileSplit = (FileSplit) split;

    Path file = fileSplit.getPath();

    /* we could have a an optional configuration that maps a regex on a
     * file name to a format. E.g. ".*-block.lzo" to LZO_BLOCK file.
     */

    // most of the cost is opening the file and
    // reading first lzo block (about 256k of uncompressed data)

    CompressionCodec codec = new CompressionCodecFactory(conf).getCodec(file);
    if (codec == null) {
      throw new IOException("No codec for file " + file + " found");
    }

    InputStream in = file.getFileSystem(conf).open(file);
    InputStream lzoIn = null;

    // check if the file starts with magic bytes for Block storage format.
    try {
      lzoIn = codec.createInputStream(in);

      for(byte magic : Protobufs.KNOWN_GOOD_POSITION_MARKER) {
        int b = lzoIn.read();
        if (b < 0 || (byte)b != magic) {
          return Format.LZO_B64LINE;
        }
      }
    } finally {
      IOUtils.closeStream(lzoIn);
      IOUtils.closeStream(in);
    }

    // the check passed
    return Format.LZO_BLOCK;
  }
private static Format determineFileFormat(InputSplit split,
                                            Configuration conf)
                                            throws IOException {
    FileSplit fileSplit = (FileSplit) split;

    Path file = fileSplit.getPath();

    /* we could have a an optional configuration that maps a regex on a
     * file name to a format. E.g. ".*-block.lzo" to LZO_BLOCK file.
     */

    // most of the cost is opening the file and
    // reading first lzo block (about 256k of uncompressed data)

    CompressionCodec codec = new CompressionCodecFactory(conf).getCodec(file);
    if (codec == null) {
      throw new IOException("No codec for file " + file + " found");
    }

    InputStream in = file.getFileSystem(conf).open(file);
    InputStream lzoIn = null;

    // check if the file starts with magic bytes for Block storage format.
    try {
      lzoIn = codec.createInputStream(in);

      for(byte magic : Protobufs.KNOWN_GOOD_POSITION_MARKER) {
        int b = lzoIn.read();
        if (b < 0 || (byte)b != magic) {
          return Format.LZO_B64LINE;
        }
      }
    } finally {
      IOUtils.closeStream(lzoIn);
      IOUtils.closeStream(in);
    }

    // the check passed
    return Format.LZO_BLOCK;
  }
private static Format determineFileFormat ( InputSplit split , Configuration conf ) throws IOException { FileSplit fileSplit = ( FileSplit ) split ; Path file = fileSplit . getPath ( ) ; /* we could have a an optional configuration that maps a regex on a
     * file name to a format. E.g. ".*-block.lzo" to LZO_BLOCK file.
     */ // most of the cost is opening the file and // reading first lzo block (about 256k of uncompressed data) CompressionCodec codec = new CompressionCodecFactory ( conf ) . getCodec ( file ) ; if ( codec == null ) { throw new IOException ( "No codec for file " + file + " found" ) ; } InputStream in = file . getFileSystem ( conf ) . open ( file ) ; InputStream lzoIn = null ; // check if the file starts with magic bytes for Block storage format. try { lzoIn = codec . createInputStream ( in ) ; for ( byte magic : Protobufs . KNOWN_GOOD_POSITION_MARKER ) { int b = lzoIn . read ( ) ; if ( b < 0 || ( byte ) b != magic ) { return Format . LZO_B64LINE ; } } } finally { IOUtils . closeStream ( lzoIn ) ; IOUtils . closeStream ( in ) ; } // the check passed return Format . LZO_BLOCK ; }
Checks to see if the input records are stored as SerializedBlock.
The block format starts with {@link Protobufs#KNOWN_GOOD_POSITION_MARKER}.
Otherwise the input is assumed to be Base64 encoded lines.
Checks to see if the input records are stored as SerializedBlock . The block format starts with {