packages/chunked-file-reader/lib/index.js
function getChunk(readerClient, slice, opConfig, logger, metadata) {
    const delimiter = opConfig.line_delimiter;

    async function getMargin(offset, length) {
        let margin = '';
        return new Promise(async (resolve) => {
            while (margin.indexOf(delimiter) === -1) {
                // reader clients must return false-y when nothing more to read.
                const chunk = await readerClient(offset, length); // eslint-disable-line no-await-in-loop, max-len
                if (!chunk) {
                    resolve(margin.split(delimiter)[0]);
                    return;
                }
                margin += chunk;
                offset += length; // eslint-disable-line no-param-reassign, max-len
            }
            // Don't read too far - next slice will get it.
            resolve(margin.split(delimiter)[0]);
        });
    }

    let needMargin = false;
    if (slice.length) {
        // Determines whether or not to grab the extra margin.
        if (slice.offset + slice.length !== slice.total) {
            needMargin = true;
        }
    }

    return readerClient(slice.offset, slice.length)
        .then(async (data) => {
            if (data.endsWith(delimiter)) {
                // Skip the margin if the raw data ends with the delimiter since
                // it will end with a complete record.
                needMargin = false;
            }
            if (needMargin) {
                // Want to minimize reads since will typically be over the
                // network. Using twice the average record size as a heuristic.
                const avgSize = _averageRecordSize(data.split(delimiter));
                data += await getMargin(slice.offset + slice.length, 2 * avgSize); // eslint-disable-line no-param-reassign, max-len
            }
            return data;
        })
        .then(data => chunkFormatter[opConfig.format](data, logger, opConfig, metadata, slice));
}
function getChunk(readerClient, slice, opConfig, logger, metadata) {
    const delimiter = opConfig.line_delimiter;

    async function getMargin(offset, length) {
        let margin = '';
        return new Promise(async (resolve) => {
            while (margin.indexOf(delimiter) === -1) {
                // reader clients must return false-y when nothing more to read.
                const chunk = await readerClient(offset, length); // eslint-disable-line no-await-in-loop, max-len
                if (!chunk) {
                    resolve(margin.split(delimiter)[0]);
                    return;
                }
                margin += chunk;
                offset += length; // eslint-disable-line no-param-reassign, max-len
            }
            // Don't read too far - next slice will get it.
            resolve(margin.split(delimiter)[0]);
        });
    }

    let needMargin = false;
    if (slice.length) {
        // Determines whether or not to grab the extra margin.
        if (slice.offset + slice.length !== slice.total) {
            needMargin = true;
        }
    }

    return readerClient(slice.offset, slice.length)
        .then(async (data) => {
            if (data.endsWith(delimiter)) {
                // Skip the margin if the raw data ends with the delimiter since
                // it will end with a complete record.
                needMargin = false;
            }
            if (needMargin) {
                // Want to minimize reads since will typically be over the
                // network. Using twice the average record size as a heuristic.
                const avgSize = _averageRecordSize(data.split(delimiter));
                data += await getMargin(slice.offset + slice.length, 2 * avgSize); // eslint-disable-line no-param-reassign, max-len
            }
            return data;
        })
        .then(data => chunkFormatter[opConfig.format](data, logger, opConfig, metadata, slice));
}
function getChunk ( readerClient , slice , opConfig , logger , metadata ) { const delimiter = opConfig . line_delimiter ; async function getMargin ( offset , length ) { let margin = '' ; return new Promise ( async ( resolve ) => { while ( margin . indexOf ( delimiter ) === - 1 ) { // reader clients must return false-y when nothing more to read. const chunk = await readerClient ( offset , length ) ; // eslint-disable-line no-await-in-loop, max-len if ( ! chunk ) { resolve ( margin . split ( delimiter ) [ 0 ] ) ; return ; } margin += chunk ; offset += length ; // eslint-disable-line no-param-reassign, max-len } // Don't read too far - next slice will get it. resolve ( margin . split ( delimiter ) [ 0 ] ) ; } ) ; } let needMargin = false ; if ( slice . length ) { // Determines whether or not to grab the extra margin. if ( slice . offset + slice . length !== slice . total ) { needMargin = true ; } } return readerClient ( slice . offset , slice . length ) . then ( async ( data ) => { if ( data . endsWith ( delimiter ) ) { // Skip the margin if the raw data ends with the delimiter since // it will end with a complete record. needMargin = false ; } if ( needMargin ) { // Want to minimize reads since will typically be over the // network. Using twice the average record size as a heuristic. const avgSize = _averageRecordSize ( data . split ( delimiter ) ) ; data += await getMargin ( slice . offset + slice . length , 2 * avgSize ) ; // eslint-disable-line no-param-reassign, max-len } return data ; } ) . then ( data => chunkFormatter [ opConfig . format ] ( data , logger , opConfig , metadata , slice ) ) ; }
This function will grab the chunk of data specified by the slice plus an extra margin if the slice does not end with the delimiter.
This function will grab the chunk of data specified by the slice plus an extra margin if the slice does not end with the delimiter .