Examples with IgfsFileRange org.apache.ignite.igfs.mapreduce.IgfsFileRange used on opensource projects

Example 1 with IgfsFileRange

use of org.apache.ignite.igfs.mapreduce.IgfsFileRange in project ignite by apache.

the class IgfsFixedLengthRecordResolver method resolveRecords.

/** {@inheritDoc} */
@Override
public IgfsFileRange resolveRecords(IgniteFileSystem fs, IgfsInputStream stream, IgfsFileRange suggestedRecord) throws IgniteException, IOException {
    long suggestedEnd = suggestedRecord.start() + suggestedRecord.length();
    long startRem = suggestedRecord.start() % recLen;
    long endRem = suggestedEnd % recLen;
    long start = Math.min(suggestedRecord.start() + (startRem != 0 ? (recLen - startRem) : 0), stream.length());
    long end = Math.min(suggestedEnd + (endRem != 0 ? (recLen - endRem) : 0), stream.length());
    assert end >= start;
    return start != end ? new IgfsFileRange(suggestedRecord.path(), start, end - start) : null;
}

Example 2 with IgfsFileRange

use of org.apache.ignite.igfs.mapreduce.IgfsFileRange in project ignite by apache.

the class IgfsByteDelimiterRecordResolverSelfTest method assertSplit.

/**
     * Check split resolution.
     *
     * @param suggestedStart Suggested start.
     * @param suggestedLen Suggested length.
     * @param expStart Expected start.
     * @param expLen Expected length.
     * @param data File data.
     * @param delims Delimiters.
     * @throws Exception If failed.
     */
public void assertSplit(long suggestedStart, long suggestedLen, long expStart, long expLen, byte[] data, byte[]... delims) throws Exception {
    write(data);
    IgfsByteDelimiterRecordResolver rslvr = resolver(delims);
    IgfsFileRange split;
    try (IgfsInputStream is = read()) {
        split = rslvr.resolveRecords(igfs, is, split(suggestedStart, suggestedLen));
    }
    assert split != null : "Split is null.";
    assert split.start() == expStart : "Incorrect start [expected=" + expStart + ", actual=" + split.start() + ']';
    assert split.length() == expLen : "Incorrect length [expected=" + expLen + ", actual=" + split.length() + ']';
}

Example 3 with IgfsFileRange

use of org.apache.ignite.igfs.mapreduce.IgfsFileRange in project ignite by apache.

the class IgfsByteDelimiterRecordResolverSelfTest method assertSplitNull.

/**
     * Check the split resolution resulted in {@code null}.
     *
     * @param suggestedStart Suggested start.
     * @param suggestedLen Suggested length.
     * @param data File data.
     * @param delims Delimiters.
     * @throws Exception If failed.
     */
public void assertSplitNull(long suggestedStart, long suggestedLen, byte[] data, byte[]... delims) throws Exception {
    write(data);
    IgfsByteDelimiterRecordResolver rslvr = resolver(delims);
    IgfsFileRange split;
    try (IgfsInputStream is = read()) {
        split = rslvr.resolveRecords(igfs, is, split(suggestedStart, suggestedLen));
    }
    assert split == null : "Split is not null.";
}

Example 4 with IgfsFileRange

use of org.apache.ignite.igfs.mapreduce.IgfsFileRange in project ignite by apache.

the class IgfsFixedLengthRecordResolverSelfTest method assertSplitNull.

/**
     * Check the split resolution resulted in {@code null}.
     *
     * @param suggestedStart Suggested start.
     * @param suggestedLen Suggested length.
     * @param data File data.
     * @param len Length.
     * @throws Exception If failed.
     */
public void assertSplitNull(long suggestedStart, long suggestedLen, byte[] data, int len) throws Exception {
    write(data);
    IgfsFixedLengthRecordResolver rslvr = resolver(len);
    IgfsFileRange split;
    try (IgfsInputStream is = read()) {
        split = rslvr.resolveRecords(igfs, is, split(suggestedStart, suggestedLen));
    }
    assert split == null : "Split is not null.";
}

Example 5 with IgfsFileRange

use of org.apache.ignite.igfs.mapreduce.IgfsFileRange in project ignite by apache.

the class IgfsStringDelimiterRecordResolverSelfTest method assertSplitNull.

/**
     * Check the split resolution resulted in {@code null}.
     *
     * @param suggestedStart Suggested start.
     * @param suggestedLen Suggested length.
     * @param data File data.
     * @param delims Delimiters.
     * @throws Exception If failed.
     */
public void assertSplitNull(long suggestedStart, long suggestedLen, byte[] data, String... delims) throws Exception {
    write(data);
    IgfsStringDelimiterRecordResolver rslvr = resolver(delims);
    IgfsFileRange split;
    try (IgfsInputStream is = read()) {
        split = rslvr.resolveRecords(igfs, is, split(suggestedStart, suggestedLen));
    }
    assert split == null : "Split is not null.";
}