Examples with IgfsFileRange org.apache.ignite.igfs.mapreduce.IgfsFileRange used on opensource projects

Example 6 with IgfsFileRange

use of org.apache.ignite.igfs.mapreduce.IgfsFileRange in project ignite by apache.

the class IgfsStringDelimiterRecordResolverSelfTest method assertSplit.

/**
 * Check split resolution.
 *
 * @param suggestedStart Suggested start.
 * @param suggestedLen Suggested length.
 * @param expStart Expected start.
 * @param expLen Expected length.
 * @param data File data.
 * @param delims Delimiters.
 * @throws Exception If failed.
 */
public void assertSplit(long suggestedStart, long suggestedLen, long expStart, long expLen, byte[] data, String... delims) throws Exception {
    write(data);
    IgfsByteDelimiterRecordResolver rslvr = resolver(delims);
    IgfsFileRange split;
    try (IgfsInputStream is = read()) {
        split = rslvr.resolveRecords(igfs, is, split(suggestedStart, suggestedLen));
    }
    assert split != null : "Split is null.";
    assert split.start() == expStart : "Incorrect start [expected=" + expStart + ", actual=" + split.start() + ']';
    assert split.length() == expLen : "Incorrect length [expected=" + expLen + ", actual=" + split.length() + ']';
}

Example 7 with IgfsFileRange

use of org.apache.ignite.igfs.mapreduce.IgfsFileRange in project ignite by apache.

the class IgfsJobImpl method execute.

/**
 * {@inheritDoc}
 */
@Override
public Object execute() {
    IgniteFileSystem fs = ignite.fileSystem(igfsName);
    try (IgfsInputStream in = fs.open(path)) {
        IgfsFileRange split = new IgfsFileRange(path, start, len);
        if (rslvr != null) {
            split = rslvr.resolveRecords(fs, in, split);
            if (split == null) {
                log.warning("No data found for split on local node after resolver is applied " + "[igfsName=" + igfsName + ", path=" + path + ", start=" + start + ", len=" + len + ']');
                return null;
            }
        }
        in.seek(split.start());
        return job.execute(fs, new IgfsFileRange(path, split.start(), split.length()), in);
    } catch (IOException e) {
        throw new IgniteException("Failed to execute IGFS job for file split [igfsName=" + igfsName + ", path=" + path + ", start=" + start + ", len=" + len + ']', e);
    }
}

Example 8 with IgfsFileRange

use of org.apache.ignite.igfs.mapreduce.IgfsFileRange in project ignite by apache.

the class IgfsFixedLengthRecordResolverSelfTest method assertSplit.

/**
 * Check split resolution.
 *
 * @param suggestedStart Suggested start.
 * @param suggestedLen Suggested length.
 * @param expStart Expected start.
 * @param expLen Expected length.
 * @param data File data.
 * @param len Length.
 * @throws Exception If failed.
 */
public void assertSplit(long suggestedStart, long suggestedLen, long expStart, long expLen, byte[] data, int len) throws Exception {
    write(data);
    IgfsFixedLengthRecordResolver rslvr = resolver(len);
    IgfsFileRange split;
    try (IgfsInputStream is = read()) {
        split = rslvr.resolveRecords(igfs, is, split(suggestedStart, suggestedLen));
    }
    assert split != null : "Split is null.";
    assert split.start() == expStart : "Incorrect start [expected=" + expStart + ", actual=" + split.start() + ']';
    assert split.length() == expLen : "Incorrect length [expected=" + expLen + ", actual=" + split.length() + ']';
}

Example 9 with IgfsFileRange

use of org.apache.ignite.igfs.mapreduce.IgfsFileRange in project ignite by apache.

the class IgfsNewLineDelimiterRecordResolverSelfTest method assertSplit.

/**
 * Check split resolution.
 *
 * @param suggestedStart Suggested start.
 * @param suggestedLen Suggested length.
 * @param expStart Expected start.
 * @param expLen Expected length.
 * @param data File data.
 * @throws Exception If failed.
 */
public void assertSplit(long suggestedStart, long suggestedLen, long expStart, long expLen, byte[] data) throws Exception {
    write(data);
    IgfsNewLineRecordResolver rslvr = resolver();
    IgfsFileRange split;
    try (IgfsInputStream is = read()) {
        split = rslvr.resolveRecords(igfs, is, split(suggestedStart, suggestedLen));
    }
    assert split != null : "Split is null.";
    assert split.start() == expStart : "Incorrect start [expected=" + expStart + ", actual=" + split.start() + ']';
    assert split.length() == expLen : "Incorrect length [expected=" + expLen + ", actual=" + split.length() + ']';
}

Example 10 with IgfsFileRange

use of org.apache.ignite.igfs.mapreduce.IgfsFileRange in project ignite by apache.

the class IgfsNewLineDelimiterRecordResolverSelfTest method assertSplitNull.

/**
 * Check the split resolution resulted in {@code null}.
 *
 * @param suggestedStart Suggested start.
 * @param suggestedLen Suggested length.
 * @param data File data.
 * @throws Exception If failed.
 */
public void assertSplitNull(long suggestedStart, long suggestedLen, byte[] data) throws Exception {
    write(data);
    IgfsNewLineRecordResolver rslvr = resolver();
    IgfsFileRange split;
    try (IgfsInputStream is = read()) {
        split = rslvr.resolveRecords(igfs, is, split(suggestedStart, suggestedLen));
    }
    assert split == null : "Split is not null.";
}