Examples with FrameSpiller org.apache.asterix.external.feed.dataflow.FrameSpiller used on opensource projects

Example 1 with FrameSpiller

use of org.apache.asterix.external.feed.dataflow.FrameSpiller in project asterixdb by apache.

the class FeedSpillerUnitTest method testWriteFixedSizeSpill.

/*
     * Spiller spills each 1024 frames to a file
     */
/*
     * The following tests:
     * 1. Test writer only.
     * Write 1023 frames.
     * Check only 1 file exist.
     * Write 1 more frame
     * Check two files exist.
     * Insert 1023 more frames.
     * Check that we still have 2 files.
     * Write 1 more frame.
     * Check that we have 3 files.
     * Check that we have 2048 frames to read.
     * Close the spiller
     * Check files were deleted.
     */
@org.junit.Test
public void testWriteFixedSizeSpill() {
    try {
        removeSpillFiles();
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        FrameSpiller spiller = new FrameSpiller(ctx, TEST_DATAVERSE + "_" + TEST_FEED + "_" + TEST_DATASET, new Long(NUM_FRAMES * DEFAULT_FRAME_SIZE));
        spiller.open();
        VSizeFrame frame = new VSizeFrame(ctx);
        spiller.spill(frame.getBuffer());
        Assert.assertEquals(1, spiller.remaining());
        Assert.assertEquals(1, countSpillFiles());
        for (int i = 0; i < 1022; i++) {
            spiller.spill(frame.getBuffer());
        }
        Assert.assertEquals(1023, spiller.remaining());
        Assert.assertEquals(1, countSpillFiles());
        spiller.spill(frame.getBuffer());
        Assert.assertEquals(1024, spiller.remaining());
        Assert.assertEquals(2, countSpillFiles());
        for (int i = 0; i < 1023; i++) {
            spiller.spill(frame.getBuffer());
        }
        Assert.assertEquals(2047, spiller.remaining());
        Assert.assertEquals(2, countSpillFiles());
        spiller.spill(frame.getBuffer());
        Assert.assertEquals(2048, spiller.remaining());
        Assert.assertEquals(3, countSpillFiles());
        spiller.close();
        Assert.assertEquals(0, spiller.remaining());
        Assert.assertEquals(0, countSpillFiles());
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail(th.getMessage());
    }
}

Example 2 with FrameSpiller

use of org.apache.asterix.external.feed.dataflow.FrameSpiller in project asterixdb by apache.

the class FeedSpillerUnitTest method testWriteReadFixedSizeSpill.

/*
     * 2. Test writer and reader
     * Write 1047 and Read 1042 frames.
     * Check only 1 file exists.
     * Check switchToMemory() returns false.
     * Check remaining() returns 5.
     * Read the remaining 5 frames.
     * Check switchToMemory() returns true.
     * Check that the reader returns null
     * Close the spiller.
     * Check files were deleted.
     */
@org.junit.Test
public void testWriteReadFixedSizeSpill() {
    try {
        removeSpillFiles();
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        FrameSpiller spiller = new FrameSpiller(ctx, TEST_DATAVERSE + "_" + TEST_FEED + "_" + TEST_DATASET, new Long(NUM_FRAMES * DEFAULT_FRAME_SIZE));
        spiller.open();
        VSizeFrame frame = new VSizeFrame(ctx);
        for (int i = 0; i < 1047; i++) {
            spiller.spill(frame.getBuffer());
        }
        for (int i = 0; i < 1042; i++) {
            spiller.next();
        }
        Assert.assertEquals(5, spiller.remaining());
        Assert.assertEquals(1, countSpillFiles());
        Assert.assertEquals(false, spiller.switchToMemory());
        for (int i = 0; i < 4; i++) {
            spiller.next();
        }
        Assert.assertEquals(false, spiller.next() == null);
        Assert.assertEquals(true, spiller.next() == null);
        Assert.assertEquals(true, spiller.switchToMemory());
        Assert.assertEquals(1, countSpillFiles());
        spiller.close();
        Assert.assertEquals(0, spiller.remaining());
        Assert.assertEquals(0, countSpillFiles());
        Assert.assertEquals(true, spiller.switchToMemory());
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail(th.getMessage());
    }
}