use of org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler in project asterixdb by apache.
the class InputHandlerTest method testMemoryVarSizeFrameNoDiskNoDiscard.
/*
* Spill = false
* Discard = false
* VarSizeFrame
*/
@Test
public void testMemoryVarSizeFrameNoDiskNoDiscard() {
try {
Random random = new Random();
IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
// No spill, No discard
FeedPolicyAccessor fpa = createFeedPolicyAccessor(false, false, 0L, DISCARD_ALLOWANCE);
// Non-Active Writer
TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
writer.freeze();
// FramePool
ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);
FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
handler.open();
ByteBuffer buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE);
int multiplier = 1;
// add NUM_FRAMES times
while ((multiplier <= framePool.remaining())) {
handler.nextFrame(buffer);
multiplier = random.nextInt(10) + 1;
buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * multiplier);
}
// we can't satisfy the next request
// Next call should block we will do it in a different thread
Future<?> result = EXECUTOR.submit(new Pusher(buffer, handler));
// Check that the nextFrame didn't return
if (result.isDone()) {
Assert.fail();
}
// Check that no records were discarded
Assert.assertEquals(handler.getNumDiscarded(), 0);
// Check that no records were spilled
Assert.assertEquals(handler.getNumSpilled(), 0);
// Check that number of stalled is not greater than 1
Assert.assertTrue(handler.getNumStalled() <= 1);
writer.unfreeze();
handler.close();
result.get();
} catch (Throwable th) {
th.printStackTrace();
Assert.fail();
}
Assert.assertNull(cause);
}
use of org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler in project asterixdb by apache.
the class InputHandlerTest method testMemoryFixedSizeFrameNoDiskNoDiscardSlowConsumer.
/*
* Spill = false;
* Discard = false;
* Fixed size frames
* Slow next operator
*/
@Test
public void testMemoryFixedSizeFrameNoDiskNoDiscardSlowConsumer() {
try {
int numRounds = 10;
IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
// No spill, No discard
FeedPolicyAccessor fpa = createFeedPolicyAccessor(false, false, 0L, DISCARD_ALLOWANCE);
// Non-Active Writer
TestFrameWriter writer = FrameWriterTestUtils.create(Collections.emptyList(), Collections.emptyList(), false);
// FramePool
ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);
FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
handler.open();
VSizeFrame frame = new VSizeFrame(ctx);
writer.setNextDuration(1);
// add NUM_FRAMES times
for (int i = 0; i < NUM_FRAMES * numRounds; i++) {
handler.nextFrame(frame.getBuffer());
}
// Check that no records were discarded
Assert.assertEquals(handler.getNumDiscarded(), 0);
// Check that no records were spilled
Assert.assertEquals(handler.getNumSpilled(), 0);
// Check that nextFrame was called
writer.validate(false);
handler.close();
Assert.assertEquals(writer.nextFrameCount(), (NUM_FRAMES * numRounds));
writer.validate(true);
} catch (Throwable th) {
th.printStackTrace();
Assert.fail();
}
Assert.assertNull(cause);
}
use of org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler in project asterixdb by apache.
the class InputHandlerTest method testZeroMemoryVarSizeFrameWithDiskNoDiscard.
@org.junit.Test
public void testZeroMemoryVarSizeFrameWithDiskNoDiscard() {
try {
int numRounds = 5;
Random random = new Random();
IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
// No spill, No discard
FeedPolicyAccessor fpa = createFeedPolicyAccessor(true, false, NUM_FRAMES * DEFAULT_FRAME_SIZE, DISCARD_ALLOWANCE);
// Non-Active Writer
TestFrameWriter writer = FrameWriterTestUtils.create(Collections.emptyList(), Collections.emptyList(), false);
// FramePool
ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, 0, DEFAULT_FRAME_SIZE);
FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
handler.open();
ByteBuffer buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE);
handler.nextFrame(buffer);
Assert.assertEquals(0, handler.getNumProcessedInMemory());
Assert.assertEquals(1, handler.getNumSpilled());
// add NUM_FRAMES times
for (int i = 0; i < NUM_FRAMES * numRounds; i++) {
int multiplier = random.nextInt(10) + 1;
buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * multiplier);
handler.nextFrame(buffer);
}
// Check that no records were discarded
Assert.assertEquals(handler.getNumDiscarded(), 0);
// Check that no records were spilled
Assert.assertEquals(NUM_FRAMES * numRounds + 1, handler.getNumSpilled());
writer.validate(false);
handler.close();
// Check that nextFrame was called
Assert.assertEquals(NUM_FRAMES * numRounds + 1, writer.nextFrameCount());
writer.validate(true);
} catch (Throwable th) {
th.printStackTrace();
Assert.fail();
} finally {
Assert.assertNull(cause);
}
}
use of org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler in project asterixdb by apache.
the class InputHandlerTest method testMemoryVarSizeFrameWithSpillWithDiscard.
/*
* Spill = false;
* Discard = true; discard only 5%
* Fixed size frames
*/
@Test
public void testMemoryVarSizeFrameWithSpillWithDiscard() {
try {
int numberOfMemoryFrames = 50;
int numberOfSpillFrames = 50;
int notDiscarded = 0;
int totalMinFrames = 0;
IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
// Spill budget = Memory budget, No discard
FeedPolicyAccessor fpa = createFeedPolicyAccessor(true, true, DEFAULT_FRAME_SIZE * numberOfSpillFrames, DISCARD_ALLOWANCE);
// Non-Active Writer
TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
writer.freeze();
// FramePool
ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, numberOfMemoryFrames * DEFAULT_FRAME_SIZE, DEFAULT_FRAME_SIZE);
FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
handler.open();
ByteBuffer buffer1 = ByteBuffer.allocate(DEFAULT_FRAME_SIZE);
ByteBuffer buffer2 = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * 2);
ByteBuffer buffer3 = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * 3);
ByteBuffer buffer4 = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * 4);
ByteBuffer buffer5 = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * 5);
while (true) {
if (totalMinFrames + 1 < numberOfMemoryFrames) {
handler.nextFrame(buffer1);
notDiscarded++;
totalMinFrames++;
} else {
break;
}
if (totalMinFrames + 2 < numberOfMemoryFrames) {
notDiscarded++;
totalMinFrames += 2;
handler.nextFrame(buffer2);
} else {
break;
}
if (totalMinFrames + 3 < numberOfMemoryFrames) {
notDiscarded++;
totalMinFrames += 3;
handler.nextFrame(buffer3);
} else {
break;
}
}
// Now we need to verify that the frame pool memory has been consumed!
Assert.assertTrue(framePool.remaining() < 3);
Assert.assertEquals(0, handler.getNumSpilled());
Assert.assertEquals(0, handler.getNumStalled());
Assert.assertEquals(0, handler.getNumDiscarded());
while (true) {
if (handler.getNumSpilled() < numberOfSpillFrames) {
notDiscarded++;
handler.nextFrame(buffer3);
} else {
break;
}
if (handler.getNumSpilled() < numberOfSpillFrames) {
notDiscarded++;
handler.nextFrame(buffer4);
} else {
break;
}
if (handler.getNumSpilled() < numberOfSpillFrames) {
notDiscarded++;
handler.nextFrame(buffer5);
} else {
break;
}
}
Assert.assertTrue(framePool.remaining() < 3);
Assert.assertEquals(handler.framesOnDisk(), handler.getNumSpilled());
Assert.assertEquals(handler.framesOnDisk(), numberOfSpillFrames);
Assert.assertEquals(0, handler.getNumStalled());
Assert.assertEquals(0, handler.getNumDiscarded());
// We can only discard one frame
double numDiscarded = 0;
boolean nextShouldDiscard = ((numDiscarded + 1.0) / (handler.getTotal() + 1.0)) <= fpa.getMaxFractionDiscard();
while (nextShouldDiscard) {
handler.nextFrame(buffer5);
numDiscarded++;
nextShouldDiscard = ((numDiscarded + 1.0) / (handler.getTotal() + 1.0)) <= fpa.getMaxFractionDiscard();
}
Assert.assertTrue(framePool.remaining() < 3);
Assert.assertEquals(handler.framesOnDisk(), handler.getNumSpilled());
Assert.assertEquals(0, handler.getNumStalled());
Assert.assertEquals((int) numDiscarded, handler.getNumDiscarded());
// Next Call should block since we're exceeding the discard allowance
Future<?> result = EXECUTOR.submit(new Pusher(buffer5, handler));
if (result.isDone()) {
Assert.fail("The producer should switch to stall mode since it is exceeding the discard allowance");
}
// consume memory frames
writer.unfreeze();
result.get();
handler.close();
Assert.assertEquals(writer.nextFrameCount(), notDiscarded + 1);
} catch (Throwable th) {
th.printStackTrace();
Assert.fail();
}
Assert.assertNull(cause);
}
use of org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler in project asterixdb by apache.
the class InputHandlerTest method testMemoryFixedSizeFrameWithSpillNoDiscard.
/*
* Spill = true;
* Discard = false;
* Fixed size frames
*/
@Test
public void testMemoryFixedSizeFrameWithSpillNoDiscard() {
try {
IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
// Spill budget = Memory budget, No discard
FeedPolicyAccessor fpa = createFeedPolicyAccessor(true, false, DEFAULT_FRAME_SIZE * NUM_FRAMES, DISCARD_ALLOWANCE);
// Non-Active Writer
TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
writer.freeze();
// FramePool
ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);
FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
handler.open();
VSizeFrame frame = new VSizeFrame(ctx);
// add NUM_FRAMES times
for (int i = 0; i < NUM_FRAMES; i++) {
handler.nextFrame(frame.getBuffer());
}
// Next call should NOT block. we will do it in a different thread
Future<?> result = EXECUTOR.submit(new Pusher(frame.getBuffer(), handler));
result.get();
// Check that no records were discarded
Assert.assertEquals(handler.getNumDiscarded(), 0);
// Check that one frame is spilled
Assert.assertEquals(handler.getNumSpilled(), 1);
// consume memory frames
writer.unfreeze();
handler.close();
Assert.assertEquals(handler.framesOnDisk(), 0);
// exit
} catch (Throwable th) {
th.printStackTrace();
Assert.fail();
}
Assert.assertNull(cause);
}
Aggregations