use of org.apache.flink.api.common.typeinfo.BasicTypeInfo.STRING_TYPE_INFO in project flink by apache.
the class StreamTaskFinalCheckpointsTest method doTestTriggerStopWithSavepointWhenWaitingForFinalCheckpointOnSourceTask.
private void doTestTriggerStopWithSavepointWhenWaitingForFinalCheckpointOnSourceTask(boolean drain) throws Exception {
int finalCheckpointId = 6;
int syncSavepointId = 7;
CompletingCheckpointResponder checkpointResponder = new CompletingCheckpointResponder() {
@Override
public void acknowledgeCheckpoint(JobID jobID, ExecutionAttemptID executionAttemptID, long checkpointId, CheckpointMetrics checkpointMetrics, TaskStateSnapshot subtaskState) {
if (syncSavepointId == checkpointId) {
// complete the final checkpoint when sync savepoint acknowledged
// we should wait for the sync savepoint to complete
super.acknowledgeCheckpoint(jobID, executionAttemptID, finalCheckpointId, checkpointMetrics, subtaskState);
try {
// Give some potential time for the task to finish before the
// savepoint is notified complete
Thread.sleep(CONCURRENT_EVENT_WAIT_PERIOD_MS);
} catch (InterruptedException e) {
throw new FlinkRuntimeException(e);
}
super.acknowledgeCheckpoint(jobID, executionAttemptID, syncSavepointId, checkpointMetrics, subtaskState);
}
}
};
try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(SourceStreamTask::new, STRING_TYPE_INFO).modifyStreamConfig(config -> {
config.setCheckpointingEnabled(true);
}).setCheckpointResponder(checkpointResponder).setupOutputForSingletonOperatorChain(new StreamSource<>(new ImmediatelyFinishingSource())).build()) {
checkpointResponder.setHandlers(testHarness.streamTask::notifyCheckpointCompleteAsync, testHarness.streamTask::notifyCheckpointAbortAsync);
// Tests triggering checkpoint after received all the inputs have received
// EndOfPartition.
// start task thread
testHarness.streamTask.runMailboxLoop();
// trigger the final checkpoint
CompletableFuture<Boolean> checkpointFuture = triggerCheckpoint(testHarness, finalCheckpointId);
// trigger the synchronous savepoint
CompletableFuture<Boolean> savepointFuture = drain ? triggerStopWithSavepointDrain(testHarness, syncSavepointId) : triggerStopWithSavepointNoDrain(testHarness, syncSavepointId);
// The checkpoint 6 would be triggered successfully.
testHarness.finishProcessing();
assertTrue(checkpointFuture.isDone());
assertTrue(savepointFuture.isDone());
testHarness.getTaskStateManager().getWaitForReportLatch().await();
assertEquals(syncSavepointId, testHarness.getTaskStateManager().getReportedCheckpointId());
assertEquals(syncSavepointId, testHarness.getTaskStateManager().getNotifiedCompletedCheckpointId());
}
}
use of org.apache.flink.api.common.typeinfo.BasicTypeInfo.STRING_TYPE_INFO in project flink by apache.
the class StreamTaskFinalCheckpointsTest method testNotWaitingForAllRecordsProcessedIfCheckpointNotEnabled.
@Test
public void testNotWaitingForAllRecordsProcessedIfCheckpointNotEnabled() throws Exception {
ResultPartitionWriter[] partitionWriters = new ResultPartitionWriter[2];
try {
for (int i = 0; i < partitionWriters.length; ++i) {
partitionWriters[i] = PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED_BOUNDED);
partitionWriters[i].setup();
}
try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, STRING_TYPE_INFO).modifyStreamConfig(config -> config.setCheckpointingEnabled(false)).addInput(STRING_TYPE_INFO).addAdditionalOutput(partitionWriters).setupOperatorChain(new EmptyOperator()).finishForSingletonOperatorChain(StringSerializer.INSTANCE).build()) {
testHarness.endInput();
// In this case the result partition should not emit EndOfUserRecordsEvent.
for (ResultPartitionWriter writer : partitionWriters) {
assertEquals(0, ((PipelinedResultPartition) writer).getNumberOfQueuedBuffers());
}
}
} finally {
for (ResultPartitionWriter writer : partitionWriters) {
if (writer != null) {
writer.close();
}
}
}
}
use of org.apache.flink.api.common.typeinfo.BasicTypeInfo.STRING_TYPE_INFO in project flink by apache.
the class StreamTaskTest method testBufferSizeRecalculationStartSuccessfully.
@Test
public void testBufferSizeRecalculationStartSuccessfully() throws Exception {
int expectedThroughput = 13333;
int inputChannels = 3;
// debloat period doesn't matter, we will schedule debloating manually
Configuration config = new Configuration().set(BUFFER_DEBLOAT_PERIOD, Duration.ofHours(10)).set(BUFFER_DEBLOAT_TARGET, Duration.ofSeconds(1)).set(BUFFER_DEBLOAT_THRESHOLD_PERCENTAGES, 1).set(BUFFER_DEBLOAT_ENABLED, true);
try (StreamTaskMailboxTestHarness<String> harness = new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, STRING_TYPE_INFO).setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(config)).addInput(STRING_TYPE_INFO, inputChannels).addInput(STRING_TYPE_INFO, inputChannels).modifyGateBuilder(gateBuilder -> gateBuilder.setThroughputCalculator(bufferDebloatConfiguration -> new ThroughputCalculator(SystemClock.getInstance()) {
@Override
public long calculateThroughput() {
return expectedThroughput;
}
})).setupOutputForSingletonOperatorChain(new TestBoundedOneInputStreamOperator()).build()) {
harness.processAll();
harness.streamTask.debloat();
long lastBufferSize = -1;
for (InputGate inputGate : harness.streamTask.getEnvironment().getAllInputGates()) {
for (int i = 0; i < inputGate.getNumberOfInputChannels(); i++) {
long currentBufferSize = ((TestInputChannel) inputGate.getChannel(i)).getCurrentBufferSize();
assertThat(currentBufferSize, lessThan(MEMORY_SEGMENT_SIZE.defaultValue().getBytes()));
assertThat(currentBufferSize, greaterThan(0L));
if (lastBufferSize > 0) {
assertThat(lastBufferSize, is(currentBufferSize));
}
lastBufferSize = currentBufferSize;
}
}
}
}
use of org.apache.flink.api.common.typeinfo.BasicTypeInfo.STRING_TYPE_INFO in project flink by apache.
the class StreamTaskTest method testBufferDebloatingMultiGates.
@Test
public void testBufferDebloatingMultiGates() throws Exception {
// debloat period doesn't matter, we will schedule debloating manually
Configuration config = new Configuration().set(BUFFER_DEBLOAT_PERIOD, Duration.ofHours(10)).set(BUFFER_DEBLOAT_TARGET, Duration.ofSeconds(1)).set(BUFFER_DEBLOAT_ENABLED, true).set(BUFFER_DEBLOAT_THRESHOLD_PERCENTAGES, // disable the threshold to achieve exact buffer sizes
0);
final long throughputGate1 = 1024L;
final long throughputGate2 = 60 * 1024L;
final int inputChannelsGate1 = 1;
final int inputChannelsGate2 = 4;
final ThroughputCalculator throughputCalculator = new ThroughputCalculator(SystemClock.getInstance()) {
/* parameters are ignored */
private int callCount = 0;
@Override
public long calculateThroughput() {
if (callCount++ % 2 == 0) {
return throughputGate1;
} else {
return throughputGate2;
}
}
};
try (StreamTaskMailboxTestHarness<String> harness = new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, STRING_TYPE_INFO).setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(config)).addInput(STRING_TYPE_INFO, inputChannelsGate1).addInput(STRING_TYPE_INFO, inputChannelsGate2).modifyGateBuilder(gateBuilder -> gateBuilder.setThroughputCalculator(bufferDebloatConfiguration -> throughputCalculator)).setupOutputForSingletonOperatorChain(new TestBoundedOneInputStreamOperator()).build()) {
final IndexedInputGate[] inputGates = harness.streamTask.getEnvironment().getAllInputGates();
harness.processAll();
// call debloating until the EMA reaches the target buffer size
while (getCurrentBufferSize(inputGates[0]) == 0 || getCurrentBufferSize(inputGates[0]) > throughputGate1) {
harness.streamTask.debloat();
}
assertThat(getCurrentBufferSize(inputGates[0]), equalTo((int) throughputGate1));
assertThat(getCurrentBufferSize(inputGates[1]), equalTo((int) throughputGate2 / inputChannelsGate2));
}
}
Aggregations