use of org.apache.flink.api.java.tuple.Tuple6 in project flink by apache.
the class CsvInputFormatTest method testEmptyFields.
@Test
public void testEmptyFields() throws IOException {
try {
final String fileContent = "|0|0|0|0|0|\n" + "1||1|1|1|1|\n" + "2|2||2|2|2|\n" + "3|3|3| |3|3|\n" + "4|4|4|4||4|\n" + "5|5|5|5|5||\n";
final FileInputSplit split = createTempFile(fileContent);
final TupleTypeInfo<Tuple6<Short, Integer, Long, Float, Double, Byte>> typeInfo = TupleTypeInfo.getBasicTupleTypeInfo(Short.class, Integer.class, Long.class, Float.class, Double.class, Byte.class);
final CsvInputFormat<Tuple6<Short, Integer, Long, Float, Double, Byte>> format = new TupleCsvInputFormat<Tuple6<Short, Integer, Long, Float, Double, Byte>>(PATH, typeInfo);
format.setFieldDelimiter("|");
format.configure(new Configuration());
format.open(split);
Tuple6<Short, Integer, Long, Float, Double, Byte> result = new Tuple6<Short, Integer, Long, Float, Double, Byte>();
try {
result = format.nextRecord(result);
fail("Empty String Parse Exception was not thrown! (ShortParser)");
} catch (ParseException e) {
}
try {
result = format.nextRecord(result);
fail("Empty String Parse Exception was not thrown! (IntegerParser)");
} catch (ParseException e) {
}
try {
result = format.nextRecord(result);
fail("Empty String Parse Exception was not thrown! (LongParser)");
} catch (ParseException e) {
}
try {
result = format.nextRecord(result);
fail("Empty String Parse Exception was not thrown! (FloatParser)");
} catch (ParseException e) {
}
try {
result = format.nextRecord(result);
fail("Empty String Parse Exception was not thrown! (DoubleParser)");
} catch (ParseException e) {
}
try {
result = format.nextRecord(result);
fail("Empty String Parse Exception was not thrown! (ByteParser)");
} catch (ParseException e) {
}
result = format.nextRecord(result);
assertNull(result);
assertTrue(format.reachedEnd());
} catch (Exception ex) {
fail("Test failed due to a " + ex.getClass().getName() + ": " + ex.getMessage());
}
}
use of org.apache.flink.api.java.tuple.Tuple6 in project flink by apache.
the class AbstractFineGrainedSlotManagerITCase method testRequirementDeclaration.
private void testRequirementDeclaration(RequirementDeclarationScenario scenario) throws Exception {
final ResourceID resourceID = ResourceID.generate();
final JobID jobId = new JobID();
final SlotID slotId = SlotID.getDynamicSlotID(resourceID);
final String targetAddress = "localhost";
final ResourceRequirements requirements = ResourceRequirements.create(jobId, targetAddress, Collections.singleton(ResourceRequirement.create(DEFAULT_SLOT_RESOURCE_PROFILE, 1)));
final CompletableFuture<Tuple6<SlotID, JobID, AllocationID, ResourceProfile, String, ResourceManagerId>> requestFuture = new CompletableFuture<>();
// accept an incoming slot request
final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> {
requestFuture.complete(tuple6);
return CompletableFuture.completedFuture(Acknowledge.get());
}).createTestingTaskExecutorGateway();
final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(resourceID, taskExecutorGateway);
new Context() {
{
runTest(() -> {
if (scenario == RequirementDeclarationScenario.TASK_EXECUTOR_REGISTRATION_BEFORE_REQUIREMENT_DECLARATION) {
runInMainThread(() -> getSlotManager().registerTaskManager(taskExecutorConnection, new SlotReport(), DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE));
}
runInMainThread(() -> getSlotManager().processResourceRequirements(requirements));
if (scenario == RequirementDeclarationScenario.TASK_EXECUTOR_REGISTRATION_AFTER_REQUIREMENT_DECLARATION) {
runInMainThread(() -> getSlotManager().registerTaskManager(taskExecutorConnection, new SlotReport(), DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE));
}
assertThat(assertFutureCompleteAndReturn(requestFuture), is(equalTo(Tuple6.of(slotId, jobId, assertFutureCompleteAndReturn(requestFuture).f2, DEFAULT_SLOT_RESOURCE_PROFILE, targetAddress, getResourceManagerId()))));
final TaskManagerSlotInformation slot = getTaskManagerTracker().getAllocatedOrPendingSlot(assertFutureCompleteAndReturn(requestFuture).f2).get();
assertEquals("The slot has not been allocated to the expected allocation id.", assertFutureCompleteAndReturn(requestFuture).f2, slot.getAllocationId());
});
}
};
}
use of org.apache.flink.api.java.tuple.Tuple6 in project flink by apache.
the class AbstractFineGrainedSlotManagerITCase method testResourceCanBeAllocatedForDifferentJobAfterFree.
/**
* Tests that a resource allocated for one job can be allocated for another job after being
* freed.
*/
private void testResourceCanBeAllocatedForDifferentJobAfterFree(SecondRequirementDeclarationTime secondRequirementDeclarationTime) throws Exception {
final CompletableFuture<AllocationID> allocationIdFuture1 = new CompletableFuture<>();
final CompletableFuture<AllocationID> allocationIdFuture2 = new CompletableFuture<>();
final ResourceRequirements resourceRequirements1 = createResourceRequirementsForSingleSlot();
final ResourceRequirements resourceRequirements2 = createResourceRequirementsForSingleSlot();
final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> {
if (!allocationIdFuture1.isDone()) {
allocationIdFuture1.complete(tuple6.f2);
} else {
allocationIdFuture2.complete(tuple6.f2);
}
return CompletableFuture.completedFuture(Acknowledge.get());
}).createTestingTaskExecutorGateway();
final ResourceID resourceID = ResourceID.generate();
final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(resourceID, taskExecutorGateway);
final SlotReport slotReport = new SlotReport();
new Context() {
{
runTest(() -> {
runInMainThread(() -> {
getSlotManager().registerTaskManager(taskManagerConnection, slotReport, DEFAULT_SLOT_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE);
getSlotManager().processResourceRequirements(resourceRequirements1);
});
final AllocationID allocationId1 = assertFutureCompleteAndReturn(allocationIdFuture1);
TaskManagerSlotInformation slot = getTaskManagerTracker().getAllocatedOrPendingSlot(allocationId1).get();
assertEquals("The slot has not been allocated to the expected job id.", resourceRequirements1.getJobId(), slot.getJobId());
if (secondRequirementDeclarationTime == SecondRequirementDeclarationTime.BEFORE_FREE) {
runInMainThread(() -> getSlotManager().processResourceRequirements(resourceRequirements2));
}
// clear resource requirements first so that the freed slot isn't
// immediately re-assigned to the job
runInMainThread(() -> {
getSlotManager().processResourceRequirements(ResourceRequirements.create(resourceRequirements1.getJobId(), resourceRequirements1.getTargetAddress(), Collections.emptyList()));
getSlotManager().freeSlot(SlotID.getDynamicSlotID(resourceID), allocationId1);
});
if (secondRequirementDeclarationTime == SecondRequirementDeclarationTime.AFTER_FREE) {
runInMainThread(() -> getSlotManager().processResourceRequirements(resourceRequirements2));
}
slot = getTaskManagerTracker().getAllocatedOrPendingSlot(assertFutureCompleteAndReturn(allocationIdFuture2)).get();
assertEquals("The slot has not been allocated to the expected job id.", resourceRequirements2.getJobId(), slot.getJobId());
});
}
};
}
use of org.apache.flink.api.java.tuple.Tuple6 in project flink by apache.
the class AbstractFineGrainedSlotManagerITCase method testAllocationUpdatesIgnoredIfSlotMarkedAsAllocatedAfterSlotReport.
@Test
public void testAllocationUpdatesIgnoredIfSlotMarkedAsAllocatedAfterSlotReport() throws Exception {
final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> {
allocationIdFuture.complete(tuple6.f2);
return CompletableFuture.completedFuture(Acknowledge.get());
}).createTestingTaskExecutorGateway();
// The fatal error handler will exit the system if there is any exceptions in handling the
// ack of request slot. We need the security manager to verify that would not happen.
final SystemExitTrackingSecurityManager trackingSecurityManager = new SystemExitTrackingSecurityManager();
System.setSecurityManager(trackingSecurityManager);
final TaskExecutorConnection taskExecutionConnection = new TaskExecutorConnection(ResourceID.generate(), taskExecutorGateway);
final SlotReport slotReport = new SlotReport();
new Context() {
{
runTest(() -> {
runInMainThread(() -> {
getSlotManager().processResourceRequirements(createResourceRequirements(new JobID(), 1));
getSlotManager().registerTaskManager(taskExecutionConnection, slotReport, DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE);
});
final AllocationID allocationId = assertFutureCompleteAndReturn(allocationIdFuture);
runInMainThread(() -> getSlotManager().reportSlotStatus(taskExecutionConnection.getInstanceID(), new SlotReport(createAllocatedSlotStatus(allocationId, DEFAULT_SLOT_RESOURCE_PROFILE))));
assertThat(trackingSecurityManager.getSystemExitFuture().isDone(), is(false));
});
}
};
System.setSecurityManager(null);
}
use of org.apache.flink.api.java.tuple.Tuple6 in project flink by apache.
the class DeclarativeSlotManagerTest method testTaskManagerUnregistration.
/**
* Tests that un-registration of task managers will free and remove all registered slots.
*/
@Test
public void testTaskManagerUnregistration() throws Exception {
final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> new CompletableFuture<>()).createTestingTaskExecutorGateway();
final TaskExecutorConnection taskManagerConnection = createTaskExecutorConnection(taskExecutorGateway);
final ResourceID resourceId = taskManagerConnection.getResourceID();
final SlotID slotId1 = new SlotID(resourceId, 0);
final SlotID slotId2 = new SlotID(resourceId, 1);
final SlotReport slotReport = new SlotReport(Arrays.asList(createAllocatedSlotStatus(slotId1), createFreeSlotStatus(slotId2)));
final ResourceRequirements resourceRequirements = createResourceRequirementsForSingleSlot();
final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setSlotTracker(slotTracker).buildAndStartWithDirectExec()) {
slotManager.registerTaskManager(taskManagerConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
assertEquals("The number registered slots does not equal the expected number.", 2, slotManager.getNumberRegisteredSlots());
slotManager.processResourceRequirements(resourceRequirements);
slotManager.unregisterTaskManager(taskManagerConnection.getInstanceID(), TEST_EXCEPTION);
assertEquals(0, slotManager.getNumberRegisteredSlots());
}
}
Aggregations