Examples with JobID org.apache.flink.api.common.JobID used on opensource projects

Example 71 with JobID

use of org.apache.flink.api.common.JobID in project flink by apache.

the class MemoryStateBackendTest method testOversizedState.

@Test
public void testOversizedState() {
    try {
        MemoryStateBackend backend = new MemoryStateBackend(10);
        CheckpointStreamFactory streamFactory = backend.createStreamFactory(new JobID(), "test_op");
        HashMap<String, Integer> state = new HashMap<>();
        state.put("hey there", 2);
        state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77);
        try {
            CheckpointStreamFactory.CheckpointStateOutputStream outStream = streamFactory.createCheckpointStateOutputStream(12, 459);
            ObjectOutputStream oos = new ObjectOutputStream(outStream);
            oos.writeObject(state);
            oos.flush();
            outStream.closeAndGetHandle();
            fail("this should cause an exception");
        } catch (IOException e) {
        // now darling, isn't that exactly what we wanted?
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 72 with JobID

use of org.apache.flink.api.common.JobID in project flink by apache.

the class OperatorStateBackendTest method testSnapshotRestore.

@Test
public void testSnapshotRestore() throws Exception {
    DefaultOperatorStateBackend operatorStateBackend = createNewOperatorStateBackend();
    ListStateDescriptor<Serializable> stateDescriptor1 = new ListStateDescriptor<>("test1", new JavaSerializer<>());
    ListStateDescriptor<Serializable> stateDescriptor2 = new ListStateDescriptor<>("test2", new JavaSerializer<>());
    ListStateDescriptor<Serializable> stateDescriptor3 = new ListStateDescriptor<>("test3", new JavaSerializer<>());
    ListState<Serializable> listState1 = operatorStateBackend.getOperatorState(stateDescriptor1);
    ListState<Serializable> listState2 = operatorStateBackend.getOperatorState(stateDescriptor2);
    ListState<Serializable> listState3 = operatorStateBackend.getBroadcastOperatorState(stateDescriptor3);
    listState1.add(42);
    listState1.add(4711);
    listState2.add(7);
    listState2.add(13);
    listState2.add(23);
    listState3.add(17);
    listState3.add(18);
    listState3.add(19);
    listState3.add(20);
    CheckpointStreamFactory streamFactory = abstractStateBackend.createStreamFactory(new JobID(), "testOperator");
    OperatorStateHandle stateHandle = FutureUtil.runIfNotDoneAndGet(operatorStateBackend.snapshot(1, 1, streamFactory, CheckpointOptions.forFullCheckpoint()));
    try {
        operatorStateBackend.close();
        operatorStateBackend.dispose();
        //TODO this is temporarily casted to test already functionality that we do not yet expose through public API
        operatorStateBackend = (DefaultOperatorStateBackend) abstractStateBackend.createOperatorStateBackend(createMockEnvironment(), "testOperator");
        operatorStateBackend.restore(Collections.singletonList(stateHandle));
        assertEquals(3, operatorStateBackend.getRegisteredStateNames().size());
        listState1 = operatorStateBackend.getOperatorState(stateDescriptor1);
        listState2 = operatorStateBackend.getOperatorState(stateDescriptor2);
        listState3 = operatorStateBackend.getBroadcastOperatorState(stateDescriptor3);
        assertEquals(3, operatorStateBackend.getRegisteredStateNames().size());
        Iterator<Serializable> it = listState1.get().iterator();
        assertEquals(42, it.next());
        assertEquals(4711, it.next());
        assertTrue(!it.hasNext());
        it = listState2.get().iterator();
        assertEquals(7, it.next());
        assertEquals(13, it.next());
        assertEquals(23, it.next());
        assertTrue(!it.hasNext());
        it = listState3.get().iterator();
        assertEquals(17, it.next());
        assertEquals(18, it.next());
        assertEquals(19, it.next());
        assertEquals(20, it.next());
        assertTrue(!it.hasNext());
        operatorStateBackend.close();
        operatorStateBackend.dispose();
    } finally {
        stateHandle.discardState();
    }
}

Example 73 with JobID

use of org.apache.flink.api.common.JobID in project flink by apache.

the class SlotManagerTest method testRequestSlotWithoutFreeSlot.

/**
	 * Tests that there are no free slots when we request, need to allocate from cluster manager master
	 */
@Test
public void testRequestSlotWithoutFreeSlot() {
    TestingSlotManager slotManager = new TestingSlotManager();
    slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE));
    assertEquals(0, slotManager.getAllocatedSlotCount());
    assertEquals(0, slotManager.getFreeSlotCount());
    assertEquals(1, slotManager.getPendingRequestCount());
    assertEquals(1, slotManager.getAllocatedContainers().size());
    assertEquals(DEFAULT_TESTING_PROFILE, slotManager.getAllocatedContainers().get(0));
}

Example 74 with JobID

use of org.apache.flink.api.common.JobID in project flink by apache.

the class SlotManagerTest method testDuplicatedSlotRequest.

/**
	 * Tests that we send duplicated slot request
	 */
@Test
public void testDuplicatedSlotRequest() {
    TestingSlotManager slotManager = new TestingSlotManager();
    directlyProvideFreeSlots(slotManager, DEFAULT_TESTING_PROFILE, 1);
    SlotRequest request1 = new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE);
    SlotRequest request2 = new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_BIG_PROFILE);
    slotManager.requestSlot(request1);
    slotManager.requestSlot(request2);
    slotManager.requestSlot(request2);
    slotManager.requestSlot(request1);
    assertEquals(1, slotManager.getAllocatedSlotCount());
    assertEquals(0, slotManager.getFreeSlotCount());
    assertEquals(1, slotManager.getPendingRequestCount());
    assertEquals(1, slotManager.getAllocatedContainers().size());
    assertEquals(DEFAULT_TESTING_BIG_PROFILE, slotManager.getAllocatedContainers().get(0));
}

Example 75 with JobID

use of org.apache.flink.api.common.JobID in project flink by apache.

the class SlotManagerTest method testRequestMultipleSlots.

/**
	 * Tests that we send multiple slot requests
	 */
@Test
public void testRequestMultipleSlots() {
    TestingSlotManager slotManager = new TestingSlotManager();
    directlyProvideFreeSlots(slotManager, DEFAULT_TESTING_PROFILE, 5);
    // request 3 normal slots
    for (int i = 0; i < 3; ++i) {
        slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE));
    }
    // request 2 big slots
    for (int i = 0; i < 2; ++i) {
        slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_BIG_PROFILE));
    }
    // request 1 normal slot again
    slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE));
    assertEquals(4, slotManager.getAllocatedSlotCount());
    assertEquals(1, slotManager.getFreeSlotCount());
    assertEquals(2, slotManager.getPendingRequestCount());
    assertEquals(2, slotManager.getAllocatedContainers().size());
    assertEquals(DEFAULT_TESTING_BIG_PROFILE, slotManager.getAllocatedContainers().get(0));
    assertEquals(DEFAULT_TESTING_BIG_PROFILE, slotManager.getAllocatedContainers().get(1));
}