Examples with InputInitializerContext org.apache.tez.runtime.api.InputInitializerContext used on opensource projects

Example 11 with InputInitializerContext

use of org.apache.tez.runtime.api.InputInitializerContext in project hive by apache.

the class TestDynamicPartitionPruner method testSingleSourceMultipleFiltersOrdering1.

@Test(timeout = 20000)
public void testSingleSourceMultipleFiltersOrdering1() throws InterruptedException, SerDeException {
    InputInitializerContext mockInitContext = mock(InputInitializerContext.class);
    doReturn(2).when(mockInitContext).getVertexNumTasks("v1");
    MapWork mapWork = createMockMapWork(new TestSource("v1", 2));
    DynamicPartitionPrunerForEventTesting pruner = new DynamicPartitionPrunerForEventTesting();
    pruner.initialize(mockInitContext, mapWork, new JobConf());
    PruneRunnable pruneRunnable = new PruneRunnable(pruner);
    Thread t = new Thread(pruneRunnable);
    t.start();
    try {
        pruneRunnable.start();
        InputInitializerEvent event = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0));
        event.setSourceVertexName("v1");
        pruner.addEvent(event);
        pruner.addEvent(event);
        pruner.addEvent(event);
        pruner.addEvent(event);
        pruner.processVertex("v1");
        pruneRunnable.awaitEnd();
        assertNoError(pruneRunnable);
        assertEquals(4, pruner.eventsProceessed.intValue());
        assertEquals(2, pruner.filteredSources.intValue());
    } finally {
        t.interrupt();
        t.join();
    }
}

Example 12 with InputInitializerContext

use of org.apache.tez.runtime.api.InputInitializerContext in project hive by apache.

the class TestDynamicPartitionPruner method testMultipleSourcesOrdering3.

@Test(timeout = 20000)
public void testMultipleSourcesOrdering3() throws InterruptedException, SerDeException {
    InputInitializerContext mockInitContext = mock(InputInitializerContext.class);
    doReturn(2).when(mockInitContext).getVertexNumTasks("v1");
    doReturn(3).when(mockInitContext).getVertexNumTasks("v2");
    MapWork mapWork = createMockMapWork(new TestSource("v1", 2), new TestSource("v2", 1));
    DynamicPartitionPrunerForEventTesting pruner = new DynamicPartitionPrunerForEventTesting();
    pruner.initialize(mockInitContext, mapWork, new JobConf());
    PruneRunnable pruneRunnable = new PruneRunnable(pruner);
    Thread t = new Thread(pruneRunnable);
    t.start();
    try {
        pruneRunnable.start();
        InputInitializerEvent eventV1 = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0));
        eventV1.setSourceVertexName("v1");
        InputInitializerEvent eventV2 = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0));
        eventV2.setSourceVertexName("v2");
        // 2 X 2 events for V1. 3 X 1 events for V2
        pruner.addEvent(eventV1);
        pruner.addEvent(eventV1);
        pruner.processVertex("v1");
        pruner.addEvent(eventV1);
        pruner.addEvent(eventV1);
        pruner.addEvent(eventV2);
        pruner.processVertex("v2");
        pruner.addEvent(eventV2);
        pruner.addEvent(eventV2);
        pruneRunnable.awaitEnd();
        assertNoError(pruneRunnable);
        assertEquals(7, pruner.eventsProceessed.intValue());
        assertEquals(3, pruner.filteredSources.intValue());
    } finally {
        t.interrupt();
        t.join();
    }
}

Example 13 with InputInitializerContext

use of org.apache.tez.runtime.api.InputInitializerContext in project hive by apache.

the class TestDynamicPartitionPruner method testExtraEvents.

@Test(timeout = 20000)
public void testExtraEvents() throws InterruptedException, IOException, HiveException, SerDeException {
    InputInitializerContext mockInitContext = mock(InputInitializerContext.class);
    doReturn(1).when(mockInitContext).getVertexNumTasks("v1");
    MapWork mapWork = createMockMapWork(new TestSource("v1", 1));
    DynamicPartitionPrunerForEventTesting pruner = new DynamicPartitionPrunerForEventTesting();
    pruner.initialize(mockInitContext, mapWork, new JobConf());
    PruneRunnable pruneRunnable = new PruneRunnable(pruner);
    Thread t = new Thread(pruneRunnable);
    t.start();
    try {
        pruneRunnable.start();
        InputInitializerEvent event = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0));
        event.setSourceVertexName("v1");
        pruner.addEvent(event);
        pruner.addEvent(event);
        pruner.processVertex("v1");
        pruneRunnable.awaitEnd();
        assertTrue(pruneRunnable.inError.get());
        assertTrue(pruneRunnable.exception instanceof IllegalStateException);
        assertEquals(2, pruner.eventsProceessed.intValue());
        assertEquals(0, pruner.filteredSources.intValue());
    } finally {
        t.interrupt();
        t.join();
    }
}

Example 14 with InputInitializerContext

use of org.apache.tez.runtime.api.InputInitializerContext in project hive by apache.

the class TestDynamicPartitionPruner method testMultipleSourcesOrdering1.

@Test(timeout = 20000)
public void testMultipleSourcesOrdering1() throws InterruptedException, SerDeException {
    InputInitializerContext mockInitContext = mock(InputInitializerContext.class);
    doReturn(2).when(mockInitContext).getVertexNumTasks("v1");
    doReturn(3).when(mockInitContext).getVertexNumTasks("v2");
    MapWork mapWork = createMockMapWork(new TestSource("v1", 2), new TestSource("v2", 1));
    DynamicPartitionPrunerForEventTesting pruner = new DynamicPartitionPrunerForEventTesting();
    pruner.initialize(mockInitContext, mapWork, new JobConf());
    PruneRunnable pruneRunnable = new PruneRunnable(pruner);
    Thread t = new Thread(pruneRunnable);
    t.start();
    try {
        pruneRunnable.start();
        InputInitializerEvent eventV1 = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0));
        eventV1.setSourceVertexName("v1");
        InputInitializerEvent eventV2 = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0));
        eventV2.setSourceVertexName("v2");
        // 2 X 2 events for V1. 3 X 1 events for V2
        pruner.addEvent(eventV1);
        pruner.addEvent(eventV1);
        pruner.addEvent(eventV1);
        pruner.addEvent(eventV1);
        pruner.addEvent(eventV2);
        pruner.addEvent(eventV2);
        pruner.addEvent(eventV2);
        pruner.processVertex("v1");
        pruner.processVertex("v2");
        pruneRunnable.awaitEnd();
        assertNoError(pruneRunnable);
        assertEquals(7, pruner.eventsProceessed.intValue());
        assertEquals(3, pruner.filteredSources.intValue());
    } finally {
        t.interrupt();
        t.join();
    }
}

Example 15 with InputInitializerContext

use of org.apache.tez.runtime.api.InputInitializerContext in project hive by apache.

the class TestDynamicPartitionPruner method testNoPruning.

@Test(timeout = 20000)
public void testNoPruning() throws InterruptedException, IOException, HiveException, SerDeException {
    InputInitializerContext mockInitContext = mock(InputInitializerContext.class);
    MapWork mapWork = mock(MapWork.class);
    DynamicPartitionPrunerForEventTesting pruner = new DynamicPartitionPrunerForEventTesting();
    pruner.initialize(mockInitContext, mapWork, new JobConf());
    PruneRunnable pruneRunnable = new PruneRunnable(pruner);
    Thread t = new Thread(pruneRunnable);
    t.start();
    try {
        pruneRunnable.start();
        pruneRunnable.awaitEnd();
        // Return immediately. No entries found for pruning. Verified via the timeout.
        assertEquals(0, pruner.eventsProceessed.intValue());
        assertEquals(0, pruner.filteredSources.intValue());
    } finally {
        t.interrupt();
        t.join();
    }
}