Examples with InputAttemptIdentifier org.apache.tez.runtime.library.common.InputAttemptIdentifier used on opensource projects

Example 11 with InputAttemptIdentifier

use of org.apache.tez.runtime.library.common.InputAttemptIdentifier in project tez by apache.

the class TestFetcher method testLocalFetchModeSetting.

@Test(timeout = 3000)
public void testLocalFetchModeSetting() throws Exception {
    TezConfiguration conf = new TezConfiguration();
    conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
    InputAttemptIdentifier[] srcAttempts = { new InputAttemptIdentifier(0, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1") };
    FetcherCallback fetcherCallback = mock(FetcherCallback.class);
    final boolean ENABLE_LOCAL_FETCH = true;
    final boolean DISABLE_LOCAL_FETCH = false;
    Fetcher.FetcherBuilder builder = new Fetcher.FetcherBuilder(fetcherCallback, null, null, ApplicationId.newInstance(0, 1), 1, null, "fetcherTest", conf, ENABLE_LOCAL_FETCH, HOST, PORT, false, true, false);
    builder.assignWork(HOST, PORT, 0, 1, Arrays.asList(srcAttempts));
    Fetcher fetcher = spy(builder.build());
    FetchResult fr = new FetchResult(HOST, PORT, 0, 1, Arrays.asList(srcAttempts));
    Fetcher.HostFetchResult hfr = new Fetcher.HostFetchResult(fr, srcAttempts, false);
    doReturn(hfr).when(fetcher).setupLocalDiskFetch();
    doReturn(null).when(fetcher).doHttpFetch();
    doNothing().when(fetcher).shutdown();
    fetcher.call();
    verify(fetcher).setupLocalDiskFetch();
    verify(fetcher, never()).doHttpFetch();
    // when enabled and hostname does not match use http fetch.
    builder = new Fetcher.FetcherBuilder(fetcherCallback, null, null, ApplicationId.newInstance(0, 1), -1, null, "fetcherTest", conf, ENABLE_LOCAL_FETCH, HOST, PORT, false, true, false);
    builder.assignWork(HOST + "_OTHER", PORT, 0, 1, Arrays.asList(srcAttempts));
    fetcher = spy(builder.build());
    doReturn(null).when(fetcher).setupLocalDiskFetch();
    doReturn(hfr).when(fetcher).doHttpFetch();
    doNothing().when(fetcher).shutdown();
    fetcher.call();
    verify(fetcher, never()).setupLocalDiskFetch();
    verify(fetcher).doHttpFetch();
    // when enabled and port does not match use http fetch.
    builder = new Fetcher.FetcherBuilder(fetcherCallback, null, null, ApplicationId.newInstance(0, 1), -1, null, "fetcherTest", conf, ENABLE_LOCAL_FETCH, HOST, PORT, false, true, false);
    builder.assignWork(HOST, PORT + 1, 0, 1, Arrays.asList(srcAttempts));
    fetcher = spy(builder.build());
    doReturn(null).when(fetcher).setupLocalDiskFetch();
    doReturn(hfr).when(fetcher).doHttpFetch();
    doNothing().when(fetcher).shutdown();
    fetcher.call();
    verify(fetcher, never()).setupLocalDiskFetch();
    verify(fetcher).doHttpFetch();
    // When disabled use http fetch
    conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, false);
    builder = new Fetcher.FetcherBuilder(fetcherCallback, null, null, ApplicationId.newInstance(0, 1), 1, null, "fetcherTest", conf, DISABLE_LOCAL_FETCH, HOST, PORT, false, true, false);
    builder.assignWork(HOST, PORT, 0, 1, Arrays.asList(srcAttempts));
    fetcher = spy(builder.build());
    doReturn(null).when(fetcher).setupLocalDiskFetch();
    doReturn(hfr).when(fetcher).doHttpFetch();
    doNothing().when(fetcher).shutdown();
    fetcher.call();
    verify(fetcher, never()).setupLocalDiskFetch();
    verify(fetcher).doHttpFetch();
}

Example 12 with InputAttemptIdentifier

use of org.apache.tez.runtime.library.common.InputAttemptIdentifier in project tez by apache.

the class TestFetcher method testInputAttemptIdentifierMap.

@Test(timeout = 5000)
public void testInputAttemptIdentifierMap() {
    InputAttemptIdentifier[] srcAttempts = { new InputAttemptIdentifier(0, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_0", false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), // duplicate entry
    new InputAttemptIdentifier(0, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_0", false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), // pipeline shuffle based identifiers, with multiple attempts
    new InputAttemptIdentifier(1, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1", false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), new InputAttemptIdentifier(1, 2, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1", false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), new InputAttemptIdentifier(1, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_2", false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 1), new InputAttemptIdentifier(1, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_3", false, InputAttemptIdentifier.SPILL_INFO.FINAL_UPDATE, 2), new InputAttemptIdentifier(2, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_3", false, InputAttemptIdentifier.SPILL_INFO.FINAL_MERGE_ENABLED, 0) };
    InputAttemptIdentifier[] expectedSrcAttempts = { new InputAttemptIdentifier(0, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_0", false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), // pipeline shuffle based identifiers
    new InputAttemptIdentifier(1, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1", false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), new InputAttemptIdentifier(1, 2, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1", false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), new InputAttemptIdentifier(1, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_2", false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 1), new InputAttemptIdentifier(1, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_3", false, InputAttemptIdentifier.SPILL_INFO.FINAL_UPDATE, 2), new InputAttemptIdentifier(2, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_3", false, InputAttemptIdentifier.SPILL_INFO.FINAL_MERGE_ENABLED, 0) };
    TezConfiguration conf = new TezConfiguration();
    conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, "true");
    int partition = 42;
    FetcherCallback callback = mock(FetcherCallback.class);
    Fetcher.FetcherBuilder builder = new Fetcher.FetcherBuilder(callback, null, null, ApplicationId.newInstance(0, 1), 1, null, "fetcherTest", conf, true, HOST, PORT, false, true, false);
    builder.assignWork(HOST, PORT, partition, 1, Arrays.asList(srcAttempts));
    Fetcher fetcher = spy(builder.build());
    fetcher.populateRemainingMap(new LinkedList<InputAttemptIdentifier>(Arrays.asList(srcAttempts)));
    Assert.assertTrue(expectedSrcAttempts.length == fetcher.srcAttemptsRemaining.size());
    Iterator<Entry<String, InputAttemptIdentifier>> iterator = fetcher.srcAttemptsRemaining.entrySet().iterator();
    int count = 0;
    while (iterator.hasNext()) {
        String key = iterator.next().getKey();
        Assert.assertTrue(expectedSrcAttempts[count++].toString().compareTo(key) == 0);
    }
}

Example 13 with InputAttemptIdentifier

use of org.apache.tez.runtime.library.common.InputAttemptIdentifier in project tez by apache.

the class TestShuffleInputEventHandlerImpl method testPipelinedShuffleEvents_WithEmptyPartitions.

/**
 * In pipelined shuffle, check if processing & exceptions are done correctly when empty
 * partitions are sent
 *
 * @throws IOException
 */
@Test(timeout = 5000)
public void testPipelinedShuffleEvents_WithEmptyPartitions() throws IOException {
    InputContext inputContext = createInputContext();
    ShuffleManager shuffleManager = createShuffleManager(inputContext);
    FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);
    ShuffleInputEventHandlerImpl handler = new ShuffleInputEventHandlerImpl(inputContext, shuffleManager, inputAllocator, null, false, 0, false);
    // 0--> 1 with spill id 0 (attemptNum 0) with empty partitions
    BitSet bitSet = new BitSet(4);
    bitSet.flip(0, 4);
    Event dme = createDataMovementEvent(true, 0, 1, 0, false, bitSet, 4, 0);
    handler.handleEvents(Collections.singletonList(dme));
    InputAttemptIdentifier expected = new InputAttemptIdentifier(1, 0, PATH_COMPONENT, false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0);
    verify(shuffleManager, times(1)).addCompletedInputWithNoData(expected);
    // 0--> 1 with spill id 1 (attemptNum 0)
    handler.handleEvents(Collections.singletonList(dme));
    dme = createDataMovementEvent(true, 0, 1, 1, false, new BitSet(), 4, 0);
    expected = new InputAttemptIdentifier(1, 0, PATH_COMPONENT, false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 1);
    verify(shuffleManager, times(2)).addCompletedInputWithNoData(expected);
    // Let attemptNum 0 be scheduled.
    shuffleManager.shuffleInfoEventsMap.get(expected.getInputIdentifier()).scheduledForDownload = true;
    // Now send attemptNum 1.  This should throw exception, because attempt #1 is already added
    dme = createDataMovementEvent(true, 0, 1, 0, false, new BitSet(), 4, 1);
    handler.handleEvents(Collections.singletonList(dme));
    verify(inputContext).killSelf(any(Throwable.class), anyString());
}

Example 14 with InputAttemptIdentifier

use of org.apache.tez.runtime.library.common.InputAttemptIdentifier in project tez by apache.

the class TestShuffleInputEventHandlerImpl method testMultipleEvents1.

@Test(timeout = 5000)
public void testMultipleEvents1() throws IOException {
    InputContext inputContext = mock(InputContext.class);
    ShuffleManager shuffleManager = mock(ShuffleManager.class);
    FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);
    ShuffleInputEventHandlerImpl handler = new ShuffleInputEventHandlerImpl(inputContext, shuffleManager, inputAllocator, null, false, 0, false);
    int taskIndex1 = 1;
    Event dme1 = createDataMovementEvent(0, taskIndex1, createEmptyPartitionByteString(0));
    int taskIndex2 = 2;
    Event dme2 = createDataMovementEvent(0, taskIndex2, null);
    List<Event> eventList = new LinkedList<Event>();
    eventList.add(dme1);
    eventList.add(dme2);
    handler.handleEvents(eventList);
    InputAttemptIdentifier expectedIdentifier1 = new InputAttemptIdentifier(taskIndex1, 0);
    CompositeInputAttemptIdentifier expectedIdentifier2 = new CompositeInputAttemptIdentifier(taskIndex2, 0, PATH_COMPONENT, 1);
    verify(shuffleManager).addCompletedInputWithNoData(eq(expectedIdentifier1));
    verify(shuffleManager).addKnownInput(eq(HOST), eq(PORT), eq(expectedIdentifier2), eq(0));
}

Example 15 with InputAttemptIdentifier

use of org.apache.tez.runtime.library.common.InputAttemptIdentifier in project tez by apache.

the class TestSimpleFetchedInputAllocator method testInMemAllocation.

@Test(timeout = 5000)
public void testInMemAllocation() throws IOException {
    String localDirs = "/tmp/" + this.getClass().getName();
    Configuration conf = new Configuration();
    long jvmMax = Runtime.getRuntime().maxMemory();
    LOG.info("jvmMax: " + jvmMax);
    float bufferPercent = 0.1f;
    conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, bufferPercent);
    conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 1.0f);
    conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, localDirs);
    long inMemThreshold = (long) (bufferPercent * jvmMax);
    LOG.info("InMemThreshold: " + inMemThreshold);
    SimpleFetchedInputAllocator inputManager = new SimpleFetchedInputAllocator("srcName", UUID.randomUUID().toString(), 123, conf, Runtime.getRuntime().maxMemory(), inMemThreshold);
    long requestSize = (long) (0.4f * inMemThreshold);
    long compressedSize = 1l;
    LOG.info("RequestSize: " + requestSize);
    FetchedInput fi1 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(1, 1));
    assertEquals(FetchedInput.Type.MEMORY, fi1.getType());
    FetchedInput fi2 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(2, 1));
    assertEquals(FetchedInput.Type.MEMORY, fi2.getType());
    // Over limit by this point. Next reserve should give back a DISK allocation
    FetchedInput fi3 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(3, 1));
    assertEquals(FetchedInput.Type.DISK, fi3.getType());
    // Freed one memory allocation. Next should be mem again.
    fi1.abort();
    fi1.free();
    FetchedInput fi4 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(4, 1));
    assertEquals(FetchedInput.Type.MEMORY, fi4.getType());
    // Freed one disk allocation. Next sould be disk again (no mem freed)
    fi3.abort();
    fi3.free();
    FetchedInput fi5 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(4, 1));
    assertEquals(FetchedInput.Type.DISK, fi5.getType());
}