use of org.apache.tez.runtime.api.OutputContext in project tez by apache.
the class TestWeightedScalingMemoryDistributor method testWeightedScalingNonConcurrent.
@Test(timeout = 5000)
public void testWeightedScalingNonConcurrent() throws TezException {
Configuration conf = new Configuration(this.conf);
conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_INPUT_OUTPUT_CONCURRENT, false);
conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_NON_CONCURRENT_INPUTS_ENABLED, true);
conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION, 0.2);
conf.setStrings(TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS, WeightedScalingMemoryDistributor.generateWeightStrings(0, 0, 1, 2, 3, 1, 1));
System.err.println(Joiner.on(",").join(conf.getStringCollection(TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS)));
MemoryDistributor dist = new MemoryDistributor(2, 2, conf);
dist.setJvmMemory(10000l);
// First request - ScatterGatherShuffleInput
MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
InputContext e1InputContext1 = createTestInputContext();
InputDescriptor e1InDesc1 = createTestInputDescriptor(OrderedGroupedKVInput.class);
dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);
// Second request - BroadcastInput
MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
InputContext e2InputContext2 = createTestInputContext();
InputDescriptor e2InDesc2 = createTestInputDescriptor(UnorderedKVInput.class);
dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
// Third request - randomOutput (simulates MROutput)
MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
OutputContext e3OutputContext1 = createTestOutputContext();
OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
dist.requestMemory(10000, e3Callback, e3OutputContext1, e3OutDesc1);
// Fourth request - OnFileSortedOutput
MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
OutputContext e4OutputContext2 = createTestOutputContext();
OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OrderedPartitionedKVOutput.class);
dist.requestMemory(10000, e4Callback, e4OutputContext2, e4OutDesc2);
// Fifth request - Processor
MemoryUpdateCallbackForTest e5Callback = new MemoryUpdateCallbackForTest();
ProcessorContext e5ProcContext = createTestProcessortContext();
ProcessorDescriptor e5ProcDesc = createTestProcessorDescriptor();
dist.requestMemory(10000, e5Callback, e5ProcContext, e5ProcDesc);
dist.makeInitialAllocations();
// Total available: 80% of 10K = 8000
// 5 requests (weight) - 10K (3), 10K(1), 10K(1), 10K(2), 10K(1)
// Overlap input and output memory
assertEquals(5250, e1Callback.assigned);
assertEquals(1750, e2Callback.assigned);
assertEquals(2333, e3Callback.assigned);
assertEquals(4666, e4Callback.assigned);
assertEquals(1000, e5Callback.assigned);
}
use of org.apache.tez.runtime.api.OutputContext in project tez by apache.
the class TestWeightedScalingMemoryDistributor method testAdditionalReserveFractionWeightedScalingNonConcurrent.
@Test(timeout = 5000)
public void testAdditionalReserveFractionWeightedScalingNonConcurrent() throws TezException {
Configuration conf = new Configuration(this.conf);
conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_INPUT_OUTPUT_CONCURRENT, false);
conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_NON_CONCURRENT_INPUTS_ENABLED, true);
conf.setStrings(TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS, WeightedScalingMemoryDistributor.generateWeightStrings(0, 0, 2, 3, 6, 1, 1));
conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ADDITIONAL_RESERVATION_FRACTION_PER_IO, 0.025d);
conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ADDITIONAL_RESERVATION_FRACTION_MAX, 0.2d);
MemoryDistributor dist = new MemoryDistributor(2, 2, conf);
dist.setJvmMemory(10000l);
// First request - ScatterGatherShuffleInput [weight 6]
MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
InputContext e1InputContext1 = createTestInputContext();
InputDescriptor e1InDesc1 = createTestInputDescriptor(OrderedGroupedKVInput.class);
dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);
// Second request - BroadcastInput [weight 2]
MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
InputContext e2InputContext2 = createTestInputContext();
InputDescriptor e2InDesc2 = createTestInputDescriptor(UnorderedKVInput.class);
dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
// Third request - randomOutput (simulates MROutput) [weight 1]
MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
OutputContext e3OutputContext1 = createTestOutputContext();
OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
dist.requestMemory(10000, e3Callback, e3OutputContext1, e3OutDesc1);
// Fourth request - OnFileSortedOutput [weight 3]
MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
OutputContext e4OutputContext2 = createTestOutputContext();
OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OrderedPartitionedKVOutput.class);
dist.requestMemory(10000, e4Callback, e4OutputContext2, e4OutDesc2);
dist.makeInitialAllocations();
// Total available: 60% of 10K = 6000
// 4 requests (weight) - 10K (6), 10K(2), 10K(1), 10K(3)
// Overlap input and output memory
assertEquals(4500, e1Callback.assigned);
assertEquals(1500, e2Callback.assigned);
assertEquals(1500, e3Callback.assigned);
assertEquals(4500, e4Callback.assigned);
}
use of org.apache.tez.runtime.api.OutputContext in project tez by apache.
the class TestWeightedScalingMemoryDistributor method testWeightedScalingNonConcurrentInputsDisabled.
@Test(timeout = 5000)
public void testWeightedScalingNonConcurrentInputsDisabled() throws TezException {
Configuration conf = new Configuration(this.conf);
conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_INPUT_OUTPUT_CONCURRENT, false);
conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_NON_CONCURRENT_INPUTS_ENABLED, false);
conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION, 0.2);
conf.setStrings(TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS, WeightedScalingMemoryDistributor.generateWeightStrings(0, 0, 1, 2, 3, 1, 1));
System.err.println(Joiner.on(",").join(conf.getStringCollection(TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS)));
MemoryDistributor dist = new MemoryDistributor(2, 2, conf);
dist.setJvmMemory(10000l);
// First request - ScatterGatherShuffleInput
MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
InputContext e1InputContext1 = createTestInputContext();
InputDescriptor e1InDesc1 = createTestInputDescriptor(OrderedGroupedKVInput.class);
dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);
// Second request - BroadcastInput
MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
InputContext e2InputContext2 = createTestInputContext();
InputDescriptor e2InDesc2 = createTestInputDescriptor(UnorderedKVInput.class);
dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
// Third request - randomOutput (simulates MROutput)
MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
OutputContext e3OutputContext1 = createTestOutputContext();
OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
dist.requestMemory(10000, e3Callback, e3OutputContext1, e3OutDesc1);
// Fourth request - OnFileSortedOutput
MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
OutputContext e4OutputContext2 = createTestOutputContext();
OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OrderedPartitionedKVOutput.class);
dist.requestMemory(10000, e4Callback, e4OutputContext2, e4OutDesc2);
// Fifth request - Processor
MemoryUpdateCallbackForTest e5Callback = new MemoryUpdateCallbackForTest();
ProcessorContext e5ProcContext = createTestProcessortContext();
ProcessorDescriptor e5ProcDesc = createTestProcessorDescriptor();
dist.requestMemory(10000, e5Callback, e5ProcContext, e5ProcDesc);
dist.makeInitialAllocations();
// Total available: 80% of 10K = 8000
// 5 requests (weight) - 10K (3), 10K(1), 10K(1), 10K(2), 10K(1)
// Overlap input and output memory
assertEquals(3000, e1Callback.assigned);
assertEquals(1000, e2Callback.assigned);
assertEquals(2333, e3Callback.assigned);
assertEquals(4666, e4Callback.assigned);
assertEquals(1000, e5Callback.assigned);
}
use of org.apache.tez.runtime.api.OutputContext in project tez by apache.
the class TestMultiMROutput method createMROutputs.
private MultiMROutput createMROutputs(Class outputFormat, boolean isMapper, boolean useLazyOutputFormat) throws InterruptedException, IOException {
String outputPath = "/tmp/output";
JobConf conf = new JobConf();
conf.setBoolean(MRConfig.IS_MAP_PROCESSOR, isMapper);
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(Text.class);
DataSinkDescriptor dataSink = MultiMROutput.createConfigBuilder(conf, outputFormat, outputPath, useLazyOutputFormat).build();
OutputContext outputContext = createMockOutputContext(dataSink.getOutputDescriptor().getUserPayload());
MultiMROutput output = new MultiMROutput(outputContext, 2);
output.initialize();
return output;
}
use of org.apache.tez.runtime.api.OutputContext in project tez by apache.
the class TestMultiMROutput method createMockOutputContext.
private OutputContext createMockOutputContext(UserPayload payload) {
OutputContext outputContext = mock(OutputContext.class);
ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
when(outputContext.getUserPayload()).thenReturn(payload);
when(outputContext.getApplicationId()).thenReturn(appId);
when(outputContext.getTaskVertexIndex()).thenReturn(1);
when(outputContext.getTaskAttemptNumber()).thenReturn(1);
when(outputContext.getCounters()).thenReturn(new TezCounters());
when(outputContext.getStatisticsReporter()).thenReturn(mock(OutputStatisticsReporter.class));
return outputContext;
}
Aggregations