use of org.apache.tez.dag.api.InputDescriptor in project tez by apache.
the class TestMemoryDistributor method testScalingNoProcessor2.
@Test(timeout = 5000)
public void testScalingNoProcessor2() throws TezException {
// Real world values
MemoryDistributor dist = new MemoryDistributor(2, 0, conf);
dist.setJvmMemory(209715200l);
// First request
MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
InputContext e1InputContext1 = createTestInputContext();
InputDescriptor e1InDesc1 = createTestInputDescriptor();
dist.requestMemory(104857600l, e1Callback, e1InputContext1, e1InDesc1);
// Second request
MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
InputContext e2InputContext2 = createTestInputContext();
InputDescriptor e2InDesc2 = createTestInputDescriptor();
dist.requestMemory(157286400l, e2Callback, e2InputContext2, e2InDesc2);
dist.makeInitialAllocations();
assertEquals(58720256l, e1Callback.assigned);
assertEquals(88080384l, e2Callback.assigned);
}
use of org.apache.tez.dag.api.InputDescriptor in project tez by apache.
the class TestMemoryDistributor method testReserveFractionConfigured.
@Test(timeout = 5000)
public void testReserveFractionConfigured() throws TezException {
Configuration conf = new Configuration(this.conf);
conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION, 0.5d);
MemoryDistributor dist = new MemoryDistributor(2, 1, conf);
dist.setJvmMemory(10000l);
// First request
MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
InputContext e1InputContext1 = createTestInputContext();
InputDescriptor e1InDesc1 = createTestInputDescriptor();
dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);
// Second request
MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
InputContext e2InputContext2 = createTestInputContext();
InputDescriptor e2InDesc2 = createTestInputDescriptor();
dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
// Third request - output
MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
OutputContext e3OutputContext1 = createTestOutputContext();
OutputDescriptor e3OutDesc2 = createTestOutputDescriptor();
dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc2);
dist.makeInitialAllocations();
// Total available: 50% of 10K = 7000
// 3 requests - 10K, 10K, 5K
// Scale down to - 2000, 2000, 1000
assertEquals(2000, e1Callback.assigned);
assertEquals(2000, e2Callback.assigned);
assertEquals(1000, e3Callback.assigned);
}
use of org.apache.tez.dag.api.InputDescriptor in project tez by apache.
the class TestMemoryDistributor method testScalingNoProcessor.
@Test(timeout = 5000)
public void testScalingNoProcessor() throws TezException {
MemoryDistributor dist = new MemoryDistributor(2, 1, conf);
dist.setJvmMemory(10000l);
// First request
MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
InputContext e1InputContext1 = createTestInputContext();
InputDescriptor e1InDesc1 = createTestInputDescriptor();
dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);
// Second request
MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
InputContext e2InputContext2 = createTestInputContext();
InputDescriptor e2InDesc2 = createTestInputDescriptor();
dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
// Third request - output
MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
OutputContext e3OutputContext1 = createTestOutputContext();
OutputDescriptor e3OutDesc2 = createTestOutputDescriptor();
dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc2);
dist.makeInitialAllocations();
// Total available: 70% of 10K = 7000
// 3 requests - 10K, 10K, 5K
// Scale down to - 2800, 2800, 1400
assertEquals(2800, e1Callback.assigned);
assertEquals(2800, e2Callback.assigned);
assertEquals(1400, e3Callback.assigned);
}
use of org.apache.tez.dag.api.InputDescriptor in project tez by apache.
the class GroupInputSpec method readFields.
@Override
public void readFields(DataInput in) throws IOException {
groupName = StringInterner.weakIntern(Text.readString(in));
int numMembers = in.readInt();
groupVertices = Lists.newArrayListWithCapacity(numMembers);
for (int i = 0; i < numMembers; ++i) {
groupVertices.add(StringInterner.weakIntern(Text.readString(in)));
}
mergedInputDescriptor = new InputDescriptor();
mergedInputDescriptor.readFields(in);
}
use of org.apache.tez.dag.api.InputDescriptor in project tez by apache.
the class InputSpec method readFields.
@Override
public void readFields(DataInput in) throws IOException {
sourceVertexName = StringInterner.weakIntern(in.readUTF());
physicalEdgeCount = in.readInt();
inputDescriptor = new InputDescriptor();
inputDescriptor.readFields(in);
}
Aggregations