Examples with DefaultHadoopShim org.apache.tez.hadoop.shim.DefaultHadoopShim used on opensource projects

Example 16 with DefaultHadoopShim

use of org.apache.tez.hadoop.shim.DefaultHadoopShim in project tez by apache.

the class TestRecoveryService method setup.

private void setup(boolean useMockFs, String[][] configs) throws Exception {
    conf = new Configuration();
    if (configs != null) {
        for (String[] config : configs) {
            conf.set(config[0], config[1]);
        }
    }
    appContext = mock(AppContext.class);
    when(appContext.getClock()).thenReturn(new SystemClock());
    when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim());
    when(appContext.getApplicationID()).thenReturn(appId);
    if (useMockFs) {
        fs = mock(FileSystem.class);
        when(appContext.getCurrentRecoveryDir()).thenReturn(new Path("mockfs:///"));
        conf.set("fs.mockfs.impl", MockFileSystem.class.getName());
        MockFileSystem.delegate = fs;
        dagFos = spy(new FSDataOutputStream(new OutputStream() {

            @Override
            public void write(int b) throws IOException {
            }
        }, null));
        summaryFos = spy(new FSDataOutputStream(new OutputStream() {

            @Override
            public void write(int b) throws IOException {
            }
        }, null));
    } else {
        when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(TEST_ROOT_DIR));
        fs = FileSystem.getLocal(conf);
        fs.delete(new Path(TEST_ROOT_DIR), true);
    }
    recoveryService = new MockRecoveryService(appContext);
    conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
    recoveryService.init(conf);
    summaryPath = TezCommonUtils.getSummaryRecoveryPath(recoveryService.recoveryPath);
    dagRecoveryPath = TezCommonUtils.getDAGRecoveryPath(recoveryService.recoveryPath, dagId.toString());
    if (useMockFs) {
        when(fs.create(eq(dagRecoveryPath), eq(false), anyInt())).thenReturn(dagFos);
        when(fs.create(eq(summaryPath), eq(false), anyInt())).thenReturn(summaryFos);
    }
}

Example 17 with DefaultHadoopShim

use of org.apache.tez.hadoop.shim.DefaultHadoopShim in project tez by apache.

the class TestTezTaskRunner2 method testTaskConfUsage.

@Test(timeout = 5000)
public void testTaskConfUsage() throws Exception {
    Configuration conf = new Configuration(false);
    conf.set("global", "global1");
    conf.set("global_override", "global1");
    String[] localDirs = null;
    Configuration taskConf = new Configuration(false);
    conf.set("global_override", "task1");
    conf.set("task", "task1");
    List<InputSpec> inputSpecList = new ArrayList<>();
    List<OutputSpec> outputSpecList = new ArrayList<>();
    TaskSpec taskSpec = new TaskSpec("dagName", "vertexName", 1, mock(ProcessorDescriptor.class), inputSpecList, outputSpecList, null, taskConf);
    TezExecutors sharedExecutor = new TezSharedExecutor(conf);
    TezTaskRunner2 taskRunner2 = new TezTaskRunner2(conf, mock(UserGroupInformation.class), localDirs, taskSpec, 1, null, null, null, mock(TaskReporter.class), null, null, "pid", null, 1000, false, new DefaultHadoopShim(), sharedExecutor);
    Assert.assertEquals("global1", taskRunner2.task.getTaskConf().get("global"));
    Assert.assertEquals("task1", taskRunner2.task.getTaskConf().get("global_override"));
    Assert.assertEquals("task1", taskRunner2.task.getTaskConf().get("task"));
    sharedExecutor.shutdownNow();
}

Example 18 with DefaultHadoopShim

use of org.apache.tez.hadoop.shim.DefaultHadoopShim in project tez by apache.

the class TestProcessorContext method testDagNumber.

@Test(timeout = 5000)
public void testDagNumber() throws IOException {
    String[] localDirs = new String[] { "dummyLocalDir" };
    int appAttemptNumber = 1;
    TezUmbilical tezUmbilical = mock(TezUmbilical.class);
    String dagName = "DAG_NAME";
    String vertexName = "VERTEX_NAME";
    int vertexParallelism = 20;
    int dagNumber = 52;
    ApplicationId appId = ApplicationId.newInstance(10000, 13);
    TezDAGID dagId = TezDAGID.getInstance(appId, dagNumber);
    TezVertexID vertexId = TezVertexID.getInstance(dagId, 6);
    TezTaskID taskId = TezTaskID.getInstance(vertexId, 4);
    TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 2);
    TaskSpec mockSpec = mock(TaskSpec.class);
    when(mockSpec.getInputs()).thenReturn(Collections.singletonList(mock(InputSpec.class)));
    when(mockSpec.getOutputs()).thenReturn(Collections.singletonList(mock(OutputSpec.class)));
    Configuration conf = new Configuration();
    TezSharedExecutor sharedExecutor = new TezSharedExecutor(conf);
    LogicalIOProcessorRuntimeTask runtimeTask = new LogicalIOProcessorRuntimeTask(mockSpec, 1, conf, new String[] { "/" }, tezUmbilical, null, null, null, null, "", null, 1024, false, new DefaultHadoopShim(), sharedExecutor);
    LogicalIOProcessorRuntimeTask mockTask = spy(runtimeTask);
    Map<String, ByteBuffer> serviceConsumerMetadata = Maps.newHashMap();
    Map<String, String> auxServiceEnv = Maps.newHashMap();
    MemoryDistributor memDist = mock(MemoryDistributor.class);
    ProcessorDescriptor processorDesc = mock(ProcessorDescriptor.class);
    InputReadyTracker inputReadyTracker = mock(InputReadyTracker.class);
    ObjectRegistry objectRegistry = new ObjectRegistryImpl();
    ExecutionContext execContext = new ExecutionContextImpl("localhost");
    long memAvailable = 10000l;
    TezProcessorContextImpl procContext = new TezProcessorContextImpl(new Configuration(), localDirs, appAttemptNumber, tezUmbilical, dagName, vertexName, vertexParallelism, taskAttemptId, null, runtimeTask, serviceConsumerMetadata, auxServiceEnv, memDist, processorDesc, inputReadyTracker, objectRegistry, execContext, memAvailable, sharedExecutor);
    assertEquals(dagNumber, procContext.getDagIdentifier());
    assertEquals(appAttemptNumber, procContext.getDAGAttemptNumber());
    assertEquals(appId, procContext.getApplicationId());
    assertEquals(dagName, procContext.getDAGName());
    assertEquals(vertexName, procContext.getTaskVertexName());
    assertEquals(vertexId.getId(), procContext.getTaskVertexIndex());
    assertTrue(Arrays.equals(localDirs, procContext.getWorkDirs()));
    // test auto call of notifyProgress
    procContext.setProgress(0.1f);
    verify(mockTask, times(1)).notifyProgressInvocation();
    sharedExecutor.shutdown();
}

Example 19 with DefaultHadoopShim

use of org.apache.tez.hadoop.shim.DefaultHadoopShim in project tez by apache.

the class TestMROutput method createLogicalTask.

public static LogicalIOProcessorRuntimeTask createLogicalTask(Configuration conf, TezUmbilical umbilical, String dagName, String vertexName, TezExecutors sharedExecutor) throws Exception {
    ProcessorDescriptor procDesc = ProcessorDescriptor.create(TestProcessor.class.getName());
    List<InputSpec> inputSpecs = Lists.newLinkedList();
    List<OutputSpec> outputSpecs = Lists.newLinkedList();
    outputSpecs.add(new OutputSpec("Null", MROutput.createConfigBuilder(conf, TestOutputFormat.class).build().getOutputDescriptor(), 1));
    TaskSpec taskSpec = new TaskSpec(TezTestUtils.getMockTaskAttemptId(0, 0, 0, 0), dagName, vertexName, -1, procDesc, inputSpecs, outputSpecs, null, null);
    FileSystem fs = FileSystem.getLocal(conf);
    Path workDir = new Path(new Path(System.getProperty("test.build.data", "/tmp")), "TestMapOutput").makeQualified(fs.getUri(), fs.getWorkingDirectory());
    return new LogicalIOProcessorRuntimeTask(taskSpec, 0, conf, new String[] { workDir.toString() }, umbilical, null, new HashMap<String, String>(), HashMultimap.<String, String>create(), null, "", new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory(), true, new DefaultHadoopShim(), sharedExecutor);
}