Examples with TezDAGID org.apache.tez.dag.records.TezDAGID used on opensource projects

Example 91 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project tez by apache.

the class TestStateChangeNotifier method testSpecificStateUpdates.

@Test(timeout = 5000)
public void testSpecificStateUpdates() {
    TezDAGID dagId = TezDAGID.getInstance("1", 1, 1);
    Vertex v1 = createMockVertex(dagId, 1);
    DAG dag = createMockDag(dagId, v1);
    StateChangeNotifierForTest tracker = new StateChangeNotifierForTest(dag);
    VertexStateUpdateListener mockListener = mock(VertexStateUpdateListener.class);
    tracker.registerForVertexUpdates(v1.getName(), EnumSet.of(VertexState.RUNNING, VertexState.SUCCEEDED), mockListener);
    List<VertexState> states = Lists.newArrayList(VertexState.RUNNING, VertexState.SUCCEEDED, VertexState.FAILED, VertexState.KILLED, VertexState.RUNNING, VertexState.SUCCEEDED);
    List<VertexState> expectedStates = Lists.newArrayList(VertexState.RUNNING, VertexState.SUCCEEDED, VertexState.RUNNING, VertexState.SUCCEEDED);
    for (VertexState state : states) {
        notifyTracker(tracker, v1, state);
    }
    ArgumentCaptor<VertexStateUpdate> argumentCaptor = ArgumentCaptor.forClass(VertexStateUpdate.class);
    verify(mockListener, times(expectedStates.size())).onStateUpdated(argumentCaptor.capture());
    List<VertexStateUpdate> stateUpdatesSent = argumentCaptor.getAllValues();
    Iterator<VertexState> expectedStateIter = expectedStates.iterator();
    for (int i = 0; i < expectedStates.size(); i++) {
        assertEquals(expectedStateIter.next(), stateUpdatesSent.get(i).getVertexState());
    }
}

Example 92 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project tez by apache.

the class TestDAGAppMaster method testDagCredentials.

@SuppressWarnings("deprecation")
private void testDagCredentials(boolean doMerge) throws IOException {
    TezConfiguration conf = new TezConfiguration();
    conf.setBoolean(TezConfiguration.TEZ_AM_CREDENTIALS_MERGE, doMerge);
    conf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
    conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, TEST_DIR.toString());
    ApplicationId appId = ApplicationId.newInstance(1, 1);
    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
    // create some sample AM credentials
    Credentials amCreds = new Credentials();
    JobTokenSecretManager jtsm = new JobTokenSecretManager();
    JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(appId.toString()));
    Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier, jtsm);
    sessionToken.setService(identifier.getJobId());
    TokenCache.setSessionToken(sessionToken, amCreds);
    TestTokenSecretManager ttsm = new TestTokenSecretManager();
    Text tokenAlias1 = new Text("alias1");
    Token<TestTokenIdentifier> amToken1 = new Token<TestTokenIdentifier>(new TestTokenIdentifier(new Text("amtoken1")), ttsm);
    amCreds.addToken(tokenAlias1, amToken1);
    Text tokenAlias2 = new Text("alias2");
    Token<TestTokenIdentifier> amToken2 = new Token<TestTokenIdentifier>(new TestTokenIdentifier(new Text("amtoken2")), ttsm);
    amCreds.addToken(tokenAlias2, amToken2);
    FileSystem fs = FileSystem.getLocal(conf);
    FSDataOutputStream sessionJarsPBOutStream = TezCommonUtils.createFileForAM(fs, new Path(TEST_DIR.toString(), TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME));
    DAGProtos.PlanLocalResourcesProto.getDefaultInstance().writeDelimitedTo(sessionJarsPBOutStream);
    sessionJarsPBOutStream.close();
    DAGAppMaster am = new DAGAppMaster(attemptId, ContainerId.newInstance(attemptId, 1), "127.0.0.1", 0, 0, new SystemClock(), 1, true, TEST_DIR.toString(), new String[] { TEST_DIR.toString() }, new String[] { TEST_DIR.toString() }, new TezApiVersionInfo().getVersion(), amCreds, "someuser", null);
    am.init(conf);
    am.start();
    // create some sample DAG credentials
    Credentials dagCreds = new Credentials();
    Token<TestTokenIdentifier> dagToken1 = new Token<TestTokenIdentifier>(new TestTokenIdentifier(new Text("dagtoken1")), ttsm);
    dagCreds.addToken(tokenAlias2, dagToken1);
    Text tokenAlias3 = new Text("alias3");
    Token<TestTokenIdentifier> dagToken2 = new Token<TestTokenIdentifier>(new TestTokenIdentifier(new Text("dagtoken2")), ttsm);
    dagCreds.addToken(tokenAlias3, dagToken2);
    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    DAGPlan dagPlan = DAGPlan.newBuilder().setName("somedag").setCredentialsBinary(DagTypeConverters.convertCredentialsToProto(dagCreds)).build();
    DAGImpl dag = am.createDAG(dagPlan, dagId);
    Credentials fetchedDagCreds = dag.getCredentials();
    am.stop();
    Token<? extends TokenIdentifier> fetchedToken1 = fetchedDagCreds.getToken(tokenAlias1);
    if (doMerge) {
        assertNotNull("AM creds missing from DAG creds", fetchedToken1);
        compareTestTokens(amToken1, fetchedDagCreds.getToken(tokenAlias1));
    } else {
        assertNull("AM creds leaked to DAG creds", fetchedToken1);
    }
    compareTestTokens(dagToken1, fetchedDagCreds.getToken(tokenAlias2));
    compareTestTokens(dagToken2, fetchedDagCreds.getToken(tokenAlias3));
}

Example 93 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project tez by apache.

the class TestAMContainer method testCredentialsTransfer.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testCredentialsTransfer() {
    WrappedContainerMultipleDAGs wc = new WrappedContainerMultipleDAGs();
    TezDAGID dagID2 = TezDAGID.getInstance("800", 500, 2);
    TezDAGID dagID3 = TezDAGID.getInstance("800", 500, 3);
    TezVertexID vertexID2 = TezVertexID.getInstance(dagID2, 1);
    TezVertexID vertexID3 = TezVertexID.getInstance(dagID3, 1);
    TezTaskID taskID2 = TezTaskID.getInstance(vertexID2, 1);
    TezTaskID taskID3 = TezTaskID.getInstance(vertexID3, 1);
    TezTaskAttemptID attempt11 = TezTaskAttemptID.getInstance(wc.taskID, 200);
    TezTaskAttemptID attempt12 = TezTaskAttemptID.getInstance(wc.taskID, 300);
    TezTaskAttemptID attempt21 = TezTaskAttemptID.getInstance(taskID2, 200);
    TezTaskAttemptID attempt22 = TezTaskAttemptID.getInstance(taskID2, 300);
    TezTaskAttemptID attempt31 = TezTaskAttemptID.getInstance(taskID3, 200);
    TezTaskAttemptID attempt32 = TezTaskAttemptID.getInstance(taskID3, 300);
    Map<String, LocalResource> LRs = new HashMap<String, LocalResource>();
    AMContainerTask fetchedTask = null;
    ArgumentCaptor<AMContainerTask> argumentCaptor = null;
    Token<TokenIdentifier> amGenToken = mock(Token.class);
    Token<TokenIdentifier> token1 = mock(Token.class);
    Token<TokenIdentifier> token3 = mock(Token.class);
    Credentials containerCredentials = new Credentials();
    TokenCache.setSessionToken(amGenToken, containerCredentials);
    Text token1Name = new Text("tokenDag1");
    Text token3Name = new Text("tokenDag3");
    Credentials dag1Credentials = new Credentials();
    dag1Credentials.addToken(new Text(token1Name), token1);
    Credentials dag3Credentials = new Credentials();
    dag3Credentials.addToken(new Text(token3Name), token3);
    wc.launchContainer(new HashMap<String, LocalResource>(), containerCredentials);
    wc.containerLaunched();
    wc.assignTaskAttempt(attempt11, LRs, dag1Credentials);
    argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
    verify(wc.tal, times(1)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID), eq(0));
    fetchedTask = argumentCaptor.getAllValues().get(0);
    assertTrue(fetchedTask.haveCredentialsChanged());
    assertNotNull(fetchedTask.getCredentials());
    assertNotNull(fetchedTask.getCredentials().getToken(token1Name));
    wc.taskAttemptSucceeded(attempt11);
    wc.assignTaskAttempt(attempt12, LRs, dag1Credentials);
    argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
    verify(wc.tal, times(2)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID), eq(0));
    fetchedTask = argumentCaptor.getAllValues().get(1);
    assertFalse(fetchedTask.haveCredentialsChanged());
    assertNull(fetchedTask.getCredentials());
    wc.taskAttemptSucceeded(attempt12);
    // Move to running a second DAG, with no credentials.
    wc.setNewDAGID(dagID2);
    wc.assignTaskAttempt(attempt21, LRs, null);
    argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
    verify(wc.tal, times(3)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID), eq(0));
    fetchedTask = argumentCaptor.getAllValues().get(2);
    assertTrue(fetchedTask.haveCredentialsChanged());
    assertNull(fetchedTask.getCredentials());
    wc.taskAttemptSucceeded(attempt21);
    wc.assignTaskAttempt(attempt22, LRs, null);
    argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
    verify(wc.tal, times(4)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID), eq(0));
    fetchedTask = argumentCaptor.getAllValues().get(3);
    assertFalse(fetchedTask.haveCredentialsChanged());
    assertNull(fetchedTask.getCredentials());
    wc.taskAttemptSucceeded(attempt22);
    // Move to running a third DAG, with Credentials this time
    wc.setNewDAGID(dagID3);
    wc.assignTaskAttempt(attempt31, LRs, dag3Credentials);
    argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
    verify(wc.tal, times(5)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID), eq(0));
    fetchedTask = argumentCaptor.getAllValues().get(4);
    assertTrue(fetchedTask.haveCredentialsChanged());
    assertNotNull(fetchedTask.getCredentials());
    assertNotNull(fetchedTask.getCredentials().getToken(token3Name));
    assertNull(fetchedTask.getCredentials().getToken(token1Name));
    wc.taskAttemptSucceeded(attempt31);
    wc.assignTaskAttempt(attempt32, LRs, dag1Credentials);
    argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
    verify(wc.tal, times(6)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID), eq(0));
    fetchedTask = argumentCaptor.getAllValues().get(5);
    assertFalse(fetchedTask.haveCredentialsChanged());
    assertNull(fetchedTask.getCredentials());
    wc.taskAttemptSucceeded(attempt32);
}

Example 94 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project tez by apache.

the class TestHistoryEventHandler method testLogLevelWithRecovery.

private void testLogLevelWithRecovery(HistoryLogLevel level, int expectedCount) {
    HistoryEventHandler handler = createHandler(level);
    InMemoryHistoryLoggingService.events.clear();
    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    List<DAGHistoryEvent> events = makeHistoryEvents(dagId, handler.getConfig());
    events.set(1, new DAGHistoryEvent(dagId, new DAGRecoveredEvent(attemptId, dagId, "test", user, 0, null)));
    for (DAGHistoryEvent event : events) {
        handler.handle(event);
    }
    assertEquals("Failed for level: " + level, expectedCount, InMemoryHistoryLoggingService.events.size());
    handler.stop();
}

Example 95 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project tez by apache.

the class TestDAGUtils method testConvertDAGPlanToATSMap.

@Test(timeout = 5000)
@SuppressWarnings("unchecked")
public void testConvertDAGPlanToATSMap() throws IOException, JSONException {
    DAGPlan dagPlan = createDAG();
    Map<String, TezVertexID> idNameMap = new HashMap<String, TezVertexID>();
    ApplicationId appId = ApplicationId.newInstance(1, 1);
    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    TezVertexID vId1 = TezVertexID.getInstance(dagId, 1);
    TezVertexID vId2 = TezVertexID.getInstance(dagId, 2);
    TezVertexID vId3 = TezVertexID.getInstance(dagId, 3);
    idNameMap.put("vertex1", vId1);
    idNameMap.put("vertex2", vId2);
    idNameMap.put("vertex3", vId3);
    Map<String, Object> atsMap = DAGUtils.convertDAGPlanToATSMap(dagPlan);
    Assert.assertTrue(atsMap.containsKey(DAGUtils.DAG_NAME_KEY));
    Assert.assertEquals("testDag", atsMap.get(DAGUtils.DAG_NAME_KEY));
    Assert.assertTrue(atsMap.containsKey(DAGUtils.DAG_INFO_KEY));
    Assert.assertTrue(atsMap.containsKey(DAGUtils.DAG_CONTEXT_KEY));
    Map<String, String> contextMap = (Map<String, String>) atsMap.get(DAGUtils.DAG_CONTEXT_KEY);
    Assert.assertEquals("context1", contextMap.get(ATSConstants.CONTEXT));
    Assert.assertEquals("callerId1", contextMap.get(ATSConstants.CALLER_ID));
    Assert.assertEquals("callerType1", contextMap.get(ATSConstants.CALLER_TYPE));
    Assert.assertEquals("desc1", contextMap.get(ATSConstants.DESCRIPTION));
    Assert.assertEquals("dagInfo", atsMap.get(DAGUtils.DAG_INFO_KEY));
    Assert.assertEquals(dagPlan.getName(), atsMap.get(DAGUtils.DAG_NAME_KEY));
    Assert.assertTrue(atsMap.containsKey("version"));
    Assert.assertEquals(2, atsMap.get("version"));
    Assert.assertTrue(atsMap.containsKey(DAGUtils.VERTICES_KEY));
    Assert.assertTrue(atsMap.containsKey(DAGUtils.EDGES_KEY));
    Assert.assertTrue(atsMap.containsKey(DAGUtils.VERTEX_GROUPS_KEY));
    Assert.assertEquals(3, ((Collection<?>) atsMap.get(DAGUtils.VERTICES_KEY)).size());
    Set<String> inEdgeIds = new HashSet<String>();
    Set<String> outEdgeIds = new HashSet<String>();
    int additionalInputCount = 0;
    int additionalOutputCount = 0;
    for (Object o : ((Collection<?>) atsMap.get(DAGUtils.VERTICES_KEY))) {
        Map<String, Object> v = (Map<String, Object>) o;
        Assert.assertTrue(v.containsKey(DAGUtils.VERTEX_NAME_KEY));
        String vName = (String) v.get(DAGUtils.VERTEX_NAME_KEY);
        Assert.assertTrue(v.containsKey(DAGUtils.PROCESSOR_CLASS_KEY));
        Assert.assertTrue(v.containsKey(DAGUtils.USER_PAYLOAD_AS_TEXT));
        if (v.containsKey(DAGUtils.IN_EDGE_IDS_KEY)) {
            inEdgeIds.addAll(((Collection<String>) v.get(DAGUtils.IN_EDGE_IDS_KEY)));
        }
        if (v.containsKey(DAGUtils.OUT_EDGE_IDS_KEY)) {
            outEdgeIds.addAll(((Collection<String>) v.get(DAGUtils.OUT_EDGE_IDS_KEY)));
        }
        Assert.assertTrue(idNameMap.containsKey(vName));
        String procPayload = vName + " Processor HistoryText";
        Assert.assertEquals(procPayload, v.get(DAGUtils.USER_PAYLOAD_AS_TEXT));
        if (v.containsKey(DAGUtils.ADDITIONAL_INPUTS_KEY)) {
            additionalInputCount += ((Collection<?>) v.get(DAGUtils.ADDITIONAL_INPUTS_KEY)).size();
            for (Object input : ((Collection<?>) v.get(DAGUtils.ADDITIONAL_INPUTS_KEY))) {
                Map<String, Object> inputMap = (Map<String, Object>) input;
                Assert.assertTrue(inputMap.containsKey(DAGUtils.NAME_KEY));
                Assert.assertTrue(inputMap.containsKey(DAGUtils.CLASS_KEY));
                Assert.assertFalse(inputMap.containsKey(DAGUtils.INITIALIZER_KEY));
                Assert.assertEquals("input HistoryText", inputMap.get(DAGUtils.USER_PAYLOAD_AS_TEXT));
            }
        }
        if (v.containsKey(DAGUtils.ADDITIONAL_OUTPUTS_KEY)) {
            additionalOutputCount += ((Collection<?>) v.get(DAGUtils.ADDITIONAL_OUTPUTS_KEY)).size();
            for (Object output : ((Collection<?>) v.get(DAGUtils.ADDITIONAL_OUTPUTS_KEY))) {
                Map<String, Object> outputMap = (Map<String, Object>) output;
                Assert.assertTrue(outputMap.containsKey(DAGUtils.NAME_KEY));
                Assert.assertTrue(outputMap.containsKey(DAGUtils.CLASS_KEY));
                Assert.assertTrue(outputMap.containsKey(DAGUtils.INITIALIZER_KEY));
                Assert.assertEquals("uvOut HistoryText", outputMap.get(DAGUtils.USER_PAYLOAD_AS_TEXT));
            }
        }
    }
    // 1 input
    Assert.assertEquals(1, additionalInputCount);
    // 3 outputs due to vertex group
    Assert.assertEquals(3, additionalOutputCount);
    // 1 edge translates to 2 due to vertex group
    Assert.assertEquals(2, inEdgeIds.size());
    Assert.assertEquals(2, outEdgeIds.size());
    for (Object o : ((Collection<?>) atsMap.get(DAGUtils.EDGES_KEY))) {
        Map<String, Object> e = (Map<String, Object>) o;
        Assert.assertTrue(inEdgeIds.contains(e.get(DAGUtils.EDGE_ID_KEY)));
        Assert.assertTrue(outEdgeIds.contains(e.get(DAGUtils.EDGE_ID_KEY)));
        Assert.assertTrue(e.containsKey(DAGUtils.INPUT_VERTEX_NAME_KEY));
        Assert.assertTrue(e.containsKey(DAGUtils.OUTPUT_VERTEX_NAME_KEY));
        Assert.assertEquals(DataMovementType.SCATTER_GATHER.name(), e.get(DAGUtils.DATA_MOVEMENT_TYPE_KEY));
        Assert.assertEquals(DataSourceType.PERSISTED.name(), e.get(DAGUtils.DATA_SOURCE_TYPE_KEY));
        Assert.assertEquals(SchedulingType.SEQUENTIAL.name(), e.get(DAGUtils.SCHEDULING_TYPE_KEY));
        Assert.assertEquals("dummy output class", e.get(DAGUtils.EDGE_SOURCE_CLASS_KEY));
        Assert.assertEquals("dummy input class", e.get(DAGUtils.EDGE_DESTINATION_CLASS_KEY));
        Assert.assertEquals("Dummy History Text", e.get(DAGUtils.OUTPUT_USER_PAYLOAD_AS_TEXT));
        Assert.assertEquals("Dummy History Text", e.get(DAGUtils.INPUT_USER_PAYLOAD_AS_TEXT));
    }
    for (Object o : ((Collection<?>) atsMap.get(DAGUtils.VERTEX_GROUPS_KEY))) {
        Map<String, Object> e = (Map<String, Object>) o;
        Assert.assertEquals("uv12", e.get(DAGUtils.VERTEX_GROUP_NAME_KEY));
        Assert.assertTrue(e.containsKey(DAGUtils.VERTEX_GROUP_MEMBERS_KEY));
        Assert.assertTrue(e.containsKey(DAGUtils.VERTEX_GROUP_OUTPUTS_KEY));
        Assert.assertTrue(e.containsKey(DAGUtils.VERTEX_GROUP_EDGE_MERGED_INPUTS_KEY));
    }
}