Examples with DAGPlan org.apache.tez.dag.api.records.DAGProtos.DAGPlan used on opensource projects

Example 46 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestRecoveryParser method testRecoverableSummary_VertexInCommitting.

@Test(timeout = 5000)
public void testRecoverableSummary_VertexInCommitting() throws IOException {
    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
    TezDAGID dagID = TezDAGID.getInstance(appId, 1);
    AppContext appContext = mock(AppContext.class);
    when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1"));
    when(appContext.getClock()).thenReturn(new SystemClock());
    when(mockDAGImpl.getID()).thenReturn(dagID);
    RecoveryService rService = new RecoveryService(appContext);
    Configuration conf = new Configuration();
    conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
    rService.init(conf);
    rService.start();
    DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
    // write a DAGSubmittedEvent first to initialize summaryStream
    rService.handle(new DAGHistoryEvent(dagID, new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1), null, "user", new Configuration(), null, null)));
    // It should be fine to skip other events, just for testing.
    rService.handle(new DAGHistoryEvent(dagID, new VertexCommitStartedEvent(TezVertexID.getInstance(dagID, 0), 0L)));
    rService.stop();
    DAGRecoveryData dagData = parser.parseRecoveryData();
    assertEquals(dagID, dagData.recoveredDagID);
    assertTrue(dagData.nonRecoverable);
    assertTrue(dagData.reason.contains("Vertex Commit was in progress"));
}

Example 47 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestDAGImpl method createTestDAGPlan.

public static DAGPlan createTestDAGPlan() {
    LOG.info("Setting up dag plan");
    DAGPlan dag = DAGPlan.newBuilder().setName("testverteximpl").setDagConf(ConfigurationProto.newBuilder().addConfKeyValues(PlanKeyValuePair.newBuilder().setKey(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS).setValue(3 + ""))).addVertex(VertexPlan.newBuilder().setName("vertex1").setType(PlanVertexType.NORMAL).addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("host1").addRack("rack1").build()).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(1).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x1.y1").build()).setVertexConf(ConfigurationProto.newBuilder().addConfKeyValues(PlanKeyValuePair.newBuilder().setKey(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS).setValue(2 + ""))).addOutEdgeId("e1").build()).addVertex(VertexPlan.newBuilder().setName("vertex2").setType(PlanVertexType.NORMAL).addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("host2").addRack("rack2").build()).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(2).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x2.y2").build()).addOutEdgeId("e2").build()).addVertex(VertexPlan.newBuilder().setName("vertex3").setType(PlanVertexType.NORMAL).setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("x3.y3")).addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("host3").addRack("rack3").build()).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(2).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("foo").setTaskModule("x3.y3").build()).addInEdgeId("e1").addInEdgeId("e2").addOutEdgeId("e3").addOutEdgeId("e4").build()).addVertex(VertexPlan.newBuilder().setName("vertex4").setType(PlanVertexType.NORMAL).addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("host4").addRack("rack4").build()).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(2).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x4.y4").build()).addInEdgeId("e3").addOutEdgeId("e5").build()).addVertex(VertexPlan.newBuilder().setName("vertex5").setType(PlanVertexType.NORMAL).addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("host5").addRack("rack5").build()).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(2).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x5.y5").build()).addInEdgeId("e4").addOutEdgeId("e6").build()).addVertex(VertexPlan.newBuilder().setName("vertex6").setType(PlanVertexType.NORMAL).addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("host6").addRack("rack6").build()).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(2).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x6.y6").build()).addInEdgeId("e5").addInEdgeId("e6").build()).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3_v1")).setInputVertexName("vertex1").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o1")).setOutputVertexName("vertex3").setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER).setId("e1").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3_v2")).setInputVertexName("vertex2").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2")).setOutputVertexName("vertex3").setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER).setId("e2").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i4_v3")).setInputVertexName("vertex3").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o3_v4")).setOutputVertexName("vertex4").setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER).setId("e3").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i5_v3")).setInputVertexName("vertex3").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o3_v5")).setOutputVertexName("vertex5").setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER).setId("e4").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i6_v4")).setInputVertexName("vertex4").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o4")).setOutputVertexName("vertex6").setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER).setId("e5").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i6_v5")).setInputVertexName("vertex5").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o5")).setOutputVertexName("vertex6").setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER).setId("e6").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).build();
    return dag;
}

Example 48 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestDAGImpl method createTestMRRDAGPlan.

private DAGPlan createTestMRRDAGPlan() {
    LOG.info("Setting up MRR dag plan");
    DAGPlan dag = DAGPlan.newBuilder().setName("testverteximpl").addVertex(VertexPlan.newBuilder().setName("vertex1").setType(PlanVertexType.NORMAL).addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("host1").addRack("rack1").build()).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(1).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x1.y1").build()).addOutputs(DAGProtos.RootInputLeafOutputProto.newBuilder().setIODescriptor(TezEntityDescriptorProto.newBuilder().setClassName("output1").build()).setName("output1").setControllerDescriptor(TezEntityDescriptorProto.newBuilder().setClassName(CountingOutputCommitter.class.getName()))).addOutEdgeId("e1").build()).addVertex(VertexPlan.newBuilder().setName("vertex2").setType(PlanVertexType.NORMAL).addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("host2").addRack("rack2").build()).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(1).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x2.y2").build()).addOutputs(DAGProtos.RootInputLeafOutputProto.newBuilder().setIODescriptor(TezEntityDescriptorProto.newBuilder().setClassName("output2").build()).setName("output2").setControllerDescriptor(TezEntityDescriptorProto.newBuilder().setClassName(CountingOutputCommitter.class.getName()))).addInEdgeId("e1").addOutEdgeId("e2").build()).addVertex(VertexPlan.newBuilder().setName("vertex3").setType(PlanVertexType.NORMAL).setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("x3.y3")).addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("host3").addRack("rack3").build()).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(1).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("foo").setTaskModule("x3.y3").build()).addOutputs(DAGProtos.RootInputLeafOutputProto.newBuilder().setIODescriptor(TezEntityDescriptorProto.newBuilder().setClassName("output3").build()).setName("output3").setControllerDescriptor(TezEntityDescriptorProto.newBuilder().setClassName(CountingOutputCommitter.class.getName()))).addInEdgeId("e2").build()).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i2")).setInputVertexName("vertex1").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o1")).setOutputVertexName("vertex2").setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER).setId("e1").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3")).setInputVertexName("vertex2").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2")).setOutputVertexName("vertex3").setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER).setId("e2").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).build();
    return dag;
}

Example 49 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestDAGImpl method createDAGWithCustomEdge.

// v1 -> v2
private DAGPlan createDAGWithCustomEdge(ExceptionLocation exLocation, boolean useLegacy) {
    LOG.info("Setting up custome edge dag plan " + exLocation + " " + useLegacy);
    DAGPlan dag = DAGPlan.newBuilder().setName("testverteximpl").addVertex(VertexPlan.newBuilder().setName("vertex1").setType(PlanVertexType.NORMAL).addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("host1").addRack("rack1").build()).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(1).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x1.y1").build()).addOutEdgeId("e1").build()).addVertex(VertexPlan.newBuilder().setName("vertex2").setType(PlanVertexType.NORMAL).setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("x2.y2")).addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("host2").addRack("rack2").build()).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(2).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("foo").setTaskModule("x2.y2").build()).addInEdgeId("e1").build()).addEdge(EdgePlan.newBuilder().setEdgeManager(TezEntityDescriptorProto.newBuilder().setClassName(useLegacy ? CustomizedEdgeManagerLegacy.class.getName() : CustomizedEdgeManager.class.getName()).setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder().setUserPayload(ByteString.copyFromUtf8(exLocation.name())))).setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("v1_v2")).setInputVertexName("vertex1").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o1")).setOutputVertexName("vertex2").setDataMovementType(PlanEdgeDataMovementType.CUSTOM).setId("e1").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).build();
    return dag;
}

Example 50 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestDAGAppMaster method testDagCredentials.

@SuppressWarnings("deprecation")
private void testDagCredentials(boolean doMerge) throws IOException {
    TezConfiguration conf = new TezConfiguration();
    conf.setBoolean(TezConfiguration.TEZ_AM_CREDENTIALS_MERGE, doMerge);
    conf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
    conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, TEST_DIR.toString());
    ApplicationId appId = ApplicationId.newInstance(1, 1);
    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
    // create some sample AM credentials
    Credentials amCreds = new Credentials();
    JobTokenSecretManager jtsm = new JobTokenSecretManager();
    JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(appId.toString()));
    Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier, jtsm);
    sessionToken.setService(identifier.getJobId());
    TokenCache.setSessionToken(sessionToken, amCreds);
    TestTokenSecretManager ttsm = new TestTokenSecretManager();
    Text tokenAlias1 = new Text("alias1");
    Token<TestTokenIdentifier> amToken1 = new Token<TestTokenIdentifier>(new TestTokenIdentifier(new Text("amtoken1")), ttsm);
    amCreds.addToken(tokenAlias1, amToken1);
    Text tokenAlias2 = new Text("alias2");
    Token<TestTokenIdentifier> amToken2 = new Token<TestTokenIdentifier>(new TestTokenIdentifier(new Text("amtoken2")), ttsm);
    amCreds.addToken(tokenAlias2, amToken2);
    FileSystem fs = FileSystem.getLocal(conf);
    FSDataOutputStream sessionJarsPBOutStream = TezCommonUtils.createFileForAM(fs, new Path(TEST_DIR.toString(), TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME));
    DAGProtos.PlanLocalResourcesProto.getDefaultInstance().writeDelimitedTo(sessionJarsPBOutStream);
    sessionJarsPBOutStream.close();
    DAGAppMaster am = new DAGAppMaster(attemptId, ContainerId.newInstance(attemptId, 1), "127.0.0.1", 0, 0, new SystemClock(), 1, true, TEST_DIR.toString(), new String[] { TEST_DIR.toString() }, new String[] { TEST_DIR.toString() }, new TezApiVersionInfo().getVersion(), amCreds, "someuser", null);
    am.init(conf);
    am.start();
    // create some sample DAG credentials
    Credentials dagCreds = new Credentials();
    Token<TestTokenIdentifier> dagToken1 = new Token<TestTokenIdentifier>(new TestTokenIdentifier(new Text("dagtoken1")), ttsm);
    dagCreds.addToken(tokenAlias2, dagToken1);
    Text tokenAlias3 = new Text("alias3");
    Token<TestTokenIdentifier> dagToken2 = new Token<TestTokenIdentifier>(new TestTokenIdentifier(new Text("dagtoken2")), ttsm);
    dagCreds.addToken(tokenAlias3, dagToken2);
    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    DAGPlan dagPlan = DAGPlan.newBuilder().setName("somedag").setCredentialsBinary(DagTypeConverters.convertCredentialsToProto(dagCreds)).build();
    DAGImpl dag = am.createDAG(dagPlan, dagId);
    Credentials fetchedDagCreds = dag.getCredentials();
    am.stop();
    Token<? extends TokenIdentifier> fetchedToken1 = fetchedDagCreds.getToken(tokenAlias1);
    if (doMerge) {
        assertNotNull("AM creds missing from DAG creds", fetchedToken1);
        compareTestTokens(amToken1, fetchedDagCreds.getToken(tokenAlias1));
    } else {
        assertNull("AM creds leaked to DAG creds", fetchedToken1);
    }
    compareTestTokens(dagToken1, fetchedDagCreds.getToken(tokenAlias2));
    compareTestTokens(dagToken2, fetchedDagCreds.getToken(tokenAlias3));
}