Search in sources :

Example 1 with ApplicationMasterLauncher

use of org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher in project hadoop by apache.

the class SLSRunner method startRM.

private void startRM() throws IOException, ClassNotFoundException {
    Configuration rmConf = new YarnConfiguration();
    String schedulerClass = rmConf.get(YarnConfiguration.RM_SCHEDULER);
    // exercise/track behaviors that are not common to the scheduler api
    if (Class.forName(schedulerClass) == CapacityScheduler.class) {
        rmConf.set(YarnConfiguration.RM_SCHEDULER, SLSCapacityScheduler.class.getName());
    } else {
        rmConf.set(YarnConfiguration.RM_SCHEDULER, ResourceSchedulerWrapper.class.getName());
        rmConf.set(SLSConfiguration.RM_SCHEDULER, schedulerClass);
    }
    rmConf.set(SLSConfiguration.METRICS_OUTPUT_DIR, metricsOutputDir);
    final SLSRunner se = this;
    rm = new ResourceManager() {

        @Override
        protected ApplicationMasterLauncher createAMLauncher() {
            return new MockAMLauncher(se, this.rmContext, amMap);
        }
    };
    rm.init(rmConf);
    rm.start();
}
Also used : Configuration(org.apache.hadoop.conf.Configuration) SLSConfiguration(org.apache.hadoop.yarn.sls.conf.SLSConfiguration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) MockAMLauncher(org.apache.hadoop.yarn.sls.resourcemanager.MockAMLauncher) ApplicationMasterLauncher(org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher) ResourceManager(org.apache.hadoop.yarn.server.resourcemanager.ResourceManager) ResourceSchedulerWrapper(org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper) SLSCapacityScheduler(org.apache.hadoop.yarn.sls.scheduler.SLSCapacityScheduler)

Example 2 with ApplicationMasterLauncher

use of org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher in project hadoop by apache.

the class TestRMAppAttemptTransitions method setUp.

@SuppressWarnings("deprecation")
@Before
public void setUp() throws Exception {
    AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE;
    if (isSecurityEnabled) {
        authMethod = AuthenticationMethod.KERBEROS;
    }
    SecurityUtil.setAuthenticationMethod(authMethod, conf);
    UserGroupInformation.setConfiguration(conf);
    InlineDispatcher rmDispatcher = new InlineDispatcher();
    ContainerAllocationExpirer containerAllocationExpirer = mock(ContainerAllocationExpirer.class);
    amLivelinessMonitor = mock(AMLivelinessMonitor.class);
    amFinishingMonitor = mock(AMLivelinessMonitor.class);
    writer = mock(RMApplicationHistoryWriter.class);
    MasterKeyData masterKeyData = amRMTokenManager.createNewMasterKey();
    when(amRMTokenManager.getMasterKey()).thenReturn(masterKeyData);
    rmContext = new RMContextImpl(rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, null, amRMTokenManager, new RMContainerTokenSecretManager(conf), nmTokenManager, clientToAMTokenManager);
    store = mock(RMStateStore.class);
    ((RMContextImpl) rmContext).setStateStore(store);
    publisher = mock(SystemMetricsPublisher.class);
    rmContext.setSystemMetricsPublisher(publisher);
    rmContext.setRMApplicationHistoryWriter(writer);
    scheduler = mock(YarnScheduler.class);
    masterService = mock(ApplicationMasterService.class);
    applicationMasterLauncher = mock(ApplicationMasterLauncher.class);
    rmDispatcher.register(RMAppAttemptEventType.class, new TestApplicationAttemptEventDispatcher());
    rmDispatcher.register(RMAppEventType.class, new TestApplicationEventDispatcher());
    rmDispatcher.register(SchedulerEventType.class, new TestSchedulerEventDispatcher());
    rmDispatcher.register(AMLauncherEventType.class, new TestAMLauncherEventDispatcher());
    rmnodeEventHandler = mock(RMNodeImpl.class);
    rmDispatcher.register(RMNodeEventType.class, rmnodeEventHandler);
    rmDispatcher.init(conf);
    rmDispatcher.start();
    ApplicationId applicationId = MockApps.newAppID(appId++);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0);
    resourceScheduler = mock(ResourceScheduler.class);
    ApplicationResourceUsageReport appResUsgRpt = mock(ApplicationResourceUsageReport.class);
    when(appResUsgRpt.getMemorySeconds()).thenReturn(0L);
    when(appResUsgRpt.getVcoreSeconds()).thenReturn(0L);
    when(resourceScheduler.getAppResourceUsageReport((ApplicationAttemptId) Matchers.any())).thenReturn(appResUsgRpt);
    spyRMContext = spy(rmContext);
    Mockito.doReturn(resourceScheduler).when(spyRMContext).getScheduler();
    final String user = MockApps.newUserName();
    final String queue = MockApps.newQueue();
    submissionContext = mock(ApplicationSubmissionContext.class);
    when(submissionContext.getQueue()).thenReturn(queue);
    Resource resource = BuilderUtils.newResource(1536, 1);
    ContainerLaunchContext amContainerSpec = BuilderUtils.newContainerLaunchContext(null, null, null, null, null, null);
    when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec);
    when(submissionContext.getResource()).thenReturn(resource);
    unmanagedAM = false;
    application = mock(RMAppImpl.class);
    applicationAttempt = new RMAppAttemptImpl(applicationAttemptId, spyRMContext, scheduler, masterService, submissionContext, new Configuration(), BuilderUtils.newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1), application);
    when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt);
    when(application.getApplicationId()).thenReturn(applicationId);
    spyRMContext.getRMApps().put(application.getApplicationId(), application);
    testAppAttemptNewState();
}
Also used : ApplicationMasterService(org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) ContainerAllocationExpirer(org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer) ApplicationMasterLauncher(org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher) InlineDispatcher(org.apache.hadoop.yarn.event.InlineDispatcher) AuthenticationMethod(org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod) SystemMetricsPublisher(org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher) YarnScheduler(org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler) ApplicationSubmissionContext(org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext) ResourceScheduler(org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler) RMContainerTokenSecretManager(org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager) RMContextImpl(org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl) MasterKeyData(org.apache.hadoop.yarn.server.security.MasterKeyData) RMAppImpl(org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl) RMStateStore(org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore) Resource(org.apache.hadoop.yarn.api.records.Resource) RMApplicationHistoryWriter(org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter) ApplicationAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId) ContainerLaunchContext(org.apache.hadoop.yarn.api.records.ContainerLaunchContext) ApplicationResourceUsageReport(org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport) RMNodeImpl(org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) Before(org.junit.Before)

Example 3 with ApplicationMasterLauncher

use of org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher in project hadoop by apache.

the class MockRMWithCustomAMLauncher method createAMLauncher.

@Override
protected ApplicationMasterLauncher createAMLauncher() {
    return new ApplicationMasterLauncher(getRMContext()) {

        @Override
        protected Runnable createRunnableLauncher(RMAppAttempt application, AMLauncherEventType event) {
            return new AMLauncher(context, application, event, getConfig()) {

                @Override
                protected ContainerManagementProtocol getContainerMgrProxy(ContainerId containerId) {
                    return containerManager;
                }

                @Override
                protected Token<AMRMTokenIdentifier> createAndSetAMRMToken() {
                    Token<AMRMTokenIdentifier> amRmToken = super.createAndSetAMRMToken();
                    InetSocketAddress serviceAddr = getConfig().getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
                    SecurityUtil.setTokenService(amRmToken, serviceAddr);
                    return amRmToken;
                }
            };
        }
    };
}
Also used : RMAppAttempt(org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt) AMRMTokenIdentifier(org.apache.hadoop.yarn.security.AMRMTokenIdentifier) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) ApplicationMasterLauncher(org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher) InetSocketAddress(java.net.InetSocketAddress) AMLauncher(org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher) AMLauncherEventType(org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType)

Example 4 with ApplicationMasterLauncher

use of org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher in project hadoop by apache.

the class TestApplicationMasterLauncher method testRetriesOnFailures.

@Test
public void testRetriesOnFailures() throws Exception {
    final ContainerManagementProtocol mockProxy = mock(ContainerManagementProtocol.class);
    final StartContainersResponse mockResponse = mock(StartContainersResponse.class);
    when(mockProxy.startContainers(any(StartContainersRequest.class))).thenThrow(new NMNotYetReadyException("foo")).thenReturn(mockResponse);
    Configuration conf = new Configuration();
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
    conf.setInt(YarnConfiguration.CLIENT_NM_CONNECT_RETRY_INTERVAL_MS, 1);
    final DrainDispatcher dispatcher = new DrainDispatcher();
    MockRM rm = new MockRMWithCustomAMLauncher(conf, null) {

        @Override
        protected ApplicationMasterLauncher createAMLauncher() {
            return new ApplicationMasterLauncher(getRMContext()) {

                @Override
                protected Runnable createRunnableLauncher(RMAppAttempt application, AMLauncherEventType event) {
                    return new AMLauncher(context, application, event, getConfig()) {

                        @Override
                        protected YarnRPC getYarnRPC() {
                            YarnRPC mockRpc = mock(YarnRPC.class);
                            when(mockRpc.getProxy(any(Class.class), any(InetSocketAddress.class), any(Configuration.class))).thenReturn(mockProxy);
                            return mockRpc;
                        }
                    };
                }
            };
        }

        @Override
        protected Dispatcher createDispatcher() {
            return dispatcher;
        }
    };
    rm.start();
    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5120);
    RMApp app = rm.submitApp(2000);
    // kick the scheduling
    nm1.nodeHeartbeat(true);
    dispatcher.await();
    MockRM.waitForState(app.getCurrentAppAttempt(), RMAppAttemptState.LAUNCHED, 500);
}
Also used : DrainDispatcher(org.apache.hadoop.yarn.event.DrainDispatcher) RMApp(org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp) RMAppAttempt(org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt) StartContainersResponse(org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) ApplicationMasterLauncher(org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher) InetSocketAddress(java.net.InetSocketAddress) AMLauncherEventType(org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType) NMNotYetReadyException(org.apache.hadoop.yarn.exceptions.NMNotYetReadyException) YarnRPC(org.apache.hadoop.yarn.ipc.YarnRPC) ContainerManagementProtocol(org.apache.hadoop.yarn.api.ContainerManagementProtocol) AMLauncher(org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher) Test(org.junit.Test)

Aggregations

ApplicationMasterLauncher (org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher)4 Configuration (org.apache.hadoop.conf.Configuration)3 YarnConfiguration (org.apache.hadoop.yarn.conf.YarnConfiguration)3 InetSocketAddress (java.net.InetSocketAddress)2 AMLauncher (org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher)2 AMLauncherEventType (org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType)2 RMAppAttempt (org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt)2 AuthenticationMethod (org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod)1 ContainerManagementProtocol (org.apache.hadoop.yarn.api.ContainerManagementProtocol)1 StartContainersResponse (org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse)1 ApplicationAttemptId (org.apache.hadoop.yarn.api.records.ApplicationAttemptId)1 ApplicationId (org.apache.hadoop.yarn.api.records.ApplicationId)1 ApplicationResourceUsageReport (org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport)1 ApplicationSubmissionContext (org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext)1 ContainerId (org.apache.hadoop.yarn.api.records.ContainerId)1 ContainerLaunchContext (org.apache.hadoop.yarn.api.records.ContainerLaunchContext)1 Resource (org.apache.hadoop.yarn.api.records.Resource)1 DrainDispatcher (org.apache.hadoop.yarn.event.DrainDispatcher)1 InlineDispatcher (org.apache.hadoop.yarn.event.InlineDispatcher)1 NMNotYetReadyException (org.apache.hadoop.yarn.exceptions.NMNotYetReadyException)1