Examples with AuxiliaryService org.apache.hadoop.yarn.server.api.AuxiliaryService used on opensource projects

Example 1 with AuxiliaryService

use of org.apache.hadoop.yarn.server.api.AuxiliaryService in project hadoop by apache.

the class AuxServices method serviceStart.

@Override
public void serviceStart() throws Exception {
    //      monitor for health, shutdown/restart(?) if any should die
    for (Map.Entry<String, AuxiliaryService> entry : serviceMap.entrySet()) {
        AuxiliaryService service = entry.getValue();
        String name = entry.getKey();
        service.start();
        service.registerServiceListener(this);
        ByteBuffer meta = service.getMetaData();
        if (meta != null) {
            serviceMetaData.put(name, meta);
        }
    }
    super.serviceStart();
}

Example 2 with AuxiliaryService

use of org.apache.hadoop.yarn.server.api.AuxiliaryService in project hadoop by apache.

the class AuxiliaryServiceWithCustomClassLoader method getInstance.

public static AuxiliaryServiceWithCustomClassLoader getInstance(Configuration conf, String className, String appClassPath) throws IOException, ClassNotFoundException {
    String[] systemClasses = conf.getTrimmedStrings(String.format(YarnConfiguration.NM_AUX_SERVICES_SYSTEM_CLASSES, className));
    ClassLoader customClassLoader = createAuxServiceClassLoader(appClassPath, systemClasses);
    Class<?> clazz = Class.forName(className, true, customClassLoader);
    Class<? extends AuxiliaryService> sClass = clazz.asSubclass(AuxiliaryService.class);
    AuxiliaryService wrapped = ReflectionUtils.newInstance(sClass, conf);
    return new AuxiliaryServiceWithCustomClassLoader(className + " with custom class loader", wrapped, customClassLoader);
}

Example 3 with AuxiliaryService

use of org.apache.hadoop.yarn.server.api.AuxiliaryService in project hadoop by apache.

the class TestAuxServices method testAuxEventDispatch.

@Test
public void testAuxEventDispatch() {
    Configuration conf = new Configuration();
    conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
    conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"), ServiceA.class, Service.class);
    conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"), ServiceB.class, Service.class);
    conf.setInt("A.expected.init", 1);
    conf.setInt("B.expected.stop", 1);
    final AuxServices aux = new AuxServices();
    aux.init(conf);
    aux.start();
    ApplicationId appId1 = ApplicationId.newInstance(0, 65);
    ByteBuffer buf = ByteBuffer.allocate(6);
    buf.putChar('A');
    buf.putInt(65);
    buf.flip();
    AuxServicesEvent event = new AuxServicesEvent(AuxServicesEventType.APPLICATION_INIT, "user0", appId1, "Asrv", buf);
    aux.handle(event);
    ApplicationId appId2 = ApplicationId.newInstance(0, 66);
    event = new AuxServicesEvent(AuxServicesEventType.APPLICATION_STOP, "user0", appId2, "Bsrv", null);
    // verify all services got the stop event 
    aux.handle(event);
    Collection<AuxiliaryService> servs = aux.getServices();
    for (AuxiliaryService serv : servs) {
        ArrayList<Integer> appIds = ((LightService) serv).getAppIdsStopped();
        assertEquals("app not properly stopped", 1, appIds.size());
        assertTrue("wrong app stopped", appIds.contains((Integer) 66));
    }
    for (AuxiliaryService serv : servs) {
        assertNull(((LightService) serv).containerId);
        assertNull(((LightService) serv).resource);
    }
    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId1, 1);
    ContainerTokenIdentifier cti = new ContainerTokenIdentifier(ContainerId.newContainerId(attemptId, 1), "", "", Resource.newInstance(1, 1), 0, 0, 0, Priority.newInstance(0), 0);
    Context context = mock(Context.class);
    Container container = new ContainerImpl(new YarnConfiguration(), null, null, null, null, cti, context);
    ContainerId containerId = container.getContainerId();
    Resource resource = container.getResource();
    event = new AuxServicesEvent(AuxServicesEventType.CONTAINER_INIT, container);
    aux.handle(event);
    for (AuxiliaryService serv : servs) {
        assertEquals(containerId, ((LightService) serv).containerId);
        assertEquals(resource, ((LightService) serv).resource);
        ((LightService) serv).containerId = null;
        ((LightService) serv).resource = null;
    }
    event = new AuxServicesEvent(AuxServicesEventType.CONTAINER_STOP, container);
    aux.handle(event);
    for (AuxiliaryService serv : servs) {
        assertEquals(containerId, ((LightService) serv).containerId);
        assertEquals(resource, ((LightService) serv).resource);
    }
}

Example 4 with AuxiliaryService

use of org.apache.hadoop.yarn.server.api.AuxiliaryService in project hadoop by apache.

the class AuxServices method handle.

@Override
public void handle(AuxServicesEvent event) {
    LOG.info("Got event " + event.getType() + " for appId " + event.getApplicationID());
    switch(event.getType()) {
        case APPLICATION_INIT:
            LOG.info("Got APPLICATION_INIT for service " + event.getServiceID());
            AuxiliaryService service = null;
            try {
                service = serviceMap.get(event.getServiceID());
                service.initializeApplication(new ApplicationInitializationContext(event.getUser(), event.getApplicationID(), event.getServiceData()));
            } catch (Throwable th) {
                logWarningWhenAuxServiceThrowExceptions(service, AuxServicesEventType.APPLICATION_INIT, th);
            }
            break;
        case APPLICATION_STOP:
            for (AuxiliaryService serv : serviceMap.values()) {
                try {
                    serv.stopApplication(new ApplicationTerminationContext(event.getApplicationID()));
                } catch (Throwable th) {
                    logWarningWhenAuxServiceThrowExceptions(serv, AuxServicesEventType.APPLICATION_STOP, th);
                }
            }
            break;
        case CONTAINER_INIT:
            for (AuxiliaryService serv : serviceMap.values()) {
                try {
                    serv.initializeContainer(new ContainerInitializationContext(event.getUser(), event.getContainer().getContainerId(), event.getContainer().getResource(), event.getContainer().getContainerTokenIdentifier().getContainerType()));
                } catch (Throwable th) {
                    logWarningWhenAuxServiceThrowExceptions(serv, AuxServicesEventType.CONTAINER_INIT, th);
                }
            }
            break;
        case CONTAINER_STOP:
            for (AuxiliaryService serv : serviceMap.values()) {
                try {
                    serv.stopContainer(new ContainerTerminationContext(event.getUser(), event.getContainer().getContainerId(), event.getContainer().getResource(), event.getContainer().getContainerTokenIdentifier().getContainerType()));
                } catch (Throwable th) {
                    logWarningWhenAuxServiceThrowExceptions(serv, AuxServicesEventType.CONTAINER_STOP, th);
                }
            }
            break;
        default:
            throw new RuntimeException("Unknown type: " + event.getType());
    }
}

Example 5 with AuxiliaryService

use of org.apache.hadoop.yarn.server.api.AuxiliaryService in project hadoop by apache.

the class AuxServices method serviceInit.

@Override
public void serviceInit(Configuration conf) throws Exception {
    final FsPermission storeDirPerms = new FsPermission((short) 0700);
    Path stateStoreRoot = null;
    FileSystem stateStoreFs = null;
    boolean recoveryEnabled = conf.getBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, YarnConfiguration.DEFAULT_NM_RECOVERY_ENABLED);
    if (recoveryEnabled) {
        stateStoreRoot = new Path(conf.get(YarnConfiguration.NM_RECOVERY_DIR), STATE_STORE_ROOT_NAME);
        stateStoreFs = FileSystem.getLocal(conf);
    }
    Collection<String> auxNames = conf.getStringCollection(YarnConfiguration.NM_AUX_SERVICES);
    for (final String sName : auxNames) {
        try {
            Preconditions.checkArgument(validateAuxServiceName(sName), "The ServiceName: " + sName + " set in " + YarnConfiguration.NM_AUX_SERVICES + " is invalid." + "The valid service name should only contain a-zA-Z0-9_ " + "and can not start with numbers");
            String classKey = String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, sName);
            String className = conf.get(classKey);
            final String appClassPath = conf.get(String.format(YarnConfiguration.NM_AUX_SERVICES_CLASSPATH, sName));
            AuxiliaryService s = null;
            boolean useCustomerClassLoader = appClassPath != null && !appClassPath.isEmpty() && className != null && !className.isEmpty();
            if (useCustomerClassLoader) {
                s = AuxiliaryServiceWithCustomClassLoader.getInstance(conf, className, appClassPath);
                LOG.info("The aux service:" + sName + " are using the custom classloader");
            } else {
                Class<? extends AuxiliaryService> sClass = conf.getClass(classKey, null, AuxiliaryService.class);
                if (sClass == null) {
                    throw new RuntimeException("No class defined for " + sName);
                }
                s = ReflectionUtils.newInstance(sClass, conf);
            }
            if (s == null) {
                throw new RuntimeException("No object created for " + sName);
            }
            // TODO better use s.getName()?
            if (!sName.equals(s.getName())) {
                LOG.warn("The Auxiliary Service named '" + sName + "' in the " + "configuration is for " + s.getClass() + " which has " + "a name of '" + s.getName() + "'. Because these are " + "not the same tools trying to send ServiceData and read " + "Service Meta Data may have issues unless the refer to " + "the name in the config.");
            }
            addService(sName, s);
            if (recoveryEnabled) {
                Path storePath = new Path(stateStoreRoot, sName);
                stateStoreFs.mkdirs(storePath, storeDirPerms);
                s.setRecoveryPath(storePath);
            }
            s.init(conf);
        } catch (RuntimeException e) {
            LOG.fatal("Failed to initialize " + sName, e);
            throw e;
        }
    }
    super.serviceInit(conf);
}