Examples with YarnConfiguration org.apache.hadoop.yarn.conf.YarnConfiguration used on opensource projects

Example 36 with YarnConfiguration

use of org.apache.hadoop.yarn.conf.YarnConfiguration in project hadoop by apache.

the class TestTimelineServiceClientIntegration method setupClass.

@BeforeClass
public static void setupClass() throws Exception {
    try {
        collectorManager = new MockNodeTimelineCollectorManager();
        conf = new YarnConfiguration();
        // enable timeline service v.2
        conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
        conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
        conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS, FileSystemTimelineWriterImpl.class, TimelineWriter.class);
        conf.set(FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT, ROOT_DIR);
        auxService = PerNodeTimelineCollectorsAuxService.launchServer(new String[0], collectorManager, conf);
        auxService.addApplication(ApplicationId.newInstance(0, 1));
    } catch (ExitUtil.ExitException e) {
        fail();
    }
}

Example 37 with YarnConfiguration

use of org.apache.hadoop.yarn.conf.YarnConfiguration in project hadoop by apache.

the class TestOverrideTimelineStoreYarnClient method testLifecycleAndOverride.

@Test
public void testLifecycleAndOverride() throws Throwable {
    YarnConfiguration conf = new YarnConfiguration();
    try (NoRMStore store = new NoRMStore()) {
        store.init(conf);
        store.start();
        Assert.assertEquals(EntityGroupFSTimelineStore.AppState.ACTIVE, store.getAppState(ApplicationId.newInstance(1, 1)));
        store.stop();
    }
}

Example 38 with YarnConfiguration

use of org.apache.hadoop.yarn.conf.YarnConfiguration in project hive by apache.

the class ATSHook method setupAtsExecutor.

private static void setupAtsExecutor(HiveConf conf) {
    synchronized (LOCK) {
        if (executor == null) {
            // The call to ATS appears to block indefinitely, blocking the ATS thread while
            // the hook continues to submit work to the ExecutorService with each query.
            // Over time the queued items can cause OOM as the HookContext seems to contain
            // some items which use a lot of memory.
            // Prevent this situation by creating executor with bounded capacity -
            // the event will not be sent to ATS if there are too many outstanding work submissions.
            int queueCapacity = conf.getIntVar(HiveConf.ConfVars.ATSHOOKQUEUECAPACITY);
            // Executor to create the ATS events.
            // This can use significant resources and should not be done on the main query thread.
            LOG.info("Creating ATS executor queue with capacity " + queueCapacity);
            BlockingQueue<Runnable> queue = new LinkedBlockingQueue<Runnable>(queueCapacity);
            ThreadFactory threadFactory = new ThreadFactoryBuilder().setDaemon(true).setNameFormat("ATS Logger %d").build();
            executor = new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, queue, threadFactory);
            // Create a separate thread to send the events.
            // Keep separate from the creating events in case the send blocks.
            BlockingQueue<Runnable> senderQueue = new LinkedBlockingQueue<Runnable>(queueCapacity);
            senderExecutor = new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, senderQueue, threadFactory);
            YarnConfiguration yarnConf = new YarnConfiguration();
            timelineClient = TimelineClient.createTimelineClient();
            timelineClient.init(yarnConf);
            timelineClient.start();
            ShutdownHookManager.addShutdownHook(new Runnable() {

                @Override
                public void run() {
                    try {
                        executor.shutdown();
                        executor.awaitTermination(WAIT_TIME, TimeUnit.SECONDS);
                        executor = null;
                    } catch (InterruptedException ie) {
                    /* ignore */
                    }
                    timelineClient.stop();
                }
            });
        }
    }
}

Example 39 with YarnConfiguration

use of org.apache.hadoop.yarn.conf.YarnConfiguration in project weave by continuuity.

the class WeaveContainerMain method main.

/**
   * Main method for launching a {@link WeaveContainerService} which runs
   * a {@link com.continuuity.weave.api.WeaveRunnable}.
   */
public static void main(final String[] args) throws Exception {
    // Try to load the secure store from localized file, which AM requested RM to localize it for this container.
    loadSecureStore();
    String zkConnectStr = System.getenv(EnvKeys.WEAVE_ZK_CONNECT);
    File weaveSpecFile = new File(Constants.Files.WEAVE_SPEC);
    RunId appRunId = RunIds.fromString(System.getenv(EnvKeys.WEAVE_APP_RUN_ID));
    RunId runId = RunIds.fromString(System.getenv(EnvKeys.WEAVE_RUN_ID));
    String runnableName = System.getenv(EnvKeys.WEAVE_RUNNABLE_NAME);
    int instanceId = Integer.parseInt(System.getenv(EnvKeys.WEAVE_INSTANCE_ID));
    int instanceCount = Integer.parseInt(System.getenv(EnvKeys.WEAVE_INSTANCE_COUNT));
    ZKClientService zkClientService = ZKClientServices.delegate(ZKClients.reWatchOnExpire(ZKClients.retryOnFailure(ZKClientService.Builder.of(zkConnectStr).build(), RetryStrategies.fixDelay(1, TimeUnit.SECONDS))));
    DiscoveryService discoveryService = new ZKDiscoveryService(zkClientService);
    WeaveSpecification weaveSpec = loadWeaveSpec(weaveSpecFile);
    renameLocalFiles(weaveSpec.getRunnables().get(runnableName));
    WeaveRunnableSpecification runnableSpec = weaveSpec.getRunnables().get(runnableName).getRunnableSpecification();
    ContainerInfo containerInfo = new EnvContainerInfo();
    Arguments arguments = decodeArgs();
    BasicWeaveContext context = new BasicWeaveContext(runId, appRunId, containerInfo.getHost(), arguments.getRunnableArguments().get(runnableName).toArray(new String[0]), arguments.getArguments().toArray(new String[0]), runnableSpec, instanceId, discoveryService, instanceCount, containerInfo.getMemoryMB(), containerInfo.getVirtualCores());
    Configuration conf = new YarnConfiguration(new HdfsConfiguration(new Configuration()));
    Service service = new WeaveContainerService(context, containerInfo, getContainerZKClient(zkClientService, appRunId, runnableName), runId, runnableSpec, getClassLoader(), createAppLocation(conf));
    new WeaveContainerMain().doMain(zkClientService, service);
}

Example 40 with YarnConfiguration

use of org.apache.hadoop.yarn.conf.YarnConfiguration in project weave by continuuity.

the class AMRMClientImpl method start.

@Override
public synchronized void start() {
    final YarnConfiguration conf = new YarnConfiguration(getConfig());
    final YarnRPC rpc = YarnRPC.create(conf);
    final InetSocketAddress rmAddress = conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
    UserGroupInformation currentUser;
    try {
        currentUser = UserGroupInformation.getCurrentUser();
    } catch (IOException e) {
        throw new YarnException(e);
    }
    if (UserGroupInformation.isSecurityEnabled()) {
        String tokenURLEncodedStr = System.getenv().get(ApplicationConstants.APPLICATION_MASTER_TOKEN_ENV_NAME);
        Token<? extends TokenIdentifier> token = new Token<TokenIdentifier>();
        try {
            token.decodeFromUrlString(tokenURLEncodedStr);
        } catch (IOException e) {
            throw new YarnException(e);
        }
        SecurityUtil.setTokenService(token, rmAddress);
        if (LOG.isDebugEnabled()) {
            LOG.debug("AppMasterToken is " + token);
        }
        currentUser.addToken(token);
    }
    rmClient = currentUser.doAs(new PrivilegedAction<AMRMProtocol>() {

        @Override
        public AMRMProtocol run() {
            return (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress, conf);
        }
    });
    LOG.debug("Connecting to ResourceManager at " + rmAddress);
    super.start();
}