Examples with LlapServiceInstance org.apache.hadoop.hive.llap.registry.LlapServiceInstance used on opensource projects

Example 21 with LlapServiceInstance

use of org.apache.hadoop.hive.llap.registry.LlapServiceInstance in project hive by apache.

the class LlapTaskSchedulerService method getTotalResources.

@Override
public Resource getTotalResources() {
    int memory = 0;
    int vcores = 0;
    readLock.lock();
    try {
        int numInstancesFound = 0;
        for (LlapServiceInstance inst : activeInstances.getAll()) {
            Resource r = inst.getResource();
            memory += r.getMemory();
            vcores += r.getVirtualCores();
            numInstancesFound++;
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("GetTotalResources: numInstancesFound={}, totalMem={}, totalVcores={}", numInstancesFound, memory, vcores);
        }
    } finally {
        readLock.unlock();
    }
    return Resource.newInstance(memory, vcores);
}

Example 22 with LlapServiceInstance

use of org.apache.hadoop.hive.llap.registry.LlapServiceInstance in project hive by apache.

the class LlapTaskSchedulerService method start.

@Override
public void start() throws IOException {
    if (pluginEndpoint != null) {
        pluginEndpoint.start();
    }
    writeLock.lock();
    try {
        scheduledLoggingExecutor.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                readLock.lock();
                try {
                    if (dagRunning) {
                        LOG.info("Stats for current dag: {}", dagStats);
                    }
                } finally {
                    readLock.unlock();
                }
            }
        }, 0, 10000L, TimeUnit.MILLISECONDS);
        nodeEnablerFuture = nodeEnabledExecutor.submit(nodeEnablerCallable);
        Futures.addCallback(nodeEnablerFuture, new LoggingFutureCallback("NodeEnablerThread", LOG), MoreExecutors.directExecutor());
        delayedTaskSchedulerFuture = delayedTaskSchedulerExecutor.submit(delayedTaskSchedulerCallable);
        Futures.addCallback(delayedTaskSchedulerFuture, new LoggingFutureCallback("DelayedTaskSchedulerThread", LOG), MoreExecutors.directExecutor());
        preemptTaskSchedulerFuture = preemptSchedulerExecutor.submit(preemptSchedulerCallable);
        Futures.addCallback(preemptTaskSchedulerFuture, new LoggingFutureCallback("PreemptTaskSchedulerThread", LOG), MoreExecutors.directExecutor());
        schedulerFuture = schedulerExecutor.submit(schedulerCallable);
        Futures.addCallback(schedulerFuture, new LoggingFutureCallback("SchedulerThread", LOG), MoreExecutors.directExecutor());
        registry.start();
        activeInstances = registry.getInstances();
        registry.registerStateChangeListener(new NodeStateChangeListener());
        for (LlapServiceInstance inst : activeInstances.getAll()) {
            registerAndAddNode(new NodeInfo(inst, nodeBlacklistConf, clock, numSchedulableTasksPerNode, metrics), inst);
        }
        if (amRegistry != null) {
            amRegistry.start();
            int pluginPort = pluginEndpoint != null ? pluginEndpoint.getActualPort() : -1;
            amRegistry.register(amPort, pluginPort, HiveConf.getVar(conf, ConfVars.HIVESESSIONID), serializedToken, jobIdForToken, 0);
        }
    } finally {
        writeLock.unlock();
    }
}

Example 23 with LlapServiceInstance

use of org.apache.hadoop.hive.llap.registry.LlapServiceInstance in project hive by apache.

the class LlapTaskSchedulerService method getResourceAvailability.

private Pair<Resource, Map<String, List<NodeInfo>>> getResourceAvailability() {
    int memory = 0;
    int vcores = 0;
    int numInstancesFound = 0;
    Map<String, List<NodeInfo>> availableHostMap;
    readLock.lock();
    try {
        // maintain insertion order (needed for Next slot in locality miss)
        availableHostMap = new LinkedHashMap<>(instanceToNodeMap.size());
        Collection<LlapServiceInstance> instances = consistentSplits ? // might also include Inactive instances
        activeInstances.getAllInstancesOrdered(true) : // if consistent splits are NOT used we don't need the ordering as there will be no cache benefit anyways
        activeInstances.getAll();
        boolean foundSlot = false;
        for (LlapServiceInstance inst : instances) {
            NodeInfo nodeInfo = instanceToNodeMap.get(inst.getWorkerIdentity());
            if (nodeInfo != null) {
                List<NodeInfo> hostList = availableHostMap.get(nodeInfo.getHost());
                if (hostList == null) {
                    hostList = new ArrayList<>();
                    availableHostMap.put(nodeInfo.getHost(), hostList);
                }
                if (!(inst instanceof InactiveServiceInstance)) {
                    Resource r = inst.getResource();
                    memory += r.getMemory();
                    vcores += r.getVirtualCores();
                    numInstancesFound++;
                    // Hosts, however, exist even for nodes that do not currently have resources
                    if (nodeInfo.canAcceptTask()) {
                        foundSlot = true;
                        hostList.add(nodeInfo);
                    }
                }
            } else {
                LOG.warn("Null NodeInfo when attempting to get available resources for " + inst.getWorkerIdentity());
            }
        }
        // set it false here to bail out early when we know there are no resources available.
        if (!foundSlot) {
            isClusterCapacityFull.set(true);
        }
    } finally {
        readLock.unlock();
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Available resources: numInstancesFound={}, totalMem={}, totalVcores={} availableHosts: {}", numInstancesFound, memory, vcores, availableHostMap.size());
    }
    return new ImmutablePair<>(Resource.newInstance(memory, vcores), availableHostMap);
}

Example 24 with LlapServiceInstance

use of org.apache.hadoop.hive.llap.registry.LlapServiceInstance in project hive by apache.

the class BlacklistingLlapMetricsListener method setCapacity.

protected void setCapacity(String workerIdentity, int newExecutorNum, int newWaitQueueSize) throws IOException, ServiceException {
    long currentTime = System.currentTimeMillis();
    if (currentTime > nextCheckTime) {
        LlapZookeeperRegistryImpl.ConfigChangeLockResult lockResult = registry.lockForConfigChange(currentTime, currentTime + this.minConfigChangeDelayMs);
        LOG.debug("Got result for lock check: {}", lockResult);
        if (lockResult.isSuccess()) {
            LOG.info("Setting capacity for workerIdentity={} to newExecutorNum={}, newWaitQueueSize={}", workerIdentity, newExecutorNum, newWaitQueueSize);
            LlapServiceInstance serviceInstance = registry.getInstances().getInstance(workerIdentity);
            LlapManagementProtocolClientImpl client = clientFactory.create(serviceInstance);
            client.setCapacity(null, SetCapacityRequestProto.newBuilder().setExecutorNum(newExecutorNum).setQueueSize(newWaitQueueSize).build());
        }
        if (lockResult.getNextConfigChangeTime() > -1L) {
            nextCheckTime = lockResult.getNextConfigChangeTime();
        }
    } else {
        LOG.debug("Skipping check. Current time {} and we are waiting for {}.", currentTime, nextCheckTime);
    }
}

Example 25 with LlapServiceInstance

use of org.apache.hadoop.hive.llap.registry.LlapServiceInstance in project hive by apache.

the class TestUtils method testGetSplitLocationProvider.

@Test
public void testGetSplitLocationProvider() throws IOException, URISyntaxException {
    // Create test LlapServiceInstances to make sure that we can handle all of the instance types
    List<LlapServiceInstance> instances = new ArrayList<>(3);
    // Set 1 inactive instance to make sure that this does not cause problem for us
    LlapServiceInstance inactive = new InactiveServiceInstance(INACTIVE);
    instances.add(inactive);
    HiveConf conf = new HiveConf();
    conf.set(HiveConf.ConfVars.HIVE_ZOOKEEPER_QUORUM.varname, "localhost");
    LlapZookeeperRegistryImpl dynRegistry = new LlapZookeeperRegistryImpl("dyn", conf);
    Endpoint rpcEndpoint = RegistryTypeUtils.ipcEndpoint("llap", new InetSocketAddress(ACTIVE, 4000));
    Endpoint shuffle = RegistryTypeUtils.ipcEndpoint("shuffle", new InetSocketAddress(ACTIVE, 4000));
    Endpoint mng = RegistryTypeUtils.ipcEndpoint("llapmng", new InetSocketAddress(ACTIVE, 4000));
    Endpoint outputFormat = RegistryTypeUtils.ipcEndpoint("llapoutputformat", new InetSocketAddress(ACTIVE, 4000));
    Endpoint services = RegistryTypeUtils.webEndpoint("services", new URI(ACTIVE + ":4000"));
    // Set 1 active instance
    ServiceRecord enabledSrv = new ServiceRecord();
    enabledSrv.addInternalEndpoint(rpcEndpoint);
    enabledSrv.addInternalEndpoint(shuffle);
    enabledSrv.addInternalEndpoint(mng);
    enabledSrv.addInternalEndpoint(outputFormat);
    enabledSrv.addExternalEndpoint(services);
    enabledSrv.set(LlapRegistryService.LLAP_DAEMON_NUM_ENABLED_EXECUTORS, 10);
    enabledSrv.set(HiveConf.ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.varname, 100);
    LlapZookeeperRegistryImpl.DynamicServiceInstance dynamic = dynRegistry.new DynamicServiceInstance(enabledSrv);
    instances.add(dynamic);
    // Set 1 instance with 0 executors
    ServiceRecord disabledSrv = new ServiceRecord(enabledSrv);
    disabledSrv.set(LlapRegistryService.LLAP_DAEMON_NUM_ENABLED_EXECUTORS, 0);
    LlapZookeeperRegistryImpl.DynamicServiceInstance disabled = dynRegistry.new DynamicServiceInstance(disabledSrv);
    disabled.setHost(DISABLED);
    instances.add(disabled);
    when(mockRegistry.getInstances()).thenReturn(mockInstanceSet);
    when(mockInstanceSet.getAllInstancesOrdered(anyBoolean())).thenReturn(instances);
    SplitLocationProvider provider = Utils.getCustomSplitLocationProvider(mockRegistry, LOG);
    assertLocations((HostAffinitySplitLocationProvider) provider, new String[] { ACTIVE });
    // Check if fixed stuff is working as well
    LlapFixedRegistryImpl fixRegistry = new LlapFixedRegistryImpl("llap", new HiveConf());
    // Instance for testing fixed registry instances
    LlapServiceInstance fixed = fixRegistry.new FixedServiceInstance(FIXED);
    instances.remove(dynamic);
    instances.add(fixed);
    provider = Utils.getCustomSplitLocationProvider(mockRegistry, LOG);
    assertLocations((HostAffinitySplitLocationProvider) provider, new String[] { FIXED });
}