Examples with LlapManagementProtocolClientImpl org.apache.hadoop.hive.llap.impl.LlapManagementProtocolClientImpl used on opensource projects

Example 1 with LlapManagementProtocolClientImpl

use of org.apache.hadoop.hive.llap.impl.LlapManagementProtocolClientImpl in project hive by apache.

the class LlapMetricsCollector method onCreate.

@Override
public void onCreate(LlapServiceInstance serviceInstance, int ephSeqVersion) {
    LlapManagementProtocolClientImpl client = clientFactory.create(serviceInstance);
    llapClients.put(serviceInstance.getWorkerIdentity(), client);
}

Example 2 with LlapManagementProtocolClientImpl

use of org.apache.hadoop.hive.llap.impl.LlapManagementProtocolClientImpl in project hive by apache.

the class TestLlapDaemonProtocolServerImpl method testSetCapacity.

@Test(timeout = 10000)
public void testSetCapacity() throws ServiceException, IOException {
    LlapDaemonConfiguration daemonConf = new LlapDaemonConfiguration();
    int numHandlers = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_RPC_NUM_HANDLERS);
    ContainerRunner containerRunnerMock = mock(ContainerRunner.class);
    when(containerRunnerMock.setCapacity(any(SetCapacityRequestProto.class))).thenReturn(SetCapacityResponseProto.newBuilder().build());
    LlapDaemonExecutorMetrics executorMetrics = LlapDaemonExecutorMetrics.create("LLAP", "SessionId", numHandlers, 1, new int[] { 30, 60, 300 }, 0, 0L, 0);
    LlapProtocolServerImpl server = new LlapProtocolServerImpl(null, numHandlers, containerRunnerMock, new AtomicReference<InetSocketAddress>(), new AtomicReference<InetSocketAddress>(), 0, 0, 0, null, executorMetrics);
    try {
        server.init(new Configuration());
        server.start();
        InetSocketAddress serverAddr = server.getManagementBindAddress();
        LlapManagementProtocolPB client = new LlapManagementProtocolClientImpl(new Configuration(), serverAddr.getHostName(), serverAddr.getPort(), null, null);
        client.setCapacity(null, SetCapacityRequestProto.newBuilder().setExecutorNum(1).setQueueSize(1).build());
    } finally {
        server.stop();
    }
}

Example 3 with LlapManagementProtocolClientImpl

use of org.apache.hadoop.hive.llap.impl.LlapManagementProtocolClientImpl in project hive by apache.

the class TestLlapDaemonProtocolServerImpl method testGetDaemonMetrics.

@Test(timeout = 10000)
public void testGetDaemonMetrics() throws ServiceException, IOException {
    LlapDaemonConfiguration daemonConf = new LlapDaemonConfiguration();
    int numHandlers = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_RPC_NUM_HANDLERS);
    LlapDaemonExecutorMetrics executorMetrics = LlapDaemonExecutorMetrics.create("LLAP", "SessionId", numHandlers, 1, new int[] { 30, 60, 300 }, 0, 0L, 0);
    LlapProtocolServerImpl server = new LlapProtocolServerImpl(null, numHandlers, null, new AtomicReference<InetSocketAddress>(), new AtomicReference<InetSocketAddress>(), 0, 0, 0, null, executorMetrics);
    executorMetrics.addMetricsFallOffFailedTimeLost(10);
    executorMetrics.addMetricsFallOffKilledTimeLost(11);
    executorMetrics.addMetricsFallOffSuccessTimeLost(12);
    executorMetrics.addMetricsPreemptionTimeLost(13);
    executorMetrics.addMetricsPreemptionTimeToKill(14);
    executorMetrics.incrExecutorTotalExecutionFailed();
    executorMetrics.incrExecutorTotalKilled();
    executorMetrics.incrExecutorTotalRequestsHandled();
    executorMetrics.incrExecutorTotalSuccess();
    executorMetrics.incrTotalEvictedFromWaitQueue();
    executorMetrics.incrTotalRejectedRequests();
    executorMetrics.setCacheMemoryPerInstance(15);
    executorMetrics.setExecutorNumPreemptableRequests(16);
    executorMetrics.setExecutorNumQueuedRequests(17);
    executorMetrics.setJvmMaxMemory(18);
    executorMetrics.setMemoryPerInstance(19);
    executorMetrics.setNumExecutorsAvailable(20);
    executorMetrics.setWaitQueueSize(21);
    try {
        server.init(new Configuration());
        server.start();
        InetSocketAddress serverAddr = server.getManagementBindAddress();
        LlapManagementProtocolPB client = new LlapManagementProtocolClientImpl(new Configuration(), serverAddr.getHostName(), serverAddr.getPort(), null, null);
        GetDaemonMetricsResponseProto responseProto = client.getDaemonMetrics(null, GetDaemonMetricsRequestProto.newBuilder().build());
        Map<String, Long> result = new HashMap<>(responseProto.getMetricsCount());
        responseProto.getMetricsList().forEach(me -> result.put(me.getKey(), me.getValue()));
        assertTrue("Checking ExecutorFallOffFailedTimeLost", result.get("ExecutorFallOffFailedTimeLost") == 10);
        assertTrue("Checking ExecutorFallOffKilledTimeLost", result.get("ExecutorFallOffKilledTimeLost") == 11);
        assertTrue("Checking ExecutorFallOffSuccessTimeLost", result.get("ExecutorFallOffSuccessTimeLost") == 12);
        assertTrue("Checking ExecutorTotalPreemptionTimeLost", result.get("ExecutorTotalPreemptionTimeLost") == 13);
        assertTrue("Checking ExecutorTotalPreemptionTimeToKill", result.get("ExecutorTotalPreemptionTimeToKill") == 14);
        assertTrue("Checking ExecutorTotalFailed", result.get("ExecutorTotalFailed") == 1);
        assertTrue("Checking ExecutorTotalKilled", result.get("ExecutorTotalKilled") == 1);
        assertTrue("Checking ExecutorTotalRequestsHandled", result.get("ExecutorTotalRequestsHandled") == 1);
        assertTrue("Checking ExecutorTotalSuccess", result.get("ExecutorTotalSuccess") == 1);
        assertTrue("Checking ExecutorTotalEvictedFromWaitQueue", result.get("ExecutorTotalEvictedFromWaitQueue") == 1);
        assertTrue("Checking ExecutorTotalRejectedRequests", result.get("ExecutorTotalRejectedRequests") == 1);
        assertTrue("Checking ExecutorCacheMemoryPerInstance", result.get("ExecutorCacheMemoryPerInstance") == 15);
        assertTrue("Checking ExecutorNumPreemptableRequests", result.get("ExecutorNumPreemptableRequests") == 16);
        assertTrue("Checking ExecutorNumQueuedRequests", result.get("ExecutorNumQueuedRequests") == 17);
        assertTrue("Checking ExecutorJvmMaxMemory", result.get("ExecutorJvmMaxMemory") == 18);
        assertTrue("Checking ExecutorMemoryPerInstance", result.get("ExecutorMemoryPerInstance") == 19);
        assertTrue("Checking ExecutorNumExecutorsAvailable", result.get("ExecutorNumExecutorsAvailable") == 20);
        assertTrue("Checking ExecutorWaitQueueSize", result.get("ExecutorWaitQueueSize") == 21);
    } finally {
        server.stop();
    }
}

Example 4 with LlapManagementProtocolClientImpl

use of org.apache.hadoop.hive.llap.impl.LlapManagementProtocolClientImpl in project hive by apache.

the class BlacklistingLlapMetricsListener method setCapacity.

protected void setCapacity(String workerIdentity, int newExecutorNum, int newWaitQueueSize) throws IOException, ServiceException {
    long currentTime = System.currentTimeMillis();
    if (currentTime > nextCheckTime) {
        LlapZookeeperRegistryImpl.ConfigChangeLockResult lockResult = registry.lockForConfigChange(currentTime, currentTime + this.minConfigChangeDelayMs);
        LOG.debug("Got result for lock check: {}", lockResult);
        if (lockResult.isSuccess()) {
            LOG.info("Setting capacity for workerIdentity={} to newExecutorNum={}, newWaitQueueSize={}", workerIdentity, newExecutorNum, newWaitQueueSize);
            LlapServiceInstance serviceInstance = registry.getInstances().getInstance(workerIdentity);
            LlapManagementProtocolClientImpl client = clientFactory.create(serviceInstance);
            client.setCapacity(null, SetCapacityRequestProto.newBuilder().setExecutorNum(newExecutorNum).setQueueSize(newWaitQueueSize).build());
        }
        if (lockResult.getNextConfigChangeTime() > -1L) {
            nextCheckTime = lockResult.getNextConfigChangeTime();
        }
    } else {
        LOG.debug("Skipping check. Current time {} and we are waiting for {}.", currentTime, nextCheckTime);
    }
}

Example 5 with LlapManagementProtocolClientImpl

use of org.apache.hadoop.hive.llap.impl.LlapManagementProtocolClientImpl in project hive by apache.

the class LlapMetricsCollector method collectMetrics.

@VisibleForTesting
void collectMetrics() {
    for (Map.Entry<String, LlapManagementProtocolClientImpl> entry : llapClients.entrySet()) {
        String identity = entry.getKey();
        LlapManagementProtocolClientImpl client = entry.getValue();
        try {
            LlapDaemonProtocolProtos.GetDaemonMetricsResponseProto metrics = client.getDaemonMetrics(null, LlapDaemonProtocolProtos.GetDaemonMetricsRequestProto.newBuilder().build());
            LlapMetrics newMetrics = new LlapMetrics(metrics);
            instanceStatisticsMap.put(identity, newMetrics);
            if (listener != null) {
                try {
                    listener.newDaemonMetrics(identity, newMetrics);
                } catch (Throwable t) {
                    LOG.warn("LlapMetricsListener thrown an unexpected exception", t);
                }
            }
        } catch (ServiceException ex) {
            LOG.error(ex.getMessage(), ex);
            instanceStatisticsMap.remove(identity);
        }
    }
    if (listener != null) {
        try {
            listener.newClusterMetrics(getMetrics());
        } catch (Throwable t) {
            LOG.warn("LlapMetricsListener thrown an unexpected exception", t);
        }
    }
}