Examples with LlapServiceInstance org.apache.hadoop.hive.llap.registry.LlapServiceInstance used on opensource projects

Example 31 with LlapServiceInstance

use of org.apache.hadoop.hive.llap.registry.LlapServiceInstance in project hive by apache.

the class GenericUDTFGetSplits method populateLlapDaemonInfos.

private LlapDaemonInfo[] populateLlapDaemonInfos(JobConf job, SplitLocationInfo[] locations) throws IOException {
    LlapRegistryService registryService = LlapRegistryService.getClient(job);
    LlapServiceInstanceSet instanceSet = registryService.getInstances();
    Collection<LlapServiceInstance> llapServiceInstances = null;
    // this means a valid location, see makeLocationHints()
    if (locations.length == 1 && locations[0].getLocation() != null) {
        llapServiceInstances = instanceSet.getByHost(locations[0].getLocation());
    }
    // let's populate them all so that we can fetch data from any of them.
    if (CollectionUtils.isEmpty(llapServiceInstances)) {
        llapServiceInstances = instanceSet.getAll();
    }
    Preconditions.checkState(llapServiceInstances.size() > 0, "Unable to find any of the llap instances in zk registry");
    LlapDaemonInfo[] llapDaemonInfos = new LlapDaemonInfo[llapServiceInstances.size()];
    int count = 0;
    for (LlapServiceInstance inst : llapServiceInstances) {
        LlapDaemonInfo info;
        if (LlapUtil.isCloudDeployment(job)) {
            info = new LlapDaemonInfo(inst.getExternalHostname(), inst.getExternalClientsRpcPort(), inst.getOutputFormatPort());
        } else {
            info = new LlapDaemonInfo(inst.getHost(), inst.getRpcPort(), inst.getOutputFormatPort());
        }
        llapDaemonInfos[count++] = info;
    }
    return llapDaemonInfos;
}

Example 32 with LlapServiceInstance

use of org.apache.hadoop.hive.llap.registry.LlapServiceInstance in project hive by apache.

the class LlapZookeeperRegistryImpl method getAllInstancesOrdered.

// The real implementation for the instanceset... instanceset has its own copy of the
// ZK cache yet completely depends on the parent in every other aspect and is thus unneeded.
Collection<LlapServiceInstance> getAllInstancesOrdered(boolean consistentIndexes, PathChildrenCache instancesCache) {
    Map<String, Long> slotByWorker = new HashMap<String, Long>();
    Set<LlapServiceInstance> unsorted = Sets.newHashSet();
    for (ChildData childData : instancesCache.getCurrentData()) {
        if (childData == null)
            continue;
        byte[] data = childData.getData();
        if (data == null)
            continue;
        String nodeName = extractNodeName(childData);
        if (nodeName.startsWith(WORKER_PREFIX)) {
            LlapServiceInstance instances = getInstanceByPath(childData.getPath());
            if (instances != null) {
                unsorted.add(instances);
            }
        } else if (nodeName.startsWith(SLOT_PREFIX)) {
            slotByWorker.put(extractWorkerIdFromSlot(childData), Long.parseLong(nodeName.substring(SLOT_PREFIX.length())));
        } else {
            LOG.info("Ignoring unknown node {}", childData.getPath());
        }
    }
    TreeMap<Long, LlapServiceInstance> sorted = new TreeMap<>();
    long maxSlot = Long.MIN_VALUE;
    for (LlapServiceInstance worker : unsorted) {
        Long slot = slotByWorker.get(worker.getWorkerIdentity());
        if (slot == null) {
            LOG.info("Unknown slot for {}", worker.getWorkerIdentity());
            continue;
        }
        maxSlot = Math.max(maxSlot, slot);
        sorted.put(slot, worker);
    }
    if (consistentIndexes) {
        // Add dummy instances to all slots where LLAPs are MIA... I can haz insert_iterator?
        TreeMap<Long, LlapServiceInstance> dummies = new TreeMap<>();
        Iterator<Long> keyIter = sorted.keySet().iterator();
        long expected = 0;
        Long ts = null;
        while (keyIter.hasNext()) {
            Long slot = keyIter.next();
            assert slot >= expected;
            while (slot > expected) {
                if (ts == null) {
                    // Inactive nodes restart every call!
                    ts = System.nanoTime();
                }
                dummies.put(expected, new InactiveServiceInstance("inactive-" + expected + "-" + ts));
                ++expected;
            }
            ++expected;
        }
        sorted.putAll(dummies);
    }
    return sorted.values();
}