Examples with LocalityTier alluxio.wire.TieredIdentity.LocalityTier used on opensource projects

Example 6 with LocalityTier

use of alluxio.wire.TieredIdentity.LocalityTier in project alluxio by Alluxio.

the class TieredIdentityFactoryTest method outOfOrderScript.

@Test
public void outOfOrderScript() throws Exception {
    String scriptPath = setupScript("rack=myrack,node=myhost", mFolder.newFile());
    try (Closeable c = new ConfigurationRule(ImmutableMap.of(PropertyKey.LOCALITY_SCRIPT, scriptPath), mConfiguration).toResource()) {
        TieredIdentity identity = TieredIdentityFactory.create(mConfiguration);
        TieredIdentity expected = new TieredIdentity(Arrays.asList(new LocalityTier("node", "myhost"), new LocalityTier("rack", "myrack")));
        assertEquals(expected, identity);
    }
}

Example 7 with LocalityTier

use of alluxio.wire.TieredIdentity.LocalityTier in project alluxio by Alluxio.

the class LoadDefinition method selectExecutors.

@Override
public Set<Pair<WorkerInfo, ArrayList<LoadTask>>> selectExecutors(LoadConfig config, List<WorkerInfo> jobWorkerInfoList, SelectExecutorsContext context) throws Exception {
    Map<String, WorkerInfo> jobWorkersByAddress = jobWorkerInfoList.stream().collect(Collectors.toMap(info -> info.getAddress().getHost(), info -> info));
    // Filter out workers which have no local job worker available.
    List<String> missingJobWorkerHosts = new ArrayList<>();
    List<BlockWorkerInfo> workers = new ArrayList<>();
    for (BlockWorkerInfo worker : context.getFsContext().getCachedWorkers()) {
        if (jobWorkersByAddress.containsKey(worker.getNetAddress().getHost())) {
            String workerHost = worker.getNetAddress().getHost().toUpperCase();
            if (!isEmptySet(config.getExcludedWorkerSet()) && config.getExcludedWorkerSet().contains(workerHost)) {
                continue;
            }
            // If specified the locality id, the candidate worker must match one at least
            boolean match = false;
            if (worker.getNetAddress().getTieredIdentity().getTiers() != null) {
                if (!(isEmptySet(config.getLocalityIds()) && isEmptySet(config.getExcludedLocalityIds()))) {
                    boolean exclude = false;
                    for (LocalityTier tier : worker.getNetAddress().getTieredIdentity().getTiers()) {
                        if (!isEmptySet(config.getExcludedLocalityIds()) && config.getExcludedLocalityIds().contains(tier.getValue().toUpperCase())) {
                            exclude = true;
                            break;
                        }
                        if (!isEmptySet(config.getLocalityIds()) && config.getLocalityIds().contains(tier.getValue().toUpperCase())) {
                            match = true;
                            break;
                        }
                    }
                    if (exclude) {
                        continue;
                    }
                }
            }
            // Or user specified neither worker-set nor locality id
            if ((isEmptySet(config.getWorkerSet()) && isEmptySet(config.getLocalityIds())) || match || (!isEmptySet(config.getWorkerSet()) && config.getWorkerSet().contains(workerHost))) {
                workers.add(worker);
            }
        } else {
            LOG.warn("Worker on host {} has no local job worker", worker.getNetAddress().getHost());
            missingJobWorkerHosts.add(worker.getNetAddress().getHost());
        }
    }
    // Mapping from worker to block ids which that worker is supposed to load.
    Multimap<WorkerInfo, LoadTask> assignments = LinkedListMultimap.create();
    AlluxioURI uri = new AlluxioURI(config.getFilePath());
    for (FileBlockInfo blockInfo : context.getFileSystem().getStatus(uri).getFileBlockInfos()) {
        List<BlockWorkerInfo> workersWithoutBlock = getWorkersWithoutBlock(workers, blockInfo);
        int neededReplicas = config.getReplication() - blockInfo.getBlockInfo().getLocations().size();
        if (workersWithoutBlock.size() < neededReplicas) {
            String missingJobWorkersMessage = "";
            if (!missingJobWorkerHosts.isEmpty()) {
                missingJobWorkersMessage = ". The following workers could not be used because they have " + "no local job workers: " + missingJobWorkerHosts;
            }
            throw new FailedPreconditionException(String.format("Failed to find enough block workers to replicate to. Needed %s but only found %s. " + "Available workers without the block: %s" + missingJobWorkersMessage, neededReplicas, workersWithoutBlock.size(), workersWithoutBlock));
        }
        Collections.shuffle(workersWithoutBlock);
        for (int i = 0; i < neededReplicas; i++) {
            String address = workersWithoutBlock.get(i).getNetAddress().getHost();
            WorkerInfo jobWorker = jobWorkersByAddress.get(address);
            assignments.put(jobWorker, new LoadTask(blockInfo.getBlockInfo().getBlockId(), workersWithoutBlock.get(i).getNetAddress()));
        }
    }
    Set<Pair<WorkerInfo, ArrayList<LoadTask>>> result = Sets.newHashSet();
    for (Map.Entry<WorkerInfo, Collection<LoadTask>> assignment : assignments.asMap().entrySet()) {
        Collection<LoadTask> loadTasks = assignment.getValue();
        List<List<LoadTask>> partitionedTasks = CommonUtils.partition(Lists.newArrayList(loadTasks), JOBS_PER_WORKER);
        for (List<LoadTask> tasks : partitionedTasks) {
            if (!tasks.isEmpty()) {
                result.add(new Pair<>(assignment.getKey(), Lists.newArrayList(tasks)));
            }
        }
    }
    return result;
}

Example 8 with LocalityTier

use of alluxio.wire.TieredIdentity.LocalityTier in project alluxio by Alluxio.

the class ClientTestUtils method worker.

public static BlockWorkerInfo worker(long capacity, long used, String node, String rack) {
    WorkerNetAddress address = new WorkerNetAddress();
    List<LocalityTier> tiers = new ArrayList<>();
    if (node != null && !node.isEmpty()) {
        address.setHost(node);
        tiers.add(new LocalityTier(Constants.LOCALITY_NODE, node));
    }
    if (rack != null && !rack.isEmpty()) {
        tiers.add(new LocalityTier(Constants.LOCALITY_RACK, rack));
    }
    address.setTieredIdentity(new TieredIdentity(tiers));
    return new BlockWorkerInfo(address, capacity, used);
}

Example 9 with LocalityTier

use of alluxio.wire.TieredIdentity.LocalityTier in project alluxio by Alluxio.

the class TieredIdentityFactoryTest method defaultConf.

@Test
public void defaultConf() throws Exception {
    TieredIdentity identity = TieredIdentityFactory.create(mConfiguration);
    TieredIdentity expected = new TieredIdentity(Arrays.asList(new LocalityTier("node", NetworkAddressUtils.getLocalNodeName(mConfiguration)), new LocalityTier("rack", null)));
    assertEquals(expected, identity);
}

Example 10 with LocalityTier

use of alluxio.wire.TieredIdentity.LocalityTier in project alluxio by Alluxio.

the class TieredIdentityFactory method fromString.

/**
 * @param identityString tiered identity string to parse
 * @param conf Alluxio configuration
 * @return the parsed tiered identity
 */
public static TieredIdentity fromString(String identityString, AlluxioConfiguration conf) throws IOException {
    Set<String> allTiers = Sets.newHashSet(conf.getList(PropertyKey.LOCALITY_ORDER, ","));
    Map<String, String> tiers = new HashMap<>();
    for (String tier : identityString.split(",")) {
        String[] parts = tier.split("=");
        if (parts.length != 2) {
            throw new IOException(String.format("Failed to parse tiered identity. The value should be a comma-separated list " + "of key=value pairs, but was %s", identityString));
        }
        String key = parts[0].trim();
        if (tiers.containsKey(key)) {
            throw new IOException(String.format("Encountered repeated tier definition for %s when parsing tiered identity from string " + "%s", key, identityString));
        }
        if (!allTiers.contains(key)) {
            throw new IOException(String.format("Unrecognized tier: %s. The tiers defined by %s are %s", key, PropertyKey.LOCALITY_ORDER.toString(), allTiers));
        }
        tiers.put(key, parts[1].trim());
    }
    List<LocalityTier> tieredIdentity = new ArrayList<>();
    for (String localityTier : conf.getList(PropertyKey.LOCALITY_ORDER, ",")) {
        String value = tiers.getOrDefault(localityTier, null);
        tieredIdentity.add(new LocalityTier(localityTier, value));
    }
    return new TieredIdentity(tieredIdentity);
}