Search in sources :

Example 1 with LoadTask

use of alluxio.job.plan.load.LoadDefinition.LoadTask in project alluxio by Alluxio.

the class LoadDefinitionTest method skipJobWorkersWithoutLocalBlockWorkers.

@Test
public void skipJobWorkersWithoutLocalBlockWorkers() throws Exception {
    List<BlockWorkerInfo> blockWorkers = Arrays.asList(new BlockWorkerInfo(new WorkerNetAddress().setHost("host0"), 0, 0));
    Mockito.when(mMockFsContext.getCachedWorkers()).thenReturn(blockWorkers);
    createFileWithNoLocations(TEST_URI, 10);
    LoadConfig config = new LoadConfig(TEST_URI, 1, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, false);
    Set<Pair<WorkerInfo, ArrayList<LoadTask>>> assignments = new LoadDefinition().selectExecutors(config, JOB_WORKERS, new SelectExecutorsContext(1, mJobServerContext));
    Assert.assertEquals(10, assignments.size());
    Assert.assertEquals(1, assignments.iterator().next().getSecond().size());
}
Also used : LoadTask(alluxio.job.plan.load.LoadDefinition.LoadTask) WorkerNetAddress(alluxio.wire.WorkerNetAddress) BlockWorkerInfo(alluxio.client.block.BlockWorkerInfo) SelectExecutorsContext(alluxio.job.SelectExecutorsContext) Pair(alluxio.collections.Pair) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 2 with LoadTask

use of alluxio.job.plan.load.LoadDefinition.LoadTask in project alluxio by Alluxio.

the class LoadDefinition method selectExecutors.

@Override
public Set<Pair<WorkerInfo, ArrayList<LoadTask>>> selectExecutors(LoadConfig config, List<WorkerInfo> jobWorkerInfoList, SelectExecutorsContext context) throws Exception {
    Map<String, WorkerInfo> jobWorkersByAddress = jobWorkerInfoList.stream().collect(Collectors.toMap(info -> info.getAddress().getHost(), info -> info));
    // Filter out workers which have no local job worker available.
    List<String> missingJobWorkerHosts = new ArrayList<>();
    List<BlockWorkerInfo> workers = new ArrayList<>();
    for (BlockWorkerInfo worker : context.getFsContext().getCachedWorkers()) {
        if (jobWorkersByAddress.containsKey(worker.getNetAddress().getHost())) {
            String workerHost = worker.getNetAddress().getHost().toUpperCase();
            if (!isEmptySet(config.getExcludedWorkerSet()) && config.getExcludedWorkerSet().contains(workerHost)) {
                continue;
            }
            // If specified the locality id, the candidate worker must match one at least
            boolean match = false;
            if (worker.getNetAddress().getTieredIdentity().getTiers() != null) {
                if (!(isEmptySet(config.getLocalityIds()) && isEmptySet(config.getExcludedLocalityIds()))) {
                    boolean exclude = false;
                    for (LocalityTier tier : worker.getNetAddress().getTieredIdentity().getTiers()) {
                        if (!isEmptySet(config.getExcludedLocalityIds()) && config.getExcludedLocalityIds().contains(tier.getValue().toUpperCase())) {
                            exclude = true;
                            break;
                        }
                        if (!isEmptySet(config.getLocalityIds()) && config.getLocalityIds().contains(tier.getValue().toUpperCase())) {
                            match = true;
                            break;
                        }
                    }
                    if (exclude) {
                        continue;
                    }
                }
            }
            // Or user specified neither worker-set nor locality id
            if ((isEmptySet(config.getWorkerSet()) && isEmptySet(config.getLocalityIds())) || match || (!isEmptySet(config.getWorkerSet()) && config.getWorkerSet().contains(workerHost))) {
                workers.add(worker);
            }
        } else {
            LOG.warn("Worker on host {} has no local job worker", worker.getNetAddress().getHost());
            missingJobWorkerHosts.add(worker.getNetAddress().getHost());
        }
    }
    // Mapping from worker to block ids which that worker is supposed to load.
    Multimap<WorkerInfo, LoadTask> assignments = LinkedListMultimap.create();
    AlluxioURI uri = new AlluxioURI(config.getFilePath());
    for (FileBlockInfo blockInfo : context.getFileSystem().getStatus(uri).getFileBlockInfos()) {
        List<BlockWorkerInfo> workersWithoutBlock = getWorkersWithoutBlock(workers, blockInfo);
        int neededReplicas = config.getReplication() - blockInfo.getBlockInfo().getLocations().size();
        if (workersWithoutBlock.size() < neededReplicas) {
            String missingJobWorkersMessage = "";
            if (!missingJobWorkerHosts.isEmpty()) {
                missingJobWorkersMessage = ". The following workers could not be used because they have " + "no local job workers: " + missingJobWorkerHosts;
            }
            throw new FailedPreconditionException(String.format("Failed to find enough block workers to replicate to. Needed %s but only found %s. " + "Available workers without the block: %s" + missingJobWorkersMessage, neededReplicas, workersWithoutBlock.size(), workersWithoutBlock));
        }
        Collections.shuffle(workersWithoutBlock);
        for (int i = 0; i < neededReplicas; i++) {
            String address = workersWithoutBlock.get(i).getNetAddress().getHost();
            WorkerInfo jobWorker = jobWorkersByAddress.get(address);
            assignments.put(jobWorker, new LoadTask(blockInfo.getBlockInfo().getBlockId(), workersWithoutBlock.get(i).getNetAddress()));
        }
    }
    Set<Pair<WorkerInfo, ArrayList<LoadTask>>> result = Sets.newHashSet();
    for (Map.Entry<WorkerInfo, Collection<LoadTask>> assignment : assignments.asMap().entrySet()) {
        Collection<LoadTask> loadTasks = assignment.getValue();
        List<List<LoadTask>> partitionedTasks = CommonUtils.partition(Lists.newArrayList(loadTasks), JOBS_PER_WORKER);
        for (List<LoadTask> tasks : partitionedTasks) {
            if (!tasks.isEmpty()) {
                result.add(new Pair<>(assignment.getKey(), Lists.newArrayList(tasks)));
            }
        }
    }
    return result;
}
Also used : FailedPreconditionException(alluxio.exception.status.FailedPreconditionException) WorkerNetAddress(alluxio.wire.WorkerNetAddress) LoggerFactory(org.slf4j.LoggerFactory) BlockWorkerInfo(alluxio.client.block.BlockWorkerInfo) FileBlockInfo(alluxio.wire.FileBlockInfo) Multimap(com.google.common.collect.Multimap) ArrayList(java.util.ArrayList) AbstractVoidPlanDefinition(alluxio.job.plan.AbstractVoidPlanDefinition) JobUtils(alluxio.job.util.JobUtils) SerializableVoid(alluxio.job.util.SerializableVoid) Lists(com.google.common.collect.Lists) Constants(alluxio.Constants) RunTaskContext(alluxio.job.RunTaskContext) WorkerInfo(alluxio.wire.WorkerInfo) AlluxioURI(alluxio.AlluxioURI) Map(java.util.Map) LinkedListMultimap(com.google.common.collect.LinkedListMultimap) LocalityTier(alluxio.wire.TieredIdentity.LocalityTier) Logger(org.slf4j.Logger) Collection(java.util.Collection) MoreObjects(com.google.common.base.MoreObjects) Set(java.util.Set) SelectExecutorsContext(alluxio.job.SelectExecutorsContext) Pair(alluxio.collections.Pair) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) Serializable(java.io.Serializable) LoadTask(alluxio.job.plan.load.LoadDefinition.LoadTask) BlockLocation(alluxio.wire.BlockLocation) URIStatus(alluxio.client.file.URIStatus) List(java.util.List) Collections(java.util.Collections) CommonUtils(alluxio.util.CommonUtils) NotThreadSafe(javax.annotation.concurrent.NotThreadSafe) ArrayList(java.util.ArrayList) BlockWorkerInfo(alluxio.client.block.BlockWorkerInfo) WorkerInfo(alluxio.wire.WorkerInfo) FileBlockInfo(alluxio.wire.FileBlockInfo) ArrayList(java.util.ArrayList) List(java.util.List) Pair(alluxio.collections.Pair) LocalityTier(alluxio.wire.TieredIdentity.LocalityTier) LoadTask(alluxio.job.plan.load.LoadDefinition.LoadTask) BlockWorkerInfo(alluxio.client.block.BlockWorkerInfo) FailedPreconditionException(alluxio.exception.status.FailedPreconditionException) Collection(java.util.Collection) Map(java.util.Map) AlluxioURI(alluxio.AlluxioURI)

Example 3 with LoadTask

use of alluxio.job.plan.load.LoadDefinition.LoadTask in project alluxio by Alluxio.

the class LoadDefinition method runTask.

@Override
public SerializableVoid runTask(LoadConfig config, ArrayList<LoadTask> tasks, RunTaskContext context) throws Exception {
    // TODO(jiri): Replace with internal client that uses file ID once the internal client is
    // factored out of the core server module. The reason to prefer using file ID for this job is
    // to avoid the the race between "replicate" and "rename", so that even a file to replicate is
    // renamed, the job is still working on the correct file.
    URIStatus status = context.getFileSystem().getStatus(new AlluxioURI(config.getFilePath()));
    for (LoadTask task : tasks) {
        JobUtils.loadBlock(status, context.getFsContext(), task.getBlockId(), task.getWorkerNetAddress(), config.isDirectCache());
        LOG.info("Loaded file " + config.getFilePath() + " block " + task.getBlockId());
    }
    return null;
}
Also used : LoadTask(alluxio.job.plan.load.LoadDefinition.LoadTask) URIStatus(alluxio.client.file.URIStatus) AlluxioURI(alluxio.AlluxioURI)

Example 4 with LoadTask

use of alluxio.job.plan.load.LoadDefinition.LoadTask in project alluxio by Alluxio.

the class LoadDefinitionTest method loadedBySpecifiedHost.

private void loadedBySpecifiedHost(Set<String> workerSet, Set<String> excludedWorkerSet, Set<String> localityIds, Set<String> excludedLocalityIds, Set<Long> workerIds) throws Exception {
    int numBlocks = 10;
    createFileWithNoLocations(TEST_URI, numBlocks);
    LoadConfig config = new LoadConfig(TEST_URI, 1, workerSet, excludedWorkerSet, localityIds, excludedLocalityIds, false);
    Set<Pair<WorkerInfo, ArrayList<LoadTask>>> assignments = new LoadDefinition().selectExecutors(config, JOB_WORKERS, new SelectExecutorsContext(1, mJobServerContext));
    // Check that we are loading the right number of blocks.
    int totalBlockLoads = 0;
    for (Pair<WorkerInfo, ArrayList<LoadTask>> assignment : assignments) {
        totalBlockLoads += assignment.getSecond().size();
        Assert.assertTrue(workerIds.contains(assignment.getFirst().getId()));
    }
    Assert.assertEquals(numBlocks, totalBlockLoads);
}
Also used : LoadTask(alluxio.job.plan.load.LoadDefinition.LoadTask) ArrayList(java.util.ArrayList) BlockWorkerInfo(alluxio.client.block.BlockWorkerInfo) WorkerInfo(alluxio.wire.WorkerInfo) SelectExecutorsContext(alluxio.job.SelectExecutorsContext) Pair(alluxio.collections.Pair)

Example 5 with LoadTask

use of alluxio.job.plan.load.LoadDefinition.LoadTask in project alluxio by Alluxio.

the class LoadDefinitionTest method replicationSatisfied.

@Test
public void replicationSatisfied() throws Exception {
    int numBlocks = 7;
    int replication = 3;
    createFileWithNoLocations(TEST_URI, numBlocks);
    LoadConfig config = new LoadConfig(TEST_URI, replication, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, false);
    Set<Pair<WorkerInfo, ArrayList<LoadTask>>> assignments = new LoadDefinition().selectExecutors(config, JOB_WORKERS, new SelectExecutorsContext(1, mJobServerContext));
    // Check that we are loading the right number of blocks.
    int totalBlockLoads = 0;
    for (Pair<WorkerInfo, ArrayList<LoadTask>> assignment : assignments) {
        totalBlockLoads += assignment.getSecond().size();
    }
    Assert.assertEquals(numBlocks * replication, totalBlockLoads);
}
Also used : LoadTask(alluxio.job.plan.load.LoadDefinition.LoadTask) ArrayList(java.util.ArrayList) BlockWorkerInfo(alluxio.client.block.BlockWorkerInfo) WorkerInfo(alluxio.wire.WorkerInfo) SelectExecutorsContext(alluxio.job.SelectExecutorsContext) Pair(alluxio.collections.Pair) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Aggregations

LoadTask (alluxio.job.plan.load.LoadDefinition.LoadTask)6 BlockWorkerInfo (alluxio.client.block.BlockWorkerInfo)5 Pair (alluxio.collections.Pair)5 SelectExecutorsContext (alluxio.job.SelectExecutorsContext)5 WorkerInfo (alluxio.wire.WorkerInfo)4 ArrayList (java.util.ArrayList)4 Test (org.junit.Test)3 PrepareForTest (org.powermock.core.classloader.annotations.PrepareForTest)3 AlluxioURI (alluxio.AlluxioURI)2 URIStatus (alluxio.client.file.URIStatus)2 WorkerNetAddress (alluxio.wire.WorkerNetAddress)2 Map (java.util.Map)2 Constants (alluxio.Constants)1 FailedPreconditionException (alluxio.exception.status.FailedPreconditionException)1 RunTaskContext (alluxio.job.RunTaskContext)1 AbstractVoidPlanDefinition (alluxio.job.plan.AbstractVoidPlanDefinition)1 BatchedJobDefinition (alluxio.job.plan.batch.BatchedJobDefinition)1 LoadConfig (alluxio.job.plan.load.LoadConfig)1 JobUtils (alluxio.job.util.JobUtils)1 SerializableVoid (alluxio.job.util.SerializableVoid)1