use of alluxio.job.plan.load.LoadDefinition.LoadTask in project alluxio by Alluxio.
the class LoadDefinitionTest method skipJobWorkersWithoutLocalBlockWorkers.
@Test
public void skipJobWorkersWithoutLocalBlockWorkers() throws Exception {
List<BlockWorkerInfo> blockWorkers = Arrays.asList(new BlockWorkerInfo(new WorkerNetAddress().setHost("host0"), 0, 0));
Mockito.when(mMockFsContext.getCachedWorkers()).thenReturn(blockWorkers);
createFileWithNoLocations(TEST_URI, 10);
LoadConfig config = new LoadConfig(TEST_URI, 1, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, false);
Set<Pair<WorkerInfo, ArrayList<LoadTask>>> assignments = new LoadDefinition().selectExecutors(config, JOB_WORKERS, new SelectExecutorsContext(1, mJobServerContext));
Assert.assertEquals(10, assignments.size());
Assert.assertEquals(1, assignments.iterator().next().getSecond().size());
}
use of alluxio.job.plan.load.LoadDefinition.LoadTask in project alluxio by Alluxio.
the class LoadDefinition method selectExecutors.
@Override
public Set<Pair<WorkerInfo, ArrayList<LoadTask>>> selectExecutors(LoadConfig config, List<WorkerInfo> jobWorkerInfoList, SelectExecutorsContext context) throws Exception {
Map<String, WorkerInfo> jobWorkersByAddress = jobWorkerInfoList.stream().collect(Collectors.toMap(info -> info.getAddress().getHost(), info -> info));
// Filter out workers which have no local job worker available.
List<String> missingJobWorkerHosts = new ArrayList<>();
List<BlockWorkerInfo> workers = new ArrayList<>();
for (BlockWorkerInfo worker : context.getFsContext().getCachedWorkers()) {
if (jobWorkersByAddress.containsKey(worker.getNetAddress().getHost())) {
String workerHost = worker.getNetAddress().getHost().toUpperCase();
if (!isEmptySet(config.getExcludedWorkerSet()) && config.getExcludedWorkerSet().contains(workerHost)) {
continue;
}
// If specified the locality id, the candidate worker must match one at least
boolean match = false;
if (worker.getNetAddress().getTieredIdentity().getTiers() != null) {
if (!(isEmptySet(config.getLocalityIds()) && isEmptySet(config.getExcludedLocalityIds()))) {
boolean exclude = false;
for (LocalityTier tier : worker.getNetAddress().getTieredIdentity().getTiers()) {
if (!isEmptySet(config.getExcludedLocalityIds()) && config.getExcludedLocalityIds().contains(tier.getValue().toUpperCase())) {
exclude = true;
break;
}
if (!isEmptySet(config.getLocalityIds()) && config.getLocalityIds().contains(tier.getValue().toUpperCase())) {
match = true;
break;
}
}
if (exclude) {
continue;
}
}
}
// Or user specified neither worker-set nor locality id
if ((isEmptySet(config.getWorkerSet()) && isEmptySet(config.getLocalityIds())) || match || (!isEmptySet(config.getWorkerSet()) && config.getWorkerSet().contains(workerHost))) {
workers.add(worker);
}
} else {
LOG.warn("Worker on host {} has no local job worker", worker.getNetAddress().getHost());
missingJobWorkerHosts.add(worker.getNetAddress().getHost());
}
}
// Mapping from worker to block ids which that worker is supposed to load.
Multimap<WorkerInfo, LoadTask> assignments = LinkedListMultimap.create();
AlluxioURI uri = new AlluxioURI(config.getFilePath());
for (FileBlockInfo blockInfo : context.getFileSystem().getStatus(uri).getFileBlockInfos()) {
List<BlockWorkerInfo> workersWithoutBlock = getWorkersWithoutBlock(workers, blockInfo);
int neededReplicas = config.getReplication() - blockInfo.getBlockInfo().getLocations().size();
if (workersWithoutBlock.size() < neededReplicas) {
String missingJobWorkersMessage = "";
if (!missingJobWorkerHosts.isEmpty()) {
missingJobWorkersMessage = ". The following workers could not be used because they have " + "no local job workers: " + missingJobWorkerHosts;
}
throw new FailedPreconditionException(String.format("Failed to find enough block workers to replicate to. Needed %s but only found %s. " + "Available workers without the block: %s" + missingJobWorkersMessage, neededReplicas, workersWithoutBlock.size(), workersWithoutBlock));
}
Collections.shuffle(workersWithoutBlock);
for (int i = 0; i < neededReplicas; i++) {
String address = workersWithoutBlock.get(i).getNetAddress().getHost();
WorkerInfo jobWorker = jobWorkersByAddress.get(address);
assignments.put(jobWorker, new LoadTask(blockInfo.getBlockInfo().getBlockId(), workersWithoutBlock.get(i).getNetAddress()));
}
}
Set<Pair<WorkerInfo, ArrayList<LoadTask>>> result = Sets.newHashSet();
for (Map.Entry<WorkerInfo, Collection<LoadTask>> assignment : assignments.asMap().entrySet()) {
Collection<LoadTask> loadTasks = assignment.getValue();
List<List<LoadTask>> partitionedTasks = CommonUtils.partition(Lists.newArrayList(loadTasks), JOBS_PER_WORKER);
for (List<LoadTask> tasks : partitionedTasks) {
if (!tasks.isEmpty()) {
result.add(new Pair<>(assignment.getKey(), Lists.newArrayList(tasks)));
}
}
}
return result;
}
use of alluxio.job.plan.load.LoadDefinition.LoadTask in project alluxio by Alluxio.
the class LoadDefinition method runTask.
@Override
public SerializableVoid runTask(LoadConfig config, ArrayList<LoadTask> tasks, RunTaskContext context) throws Exception {
// TODO(jiri): Replace with internal client that uses file ID once the internal client is
// factored out of the core server module. The reason to prefer using file ID for this job is
// to avoid the the race between "replicate" and "rename", so that even a file to replicate is
// renamed, the job is still working on the correct file.
URIStatus status = context.getFileSystem().getStatus(new AlluxioURI(config.getFilePath()));
for (LoadTask task : tasks) {
JobUtils.loadBlock(status, context.getFsContext(), task.getBlockId(), task.getWorkerNetAddress(), config.isDirectCache());
LOG.info("Loaded file " + config.getFilePath() + " block " + task.getBlockId());
}
return null;
}
use of alluxio.job.plan.load.LoadDefinition.LoadTask in project alluxio by Alluxio.
the class LoadDefinitionTest method loadedBySpecifiedHost.
private void loadedBySpecifiedHost(Set<String> workerSet, Set<String> excludedWorkerSet, Set<String> localityIds, Set<String> excludedLocalityIds, Set<Long> workerIds) throws Exception {
int numBlocks = 10;
createFileWithNoLocations(TEST_URI, numBlocks);
LoadConfig config = new LoadConfig(TEST_URI, 1, workerSet, excludedWorkerSet, localityIds, excludedLocalityIds, false);
Set<Pair<WorkerInfo, ArrayList<LoadTask>>> assignments = new LoadDefinition().selectExecutors(config, JOB_WORKERS, new SelectExecutorsContext(1, mJobServerContext));
// Check that we are loading the right number of blocks.
int totalBlockLoads = 0;
for (Pair<WorkerInfo, ArrayList<LoadTask>> assignment : assignments) {
totalBlockLoads += assignment.getSecond().size();
Assert.assertTrue(workerIds.contains(assignment.getFirst().getId()));
}
Assert.assertEquals(numBlocks, totalBlockLoads);
}
use of alluxio.job.plan.load.LoadDefinition.LoadTask in project alluxio by Alluxio.
the class LoadDefinitionTest method replicationSatisfied.
@Test
public void replicationSatisfied() throws Exception {
int numBlocks = 7;
int replication = 3;
createFileWithNoLocations(TEST_URI, numBlocks);
LoadConfig config = new LoadConfig(TEST_URI, replication, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, false);
Set<Pair<WorkerInfo, ArrayList<LoadTask>>> assignments = new LoadDefinition().selectExecutors(config, JOB_WORKERS, new SelectExecutorsContext(1, mJobServerContext));
// Check that we are loading the right number of blocks.
int totalBlockLoads = 0;
for (Pair<WorkerInfo, ArrayList<LoadTask>> assignment : assignments) {
totalBlockLoads += assignment.getSecond().size();
}
Assert.assertEquals(numBlocks * replication, totalBlockLoads);
}
Aggregations