Examples with Lineage alluxio.master.lineage.meta.Lineage used on opensource projects

Example 1 with Lineage

use of alluxio.master.lineage.meta.Lineage in project alluxio by Alluxio.

the class LineageMaster method deleteLineageInternal.

private boolean deleteLineageInternal(long lineageId, boolean cascade) throws LineageDoesNotExistException, LineageDeletionException {
    Lineage lineage = mLineageStore.getLineage(lineageId);
    LineageDoesNotExistException.check(lineage != null, ExceptionMessage.LINEAGE_DOES_NOT_EXIST, lineageId);
    // there should not be child lineage if not cascade
    if (!cascade && !mLineageStore.getChildren(lineage).isEmpty()) {
        throw new LineageDeletionException(ExceptionMessage.DELETE_LINEAGE_WITH_CHILDREN.getMessage(lineageId));
    }
    LOG.info("Delete lineage {}", lineageId);
    mLineageStore.deleteLineage(lineageId);
    return true;
}

Example 2 with Lineage

use of alluxio.master.lineage.meta.Lineage in project alluxio by Alluxio.

the class RecomputePlanner method plan.

/**
   * @return a {@link RecomputePlan} that identifies the lineages to recompute
   */
public RecomputePlan plan() {
    List<Long> lostFiles = mFileSystemMaster.getLostFiles();
    // lineage to recompute
    Set<Lineage> toRecompute = new HashSet<>();
    if (!lostFiles.isEmpty()) {
        LOG.info("report lost files {}", lostFiles);
        // report lost files
        for (long lostFile : lostFiles) {
            if (!mLineageStore.hasOutputFile(lostFile)) {
                continue;
            }
            Lineage lineage;
            try {
                lineage = mLineageStore.getLineageOfOutputFile(lostFile);
            } catch (LineageDoesNotExistException e) {
                // should not happen
                throw new IllegalStateException(e);
            }
            try {
                if (!LineageStateUtils.isPersisted(lineage, mFileSystemMaster.getFileSystemMasterView())) {
                    toRecompute.add(lineage);
                }
            } catch (FileDoesNotExistException e) {
                // should not happen
                throw new IllegalStateException(e);
            }
        }
    }
    List<Lineage> toRecomputeAfterSort = mLineageStore.sortLineageTopologically(toRecompute);
    return new RecomputePlan(toRecomputeAfterSort);
}

Example 3 with Lineage

use of alluxio.master.lineage.meta.Lineage in project alluxio by Alluxio.

the class TestRecomputeExecutor method recomputeLauncher.

/**
   * Tests recompute executor creates a recompute plan and launches the recompute job at heartbeat.
   *
   * @throws Exception if anything wrong happens
   */
@Test
public void recomputeLauncher() throws Exception {
    long fileId = 5L;
    // mock planner
    RecomputePlanner planner = Mockito.mock(RecomputePlanner.class);
    Job job = Mockito.mock(Job.class);
    Lineage lineage = new Lineage(1, new ArrayList<Long>(), Lists.newArrayList(fileId), job);
    Mockito.when(planner.plan()).thenReturn(new RecomputePlan(Lists.newArrayList(lineage)));
    // mock file system master
    FileSystemMaster fileSystemMaster = Mockito.mock(FileSystemMaster.class);
    Mockito.when(fileSystemMaster.getFileSystemMasterView()).thenReturn(new FileSystemMasterView(fileSystemMaster));
    Mockito.when(fileSystemMaster.getLostFiles()).thenReturn(Lists.newArrayList(fileId));
    RecomputeExecutor executor = new RecomputeExecutor(planner, fileSystemMaster);
    // wait for the executor to finish running
    executor.heartbeatWithFuture().get(5, TimeUnit.SECONDS);
    executor.close();
    Mockito.verify(fileSystemMaster).resetFile(fileId);
    Mockito.verify(job).run();
}

Example 4 with Lineage

use of alluxio.master.lineage.meta.Lineage in project alluxio by Alluxio.

the class LineageMaster method getLineageInfoList.

/**
   * @return the list of all the {@link LineageInfo}s
   * @throws LineageDoesNotExistException if the lineage does not exist
   * @throws FileDoesNotExistException if any associated file does not exist
   */
public synchronized List<LineageInfo> getLineageInfoList() throws LineageDoesNotExistException, FileDoesNotExistException {
    List<LineageInfo> lineages = new ArrayList<>();
    for (Lineage lineage : mLineageStore.getAllInTopologicalOrder()) {
        LineageInfo info = new LineageInfo();
        List<Long> parents = new ArrayList<>();
        for (Lineage parent : mLineageStore.getParents(lineage)) {
            parents.add(parent.getId());
        }
        info.setParents(parents);
        List<Long> children = new ArrayList<>();
        for (Lineage child : mLineageStore.getChildren(lineage)) {
            children.add(child.getId());
        }
        info.setChildren(children);
        info.setId(lineage.getId());
        List<String> inputFiles = new ArrayList<>();
        for (long inputFileId : lineage.getInputFiles()) {
            inputFiles.add(mFileSystemMaster.getPath(inputFileId).toString());
        }
        info.setInputFiles(inputFiles);
        List<String> outputFiles = new ArrayList<>();
        for (long outputFileId : lineage.getOutputFiles()) {
            outputFiles.add(mFileSystemMaster.getPath(outputFileId).toString());
        }
        info.setOutputFiles(outputFiles);
        info.setCreationTimeMs(lineage.getCreationTime());
        info.setJob(((CommandLineJob) lineage.getJob()).generateCommandLineJobInfo());
        lineages.add(info);
    }
    return lineages;
}

Example 5 with Lineage

use of alluxio.master.lineage.meta.Lineage in project alluxio by Alluxio.

the class CheckpointLatestPlanner method generatePlan.

@Override
public CheckpointPlan generatePlan(LineageStoreView store, FileSystemMasterView fileSystemMasterView) {
    Lineage toCheckpoint = null;
    long latestCreated = 0;
    for (Lineage lineage : store.getAllLineagesInTopologicalOrder()) {
        try {
            if (!LineageStateUtils.isCompleted(lineage, fileSystemMasterView) || LineageStateUtils.isPersisted(lineage, fileSystemMasterView) || LineageStateUtils.needRecompute(lineage, fileSystemMasterView) || LineageStateUtils.isInCheckpointing(lineage, fileSystemMasterView)) {
                continue;
            }
        } catch (FileDoesNotExistException | AccessControlException e) {
            LOG.error("The lineage file does not exist", e);
            continue;
        }
        if (lineage.getCreationTime() > latestCreated) {
            latestCreated = lineage.getCreationTime();
            toCheckpoint = lineage;
        }
    }
    return toCheckpoint == null ? new CheckpointPlan(new ArrayList<Long>()) : new CheckpointPlan(Lists.newArrayList(toCheckpoint.getId()));
}