Examples with FileSystemMasterView alluxio.master.file.meta.FileSystemMasterView used on opensource projects

Example 1 with FileSystemMasterView

use of alluxio.master.file.meta.FileSystemMasterView in project alluxio by Alluxio.

the class CheckpointLatestPlannerTest method before.

/**
   * Sets up all dependencies before a test runs.
   */
@Before
public void before() {
    mLineageStore = new LineageStore(new LineageIdGenerator());
    mJob = new CommandLineJob("test", new JobConf("output"));
    mFileSystemMaster = Mockito.mock(FileSystemMaster.class);
    mPlanner = new CheckpointLatestPlanner(new LineageStoreView(mLineageStore), new FileSystemMasterView(mFileSystemMaster));
}

Example 2 with FileSystemMasterView

use of alluxio.master.file.meta.FileSystemMasterView in project alluxio by Alluxio.

the class RecomputePlannerTest method before.

/**
   * Sets up the dependencies before a test runs.
   */
@Before
public void before() {
    mLineageStore = new LineageStore(new LineageIdGenerator());
    mJob = new CommandLineJob("test", new JobConf("output"));
    mFileSystemMaster = Mockito.mock(FileSystemMaster.class);
    Mockito.when(mFileSystemMaster.getFileSystemMasterView()).thenReturn(new FileSystemMasterView(mFileSystemMaster));
    mPlanner = new RecomputePlanner(mLineageStore, mFileSystemMaster);
}

Example 3 with FileSystemMasterView

use of alluxio.master.file.meta.FileSystemMasterView in project alluxio by Alluxio.

the class TestRecomputeExecutor method recomputeLauncher.

/**
   * Tests recompute executor creates a recompute plan and launches the recompute job at heartbeat.
   *
   * @throws Exception if anything wrong happens
   */
@Test
public void recomputeLauncher() throws Exception {
    long fileId = 5L;
    // mock planner
    RecomputePlanner planner = Mockito.mock(RecomputePlanner.class);
    Job job = Mockito.mock(Job.class);
    Lineage lineage = new Lineage(1, new ArrayList<Long>(), Lists.newArrayList(fileId), job);
    Mockito.when(planner.plan()).thenReturn(new RecomputePlan(Lists.newArrayList(lineage)));
    // mock file system master
    FileSystemMaster fileSystemMaster = Mockito.mock(FileSystemMaster.class);
    Mockito.when(fileSystemMaster.getFileSystemMasterView()).thenReturn(new FileSystemMasterView(fileSystemMaster));
    Mockito.when(fileSystemMaster.getLostFiles()).thenReturn(Lists.newArrayList(fileId));
    RecomputeExecutor executor = new RecomputeExecutor(planner, fileSystemMaster);
    // wait for the executor to finish running
    executor.heartbeatWithFuture().get(5, TimeUnit.SECONDS);
    executor.close();
    Mockito.verify(fileSystemMaster).resetFile(fileId);
    Mockito.verify(job).run();
}

Example 4 with FileSystemMasterView

use of alluxio.master.file.meta.FileSystemMasterView in project alluxio by Alluxio.

the class DefaultAsyncPersistHandlerTest method persistenceFileWithBlocksOnMultipleWorkers.

/**
 * Tests the persistence of file with block on multiple workers.
 */
@Test
public void persistenceFileWithBlocksOnMultipleWorkers() throws Exception {
    DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster));
    AlluxioURI path = new AlluxioURI("/test");
    List<FileBlockInfo> blockInfoList = new ArrayList<>();
    BlockLocation location1 = new BlockLocation().setWorkerId(1);
    blockInfoList.add(new FileBlockInfo().setBlockInfo(new BlockInfo().setLocations(Lists.newArrayList(location1))));
    BlockLocation location2 = new BlockLocation().setWorkerId(2);
    blockInfoList.add(new FileBlockInfo().setBlockInfo(new BlockInfo().setLocations(Lists.newArrayList(location2))));
    long fileId = 2;
    when(mFileSystemMaster.getFileId(path)).thenReturn(fileId);
    when(mFileSystemMaster.getFileInfo(fileId)).thenReturn(new FileInfo().setLength(1).setCompleted(true));
    when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList);
    // no persist scheduled on any worker
    assertEquals(0, handler.pollFilesToPersist(1).size());
    assertEquals(0, handler.pollFilesToPersist(2).size());
}

Example 5 with FileSystemMasterView

use of alluxio.master.file.meta.FileSystemMasterView in project alluxio by Alluxio.

the class CheckpointLatestPlannerTest method schedule.

/**
   * Tests the {@link CheckpointLatestPlanner#generatePlan(LineageStoreView, FileSystemMasterView)}
   * method.
   */
@Test
public void schedule() throws Exception {
    long fileId1 = 1L;
    long fileId2 = 2L;
    long l1 = mLineageStore.createLineage(new ArrayList<Long>(), Lists.newArrayList(fileId1), mJob);
    // Sleep for 1ms to guarantee that the next lineage's creation time is later than the first's
    CommonUtils.sleepMs(1);
    long l2 = mLineageStore.createLineage(Lists.newArrayList(fileId1), Lists.newArrayList(fileId2), mJob);
    Mockito.when(mFileSystemMaster.getPersistenceState(fileId1)).thenReturn(PersistenceState.NOT_PERSISTED);
    Mockito.when(mFileSystemMaster.getPersistenceState(fileId2)).thenReturn(PersistenceState.NOT_PERSISTED);
    FileInfo fileInfo1 = new FileInfo();
    fileInfo1.setCompleted(true);
    Mockito.when(mFileSystemMaster.getFileInfo(fileId1)).thenReturn(fileInfo1);
    FileInfo fileInfo2 = new FileInfo();
    fileInfo2.setCompleted(false);
    Mockito.when(mFileSystemMaster.getFileInfo(fileId2)).thenReturn(fileInfo2);
    CheckpointPlan plan = mPlanner.generatePlan(new LineageStoreView(mLineageStore), new FileSystemMasterView(mFileSystemMaster));
    Assert.assertEquals((Long) l1, plan.getLineagesToCheckpoint().get(0));
    // complete file 2 and it's ready for checkpoint
    fileInfo2.setCompleted(true);
    plan = mPlanner.generatePlan(new LineageStoreView(mLineageStore), new FileSystemMasterView(mFileSystemMaster));
    Assert.assertEquals((Long) l2, plan.getLineagesToCheckpoint().get(0));
}