Examples with PersistFile alluxio.thrift.PersistFile used on opensource projects

Example 1 with PersistFile

use of alluxio.thrift.PersistFile in project alluxio by Alluxio.

the class DefaultAsyncPersistHandler method pollFilesToPersist.

/**
   * Polls the files to send to the given worker for persistence. It also removes files from the
   * worker entry in {@link #mWorkerToAsyncPersistFiles}.
   *
   * @param workerId the worker id
   * @return the list of files
   * @throws FileDoesNotExistException if the file does not exist
   * @throws InvalidPathException if the path is invalid
   * @throws AccessControlException if permission checking fails
   */
@Override
public synchronized List<PersistFile> pollFilesToPersist(long workerId) throws FileDoesNotExistException, InvalidPathException, AccessControlException {
    List<PersistFile> filesToPersist = new ArrayList<>();
    List<Long> fileIdsToPersist = new ArrayList<>();
    if (!mWorkerToAsyncPersistFiles.containsKey(workerId)) {
        return filesToPersist;
    }
    Set<Long> scheduledFiles = mWorkerToAsyncPersistFiles.get(workerId);
    for (long fileId : scheduledFiles) {
        FileInfo fileInfo = mFileSystemMasterView.getFileInfo(fileId);
        if (fileInfo.isCompleted()) {
            fileIdsToPersist.add(fileId);
            List<Long> blockIds = new ArrayList<>();
            for (FileBlockInfo fileBlockInfo : mFileSystemMasterView.getFileBlockInfoList(mFileSystemMasterView.getPath(fileId))) {
                blockIds.add(fileBlockInfo.getBlockInfo().getBlockId());
            }
            filesToPersist.add(new PersistFile(fileId, blockIds));
        }
    }
    mWorkerToAsyncPersistFiles.get(workerId).removeAll(fileIdsToPersist);
    return filesToPersist;
}

Example 2 with PersistFile

use of alluxio.thrift.PersistFile in project alluxio by Alluxio.

the class FileWorkerMasterSyncExecutor method heartbeat.

@Override
public void heartbeat() {
    List<Long> persistedFiles = mFileDataManager.getPersistedFiles();
    if (!persistedFiles.isEmpty()) {
        LOG.info("files {} persisted", persistedFiles);
    }
    FileSystemCommand command;
    try {
        command = mMasterClient.heartbeat(mWorkerId.get(), persistedFiles);
    } catch (IOException | AlluxioException e) {
        LOG.error("Failed to heartbeat to master", e);
        return;
    }
    // removes the persisted files that are confirmed
    mFileDataManager.clearPersistedFiles(persistedFiles);
    if (command == null) {
        LOG.error("The command sent from master is null");
        return;
    } else if (command.getCommandType() != CommandType.Persist) {
        LOG.error("The command sent from master should be PERSIST type, but was {}", command.getCommandType());
        return;
    }
    for (PersistFile persistFile : command.getCommandOptions().getPersistOptions().getPersistFiles()) {
        // Enqueue the persist request.
        mPersistFileService.execute(new FilePersister(mFileDataManager, persistFile.getFileId(), persistFile.getBlockIds()));
    }
}

Example 3 with PersistFile

use of alluxio.thrift.PersistFile in project alluxio by Alluxio.

the class DefaultAsyncPersistHandlerTest method scheduleAsyncPersist.

@Test
public void scheduleAsyncPersist() throws Exception {
    DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster));
    AlluxioURI path = new AlluxioURI("/test");
    long blockId = 0;
    long workerId = 1;
    long fileId = 2;
    List<FileBlockInfo> blockInfoList = new ArrayList<>();
    BlockLocation location = new BlockLocation().setWorkerId(workerId);
    blockInfoList.add(new FileBlockInfo().setBlockInfo(new BlockInfo().setBlockId(blockId).setLocations(Lists.newArrayList(location))));
    Mockito.when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList);
    Mockito.when(mFileSystemMaster.getFileId(path)).thenReturn(fileId);
    Mockito.when(mFileSystemMaster.getPath(fileId)).thenReturn(path);
    Mockito.when(mFileSystemMaster.getFileInfo(fileId)).thenReturn(new FileInfo().setLength(1).setCompleted(true));
    handler.scheduleAsyncPersistence(path);
    List<PersistFile> persistFiles = handler.pollFilesToPersist(workerId);
    Assert.assertEquals(1, persistFiles.size());
    Assert.assertEquals(Lists.newArrayList(blockId), persistFiles.get(0).getBlockIds());
}

Example 4 with PersistFile

use of alluxio.thrift.PersistFile in project alluxio by Alluxio.

the class FileSystemMaster method workerHeartbeat.

/**
   * Instructs a worker to persist the files.
   * <p>
   * Needs {@link Mode.Bits#WRITE} permission on the list of files.
   *
   * @param workerId the id of the worker that heartbeats
   * @param persistedFiles the files that persisted on the worker
   * @return the command for persisting the blocks of a file
   * @throws FileDoesNotExistException if the file does not exist
   * @throws InvalidPathException if the file path corresponding to the file id is invalid
   * @throws AccessControlException if permission checking fails
   */
public FileSystemCommand workerHeartbeat(long workerId, List<Long> persistedFiles) throws FileDoesNotExistException, InvalidPathException, AccessControlException {
    for (long fileId : persistedFiles) {
        try {
            // Permission checking for each file is performed inside setAttribute
            setAttribute(getPath(fileId), SetAttributeOptions.defaults().setPersisted(true));
        } catch (FileDoesNotExistException | AccessControlException | InvalidPathException e) {
            LOG.error("Failed to set file {} as persisted, because {}", fileId, e);
        }
    }
    // get the files for the given worker to persist
    List<PersistFile> filesToPersist = mAsyncPersistHandler.pollFilesToPersist(workerId);
    if (!filesToPersist.isEmpty()) {
        LOG.debug("Sent files {} to worker {} to persist", filesToPersist, workerId);
    }
    FileSystemCommandOptions options = new FileSystemCommandOptions();
    options.setPersistOptions(new PersistCommandOptions(filesToPersist));
    return new FileSystemCommand(CommandType.Persist, options);
}