Examples with PersistFile alluxio.wire.PersistFile used on opensource projects

Example 1 with PersistFile

use of alluxio.wire.PersistFile in project alluxio by Alluxio.

the class DefaultFileSystemMaster method workerHeartbeat.

@Override
public FileSystemCommand workerHeartbeat(long workerId, List<Long> persistedFiles, WorkerHeartbeatContext context) throws IOException {
    List<String> persistedUfsFingerprints = context.getOptions().getPersistedFileFingerprintsList();
    boolean hasPersistedFingerprints = persistedUfsFingerprints.size() == persistedFiles.size();
    for (int i = 0; i < persistedFiles.size(); i++) {
        long fileId = persistedFiles.get(i);
        String ufsFingerprint = hasPersistedFingerprints ? persistedUfsFingerprints.get(i) : Constants.INVALID_UFS_FINGERPRINT;
        try {
            // Permission checking for each file is performed inside setAttribute
            setAttribute(getPath(fileId), SetAttributeContext.mergeFrom(SetAttributePOptions.newBuilder().setPersisted(true)).setUfsFingerprint(ufsFingerprint));
        } catch (FileDoesNotExistException | AccessControlException | InvalidPathException e) {
            LOG.error("Failed to set file {} as persisted, because {}", fileId, e);
        }
    }
    // TODO(zac) Clean up master and worker code since this is taken care of by job service now.
    // Worker should not persist any files. Instead, files are persisted through job service.
    List<PersistFile> filesToPersist = new ArrayList<>();
    FileSystemCommandOptions commandOptions = new FileSystemCommandOptions();
    commandOptions.setPersistOptions(new PersistCommandOptions(filesToPersist));
    return new FileSystemCommand(CommandType.PERSIST, commandOptions);
}

Example 2 with PersistFile

use of alluxio.wire.PersistFile in project alluxio by Alluxio.

the class DefaultAsyncPersistHandler method pollFilesToPersist.

/**
 * Polls the files to send to the given worker for persistence. It also removes files from the
 * worker entry in {@link #mWorkerToAsyncPersistFiles}.
 *
 * @param workerId the worker id
 * @return the list of files
 * @throws FileDoesNotExistException if the file does not exist
 * @throws InvalidPathException if the path is invalid
 * @throws AccessControlException if permission checking fails
 */
@Override
public synchronized List<PersistFile> pollFilesToPersist(long workerId) throws FileDoesNotExistException, InvalidPathException, AccessControlException {
    List<PersistFile> filesToPersist = new ArrayList<>();
    List<Long> fileIdsToPersist = new ArrayList<>();
    if (!mWorkerToAsyncPersistFiles.containsKey(workerId)) {
        return filesToPersist;
    }
    Set<Long> scheduledFiles = mWorkerToAsyncPersistFiles.get(workerId);
    try {
        for (long fileId : scheduledFiles) {
            try {
                FileInfo fileInfo = mFileSystemMasterView.getFileInfo(fileId);
                if (fileInfo.isCompleted()) {
                    fileIdsToPersist.add(fileId);
                    List<Long> blockIds = new ArrayList<>();
                    for (FileBlockInfo fileBlockInfo : mFileSystemMasterView.getFileBlockInfoList(mFileSystemMasterView.getPath(fileId))) {
                        blockIds.add(fileBlockInfo.getBlockInfo().getBlockId());
                    }
                    filesToPersist.add(new PersistFile(fileId, blockIds));
                }
            } catch (FileDoesNotExistException e) {
                LOG.warn("FileId {} does not exist, ignore persistence it", fileId);
            }
        }
    } catch (UnavailableException e) {
        return filesToPersist;
    }
    mWorkerToAsyncPersistFiles.get(workerId).removeAll(fileIdsToPersist);
    return filesToPersist;
}

Example 3 with PersistFile

use of alluxio.wire.PersistFile in project alluxio by Alluxio.

the class DefaultAsyncPersistHandlerTest method scheduleAsyncPersist.

@Test
public void scheduleAsyncPersist() throws Exception {
    DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster));
    AlluxioURI path = new AlluxioURI("/test");
    long blockId = 0;
    long workerId = 1;
    long fileId = 2;
    List<FileBlockInfo> blockInfoList = new ArrayList<>();
    BlockLocation location = new BlockLocation().setWorkerId(workerId);
    blockInfoList.add(new FileBlockInfo().setBlockInfo(new BlockInfo().setBlockId(blockId).setLocations(Lists.newArrayList(location))));
    when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList);
    when(mFileSystemMaster.getFileId(path)).thenReturn(fileId);
    when(mFileSystemMaster.getPath(fileId)).thenReturn(path);
    when(mFileSystemMaster.getFileInfo(fileId)).thenReturn(new FileInfo().setLength(1).setCompleted(true));
    handler.scheduleAsyncPersistence(path);
    List<PersistFile> persistFiles = handler.pollFilesToPersist(workerId);
    assertEquals(1, persistFiles.size());
    assertEquals(Lists.newArrayList(blockId), persistFiles.get(0).getBlockIds());
}

Example 4 with PersistFile

use of alluxio.wire.PersistFile in project alluxio by Alluxio.

the class DefaultAsyncPersistHandlerTest method persistenceFileAfterDeletion.

/**
 * Tests persistence after deletion of files.
 */
@Test
public void persistenceFileAfterDeletion() throws Exception {
    DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster));
    AlluxioURI path = new AlluxioURI("/test");
    long blockId = 0;
    long workerId = 1;
    long fileId = 2;
    List<FileBlockInfo> blockInfoList = new ArrayList<>();
    BlockLocation location = new BlockLocation().setWorkerId(workerId);
    blockInfoList.add(new FileBlockInfo().setBlockInfo(new BlockInfo().setBlockId(blockId).setLocations(Lists.newArrayList(location))));
    when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList);
    when(mFileSystemMaster.getFileId(path)).thenReturn(fileId);
    when(mFileSystemMaster.getPath(fileId)).thenReturn(path);
    when(mFileSystemMaster.getFileInfo(fileId)).thenReturn(new FileInfo().setLength(1).setCompleted(true));
    handler.scheduleAsyncPersistence(path);
    when(mFileSystemMaster.getFileInfo(fileId)).thenThrow(new FileDoesNotExistException("no file"));
    List<PersistFile> persistFiles = handler.pollFilesToPersist(workerId);
    assertEquals(0, persistFiles.size());
}