Examples with FileInfo alluxio.wire.FileInfo used on opensource projects

Example 36 with FileInfo

use of alluxio.wire.FileInfo in project alluxio by Alluxio.

the class FileDataManager method fileExistsInUfs.

/**
   * Checks if the given file exists in the under storage system.
   *
   * @param fileId the file id
   * @return true if the file exists in under storage system, false otherwise
   * @throws IOException an I/O exception occurs
   */
private synchronized boolean fileExistsInUfs(long fileId) throws IOException {
    FileInfo fileInfo = mBlockWorker.getFileInfo(fileId);
    String dstPath = fileInfo.getUfsPath();
    UnderFileSystem ufs = UnderFileSystem.Factory.get(dstPath);
    return ufs.isFile(dstPath);
}

Example 37 with FileInfo

use of alluxio.wire.FileInfo in project alluxio by Alluxio.

the class FileDataManager method persistFile.

/**
   * Persists the blocks of a file into the under file system.
   *
   * @param fileId the id of the file
   * @param blockIds the list of block ids
   * @throws AlluxioException if an unexpected Alluxio exception is thrown
   * @throws IOException if the file persistence fails
   */
public void persistFile(long fileId, List<Long> blockIds) throws AlluxioException, IOException {
    Map<Long, Long> blockIdToLockId;
    synchronized (mLock) {
        blockIdToLockId = mPersistingInProgressFiles.get(fileId);
        if (blockIdToLockId == null || !blockIdToLockId.keySet().equals(new HashSet<>(blockIds))) {
            throw new IOException("Not all the blocks of file " + fileId + " are locked");
        }
    }
    String dstPath = prepareUfsFilePath(fileId);
    UnderFileSystem ufs = UnderFileSystem.Factory.get(dstPath);
    FileInfo fileInfo = mBlockWorker.getFileInfo(fileId);
    OutputStream outputStream = ufs.create(dstPath, CreateOptions.defaults().setOwner(fileInfo.getOwner()).setGroup(fileInfo.getGroup()).setMode(new Mode((short) fileInfo.getMode())));
    final WritableByteChannel outputChannel = Channels.newChannel(outputStream);
    List<Throwable> errors = new ArrayList<>();
    try {
        for (long blockId : blockIds) {
            long lockId = blockIdToLockId.get(blockId);
            if (Configuration.getBoolean(PropertyKey.WORKER_FILE_PERSIST_RATE_LIMIT_ENABLED)) {
                BlockMeta blockMeta = mBlockWorker.getBlockMeta(Sessions.CHECKPOINT_SESSION_ID, blockId, lockId);
                mPersistenceRateLimiter.acquire((int) blockMeta.getBlockSize());
            }
            // obtain block reader
            BlockReader reader = mBlockWorker.readBlockRemote(Sessions.CHECKPOINT_SESSION_ID, blockId, lockId);
            // write content out
            ReadableByteChannel inputChannel = reader.getChannel();
            BufferUtils.fastCopy(inputChannel, outputChannel);
            reader.close();
        }
    } catch (BlockDoesNotExistException | InvalidWorkerStateException e) {
        errors.add(e);
    } finally {
        // make sure all the locks are released
        for (long lockId : blockIdToLockId.values()) {
            try {
                mBlockWorker.unlockBlock(lockId);
            } catch (BlockDoesNotExistException e) {
                errors.add(e);
            }
        }
        // Process any errors
        if (!errors.isEmpty()) {
            StringBuilder errorStr = new StringBuilder();
            errorStr.append("the blocks of file").append(fileId).append(" are failed to persist\n");
            for (Throwable e : errors) {
                errorStr.append(e).append('\n');
            }
            throw new IOException(errorStr.toString());
        }
    }
    outputStream.flush();
    outputChannel.close();
    outputStream.close();
    synchronized (mLock) {
        mPersistingInProgressFiles.remove(fileId);
        mPersistedFiles.add(fileId);
    }
}

Example 38 with FileInfo

use of alluxio.wire.FileInfo in project alluxio by Alluxio.

the class FileDataManagerTest method errorHandling.

/**
   * Tests that the correct error message is provided when persisting a file fails.
   */
@Test
public void errorHandling() throws Exception {
    long fileId = 1;
    List<Long> blockIds = Lists.newArrayList(1L, 2L);
    FileInfo fileInfo = new FileInfo();
    fileInfo.setPath("test");
    Mockito.when(mBlockWorker.getFileInfo(fileId)).thenReturn(fileInfo);
    for (long blockId : blockIds) {
        Mockito.when(mBlockWorker.lockBlock(Sessions.CHECKPOINT_SESSION_ID, blockId)).thenReturn(blockId);
        Mockito.doThrow(new InvalidWorkerStateException("invalid worker")).when(mBlockWorker).readBlockRemote(Sessions.CHECKPOINT_SESSION_ID, blockId, blockId);
    }
    String ufsRoot = Configuration.get(PropertyKey.UNDERFS_ADDRESS);
    Mockito.when(mUfs.isDirectory(ufsRoot)).thenReturn(true);
    OutputStream outputStream = Mockito.mock(OutputStream.class);
    // mock BufferUtils
    PowerMockito.mockStatic(BufferUtils.class);
    String dstPath = PathUtils.concatPath(ufsRoot, fileInfo.getPath());
    fileInfo.setUfsPath(dstPath);
    Mockito.when(mUfs.create(dstPath)).thenReturn(outputStream);
    Mockito.when(mUfs.create(Mockito.anyString(), Mockito.any(CreateOptions.class))).thenReturn(outputStream);
    Mockito.when(mMockFileSystem.getStatus(Mockito.any(AlluxioURI.class))).thenReturn(new URIStatus(fileInfo));
    mManager.lockBlocks(fileId, blockIds);
    try {
        mManager.persistFile(fileId, blockIds);
        Assert.fail("the persist should fail");
    } catch (IOException e) {
        assertEquals("the blocks of file1 are failed to persist\n" + "alluxio.exception.InvalidWorkerStateException: invalid worker\n", e.getMessage());
        // verify the locks are all unlocked
        Mockito.verify(mBlockWorker).unlockBlock(1L);
        Mockito.verify(mBlockWorker).unlockBlock(2L);
    }
}

Example 39 with FileInfo

use of alluxio.wire.FileInfo in project alluxio by Alluxio.

the class URIStatusTest method fields.

/**
   * Tests getting and setting fields.
   */
@Test
public void fields() {
    FileInfo fileInfo = FileInfoTest.createRandom();
    URIStatus uriStatus = new URIStatus(fileInfo);
    Assert.assertEquals(uriStatus.getBlockIds(), fileInfo.getBlockIds());
    Assert.assertEquals(uriStatus.getBlockSizeBytes(), fileInfo.getBlockSizeBytes());
    Assert.assertEquals(uriStatus.getCreationTimeMs(), fileInfo.getCreationTimeMs());
    Assert.assertEquals(uriStatus.getFileId(), fileInfo.getFileId());
    Assert.assertEquals(uriStatus.getGroup(), fileInfo.getGroup());
    Assert.assertEquals(uriStatus.getInMemoryPercentage(), fileInfo.getInMemoryPercentage());
    Assert.assertEquals(uriStatus.getLastModificationTimeMs(), fileInfo.getLastModificationTimeMs());
    Assert.assertEquals(uriStatus.getLength(), fileInfo.getLength());
    Assert.assertEquals(uriStatus.getName(), fileInfo.getName());
    Assert.assertEquals(uriStatus.getPath(), fileInfo.getPath());
    Assert.assertEquals(uriStatus.getMode(), fileInfo.getMode());
    Assert.assertEquals(uriStatus.getPersistenceState(), fileInfo.getPersistenceState());
    Assert.assertEquals(uriStatus.getTtl(), fileInfo.getTtl());
    Assert.assertEquals(uriStatus.getTtlAction(), fileInfo.getTtlAction());
    Assert.assertEquals(uriStatus.getUfsPath(), fileInfo.getUfsPath());
    Assert.assertEquals(uriStatus.getOwner(), fileInfo.getOwner());
    Assert.assertEquals(uriStatus.isCacheable(), fileInfo.isCacheable());
    Assert.assertEquals(uriStatus.isCompleted(), fileInfo.isCompleted());
    Assert.assertEquals(uriStatus.isFolder(), fileInfo.isFolder());
    Assert.assertEquals(uriStatus.isPersisted(), fileInfo.isPersisted());
    Assert.assertEquals(uriStatus.isPinned(), fileInfo.isPinned());
    Assert.assertEquals(uriStatus.isMountPoint(), fileInfo.isMountPoint());
    Assert.assertEquals(uriStatus.getFileBlockInfos(), fileInfo.getFileBlockInfos());
    Assert.assertEquals(uriStatus.toString(), fileInfo.toString());
}

Example 40 with FileInfo

use of alluxio.wire.FileInfo in project alluxio by Alluxio.

the class WebInterfaceMemoryServlet method doGet.

/**
   * Populates attributes before redirecting to a jsp.
   *
   * @param request the {@link HttpServletRequest} object
   * @param response the {@link HttpServletResponse} object
   * @throws ServletException if the target resource throws this exception
   * @throws IOException if the target resource throws this exception
   */
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    if (SecurityUtils.isSecurityEnabled() && AuthenticatedClientUser.get() == null) {
        AuthenticatedClientUser.set(LoginUser.get().getName());
    }
    request.setAttribute("masterNodeAddress", mMaster.getRpcAddress().toString());
    request.setAttribute("fatalError", "");
    request.setAttribute("showPermissions", Configuration.getBoolean(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_ENABLED));
    List<AlluxioURI> inMemoryFiles = mMaster.getFileSystemMaster().getInMemoryFiles();
    Collections.sort(inMemoryFiles);
    List<UIFileInfo> fileInfos = new ArrayList<>(inMemoryFiles.size());
    for (AlluxioURI file : inMemoryFiles) {
        try {
            long fileId = mMaster.getFileSystemMaster().getFileId(file);
            FileInfo fileInfo = mMaster.getFileSystemMaster().getFileInfo(fileId);
            if (fileInfo != null && fileInfo.getInMemoryPercentage() == 100) {
                fileInfos.add(new UIFileInfo(fileInfo));
            }
        } catch (FileDoesNotExistException e) {
            request.setAttribute("fatalError", "Error: File does not exist " + e.getLocalizedMessage());
            getServletContext().getRequestDispatcher("/memory.jsp").forward(request, response);
            return;
        } catch (AccessControlException e) {
            request.setAttribute("permissionError", "Error: File " + file + " cannot be accessed " + e.getMessage());
            getServletContext().getRequestDispatcher("/memory.jsp").forward(request, response);
            return;
        }
    }
    request.setAttribute("inMemoryFileNum", fileInfos.size());
    // and redirect to "./memory?offset=xxx&limit=xxx"
    if (request.getParameter("offset") == null && request.getParameter("limit") == null) {
        getServletContext().getRequestDispatcher("/memory.jsp").forward(request, response);
        return;
    }
    try {
        int offset = Integer.parseInt(request.getParameter("offset"));
        int limit = Integer.parseInt(request.getParameter("limit"));
        List<UIFileInfo> sub = fileInfos.subList(offset, offset + limit);
        request.setAttribute("fileInfos", sub);
    } catch (NumberFormatException e) {
        request.setAttribute("fatalError", "Error: offset or limit parse error, " + e.getLocalizedMessage());
        getServletContext().getRequestDispatcher("/memory.jsp").forward(request, response);
        return;
    } catch (IndexOutOfBoundsException e) {
        request.setAttribute("fatalError", "Error: offset or offset + limit is out of bound, " + e.getLocalizedMessage());
        getServletContext().getRequestDispatcher("/memory.jsp").forward(request, response);
        return;
    } catch (IllegalArgumentException e) {
        request.setAttribute("fatalError", e.getLocalizedMessage());
        getServletContext().getRequestDispatcher("/memory.jsp").forward(request, response);
        return;
    }
    getServletContext().getRequestDispatcher("/memory.jsp").forward(request, response);
}