Examples with BlockDoesNotExistException alluxio.exception.BlockDoesNotExistException used on opensource projects

Example 6 with BlockDoesNotExistException

use of alluxio.exception.BlockDoesNotExistException in project alluxio by Alluxio.

the class FileDataManager method persistFile.

/**
   * Persists the blocks of a file into the under file system.
   *
   * @param fileId the id of the file
   * @param blockIds the list of block ids
   * @throws AlluxioException if an unexpected Alluxio exception is thrown
   * @throws IOException if the file persistence fails
   */
public void persistFile(long fileId, List<Long> blockIds) throws AlluxioException, IOException {
    Map<Long, Long> blockIdToLockId;
    synchronized (mLock) {
        blockIdToLockId = mPersistingInProgressFiles.get(fileId);
        if (blockIdToLockId == null || !blockIdToLockId.keySet().equals(new HashSet<>(blockIds))) {
            throw new IOException("Not all the blocks of file " + fileId + " are locked");
        }
    }
    String dstPath = prepareUfsFilePath(fileId);
    UnderFileSystem ufs = UnderFileSystem.Factory.get(dstPath);
    FileInfo fileInfo = mBlockWorker.getFileInfo(fileId);
    OutputStream outputStream = ufs.create(dstPath, CreateOptions.defaults().setOwner(fileInfo.getOwner()).setGroup(fileInfo.getGroup()).setMode(new Mode((short) fileInfo.getMode())));
    final WritableByteChannel outputChannel = Channels.newChannel(outputStream);
    List<Throwable> errors = new ArrayList<>();
    try {
        for (long blockId : blockIds) {
            long lockId = blockIdToLockId.get(blockId);
            if (Configuration.getBoolean(PropertyKey.WORKER_FILE_PERSIST_RATE_LIMIT_ENABLED)) {
                BlockMeta blockMeta = mBlockWorker.getBlockMeta(Sessions.CHECKPOINT_SESSION_ID, blockId, lockId);
                mPersistenceRateLimiter.acquire((int) blockMeta.getBlockSize());
            }
            // obtain block reader
            BlockReader reader = mBlockWorker.readBlockRemote(Sessions.CHECKPOINT_SESSION_ID, blockId, lockId);
            // write content out
            ReadableByteChannel inputChannel = reader.getChannel();
            BufferUtils.fastCopy(inputChannel, outputChannel);
            reader.close();
        }
    } catch (BlockDoesNotExistException | InvalidWorkerStateException e) {
        errors.add(e);
    } finally {
        // make sure all the locks are released
        for (long lockId : blockIdToLockId.values()) {
            try {
                mBlockWorker.unlockBlock(lockId);
            } catch (BlockDoesNotExistException e) {
                errors.add(e);
            }
        }
        // Process any errors
        if (!errors.isEmpty()) {
            StringBuilder errorStr = new StringBuilder();
            errorStr.append("the blocks of file").append(fileId).append(" are failed to persist\n");
            for (Throwable e : errors) {
                errorStr.append(e).append('\n');
            }
            throw new IOException(errorStr.toString());
        }
    }
    outputStream.flush();
    outputChannel.close();
    outputStream.close();
    synchronized (mLock) {
        mPersistingInProgressFiles.remove(fileId);
        mPersistedFiles.add(fileId);
    }
}

Example 7 with BlockDoesNotExistException

use of alluxio.exception.BlockDoesNotExistException in project alluxio by Alluxio.

the class NettyDataServerTest method blockWorkerBlockDoesNotExistExceptionCausesFileDneStatus.

@Test
public void blockWorkerBlockDoesNotExistExceptionCausesFileDneStatus() throws Exception {
    when(mBlockWorker.readBlockRemote(anyLong(), anyLong(), anyLong())).thenThrow(new BlockDoesNotExistException(""));
    RPCResponse response = request(new RPCBlockReadRequest(1, 2, 3, 4, 0));
    // Verify that the read request failed with a FILE_DNE status.
    assertEquals(RPCResponse.Status.FILE_DNE, response.getStatus());
}

Example 8 with BlockDoesNotExistException

use of alluxio.exception.BlockDoesNotExistException in project alluxio by Alluxio.

the class RemoteBlockInStreamIntegrationTest method remoteReadLock.

/**
   * Tests remote read stream lock in {@link RemoteBlockInStream}.
   */
@Test
public void remoteReadLock() throws Exception {
    HeartbeatScheduler.await(HeartbeatContext.WORKER_BLOCK_SYNC, 10, TimeUnit.SECONDS);
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
        AlluxioURI uri = new AlluxioURI(uniqPath + "/file_" + k);
        FileSystemTestUtils.createByteFile(mFileSystem, uri, mWriteAlluxio, k);
        HeartbeatScheduler.execute(HeartbeatContext.WORKER_BLOCK_SYNC);
        long blockId = mFileSystem.getStatus(uri).getBlockIds().get(0);
        BlockInfo info = AlluxioBlockStore.create().getInfo(blockId);
        WorkerNetAddress workerAddr = info.getLocations().get(0).getWorkerAddress();
        RemoteBlockInStream is = RemoteBlockInStream.create(info.getBlockId(), info.getLength(), workerAddr, FileSystemContext.INSTANCE, InStreamOptions.defaults());
        Assert.assertEquals(0, is.read());
        mFileSystem.delete(uri);
        HeartbeatScheduler.execute(HeartbeatContext.WORKER_BLOCK_SYNC);
        // The file has been deleted.
        Assert.assertFalse(mFileSystem.exists(uri));
        // Look! We can still read the deleted file since we have a lock!
        byte[] ret = new byte[k / 2];
        Assert.assertEquals(k / 2, is.read(ret, 0, k / 2));
        is.close();
        Assert.assertFalse(mFileSystem.exists(uri));
        // Try to create an in stream again, and it should fail.
        RemoteBlockInStream is2 = null;
        try {
            is2 = RemoteBlockInStream.create(info.getBlockId(), info.getLength(), workerAddr, FileSystemContext.INSTANCE, InStreamOptions.defaults());
        } catch (IOException e) {
            Assert.assertTrue(e.getCause() instanceof BlockDoesNotExistException);
        } finally {
            if (is2 != null) {
                is2.close();
            }
        }
    }
}

Example 9 with BlockDoesNotExistException

use of alluxio.exception.BlockDoesNotExistException in project alluxio by Alluxio.

the class AlluxioWorkerRestServiceHandler method getWebUIBlockInfo.

/**
 * Gets web ui block info page data.
 *
 * @param requestPath the request path
 * @param requestOffset the request offset
 * @param requestLimit the request limit
 * @return the response object
 */
@GET
@Path(WEBUI_BLOCKINFO)
public Response getWebUIBlockInfo(@QueryParam("path") String requestPath, @DefaultValue("0") @QueryParam("offset") String requestOffset, @DefaultValue("20") @QueryParam("limit") String requestLimit) {
    return RestUtils.call(() -> {
        WorkerWebUIBlockInfo response = new WorkerWebUIBlockInfo();
        if (!ServerConfiguration.getBoolean(PropertyKey.WEB_FILE_INFO_ENABLED)) {
            return response;
        }
        response.setFatalError("").setInvalidPathError("");
        if (!(requestPath == null || requestPath.isEmpty())) {
            // Display file block info
            try {
                URIStatus status = mFsClient.getStatus(new AlluxioURI(requestPath));
                UIFileInfo uiFileInfo = new UIFileInfo(status, ServerConfiguration.global(), new WorkerStorageTierAssoc().getOrderedStorageAliases());
                for (long blockId : status.getBlockIds()) {
                    if (mBlockWorker.hasBlockMeta(blockId)) {
                        BlockMeta blockMeta = mBlockWorker.getVolatileBlockMeta(blockId);
                        long blockSize = blockMeta.getBlockSize();
                        // The block last access time is not available. Use -1 for now.
                        // It's not necessary to show location information here since
                        // we are viewing at the context of this worker.
                        uiFileInfo.addBlock(blockMeta.getBlockLocation().tierAlias(), blockId, blockSize, -1);
                    }
                }
                List<ImmutablePair<String, List<UIFileBlockInfo>>> fileBlocksOnTier = new ArrayList<>();
                for (Map.Entry<String, List<UIFileBlockInfo>> e : uiFileInfo.getBlocksOnTier().entrySet()) {
                    fileBlocksOnTier.add(new ImmutablePair<>(e.getKey(), e.getValue()));
                }
                response.setFileBlocksOnTier(fileBlocksOnTier).setBlockSizeBytes(uiFileInfo.getBlockSizeBytes()).setPath(requestPath);
            } catch (FileDoesNotExistException e) {
                response.setFatalError("Error: Invalid Path " + e.getMessage());
            } catch (IOException e) {
                response.setInvalidPathError("Error: File " + requestPath + " is not available " + e.getMessage());
            } catch (BlockDoesNotExistException e) {
                response.setFatalError("Error: block not found. " + e.getMessage());
            } catch (AlluxioException e) {
                response.setFatalError("Error: alluxio exception. " + e.getMessage());
            }
        }
        Set<Long> unsortedFileIds = new HashSet<>();
        BlockStoreMeta storeMeta = mBlockWorker.getStoreMetaFull();
        for (List<Long> blockIds : storeMeta.getBlockList().values()) {
            for (long blockId : blockIds) {
                unsortedFileIds.add(BlockId.getFileId(blockId));
            }
        }
        List<Long> fileIds = new ArrayList<>(unsortedFileIds);
        Collections.sort(fileIds);
        response.setNTotalFile(unsortedFileIds.size()).setOrderedTierAliases(new WorkerStorageTierAssoc().getOrderedStorageAliases());
        try {
            int offset = Integer.parseInt(requestOffset);
            int limit = Integer.parseInt(requestLimit);
            // make the limit the total number of files if request limit is > than what is available
            limit = offset == 0 && limit > fileIds.size() ? fileIds.size() : limit;
            // offset+limit can't be greater than the size of the list
            limit = offset + limit > fileIds.size() ? fileIds.size() - offset : limit;
            int sum = Math.addExact(offset, limit);
            List<Long> subFileIds = fileIds.subList(offset, sum);
            List<UIFileInfo> uiFileInfos = new ArrayList<>(subFileIds.size());
            for (long fileId : subFileIds) {
                try {
                    URIStatus status = new URIStatus(mBlockWorker.getFileInfo(fileId));
                    UIFileInfo uiFileInfo = new UIFileInfo(status, ServerConfiguration.global(), new WorkerStorageTierAssoc().getOrderedStorageAliases());
                    for (long blockId : status.getBlockIds()) {
                        if (mBlockWorker.hasBlockMeta(blockId)) {
                            BlockMeta blockMeta = mBlockWorker.getVolatileBlockMeta(blockId);
                            long blockSize = blockMeta.getBlockSize();
                            // The block last access time is not available. Use -1 for now.
                            // It's not necessary to show location information here since
                            // we are viewing at the context of this worker.
                            uiFileInfo.addBlock(blockMeta.getBlockLocation().tierAlias(), blockId, blockSize, -1);
                        }
                    }
                    if (!uiFileInfo.getBlockIds().isEmpty()) {
                        uiFileInfos.add(uiFileInfo);
                    }
                } catch (Exception e) {
                    // The file might have been deleted, log a warning and ignore this file.
                    LOG.warn("Unable to get file info for fileId {}. {}", fileId, e.toString());
                }
            }
            response.setFileInfos(uiFileInfos);
        } catch (NumberFormatException e) {
            response.setFatalError("Error: offset or limit parse error, " + e.getLocalizedMessage());
        } catch (ArithmeticException e) {
            response.setFatalError("Error: offset or offset + limit is out ofbound, " + e.getLocalizedMessage());
        } catch (Exception e) {
            response.setFatalError(e.getLocalizedMessage());
        }
        return response;
    }, ServerConfiguration.global());
}

Example 10 with BlockDoesNotExistException

use of alluxio.exception.BlockDoesNotExistException in project alluxio by Alluxio.

the class TieredBlockStore method commitBlockInternal.

/**
 * Commits a temp block.
 *
 * @param sessionId the id of session
 * @param blockId the id of block
 * @param pinOnCreate is block pinned on create
 * @return destination location to move the block
 * @throws BlockDoesNotExistException if block id can not be found in temporary blocks
 * @throws BlockAlreadyExistsException if block id already exists in committed blocks
 * @throws InvalidWorkerStateException if block id is not owned by session id
 */
private BlockStoreLocation commitBlockInternal(long sessionId, long blockId, boolean pinOnCreate) throws BlockAlreadyExistsException, InvalidWorkerStateException, BlockDoesNotExistException, IOException {
    // When committing TempBlockMeta, the final BlockMeta calculates the block size according to
    // the actual file size of this TempBlockMeta. Therefore, commitTempBlockMeta must happen
    // after moving actual block file to its committed path.
    BlockStoreLocation loc;
    String srcPath;
    String dstPath;
    TempBlockMeta tempBlockMeta;
    try (LockResource r = new LockResource(mMetadataReadLock)) {
        checkTempBlockOwnedBySession(sessionId, blockId);
        tempBlockMeta = mMetaManager.getTempBlockMeta(blockId);
        srcPath = tempBlockMeta.getPath();
        dstPath = tempBlockMeta.getCommitPath();
        loc = tempBlockMeta.getBlockLocation();
    }
    // Heavy IO is guarded by block lock but not metadata lock. This may throw IOException.
    FileUtils.move(srcPath, dstPath);
    try (LockResource r = new LockResource(mMetadataWriteLock)) {
        mMetaManager.commitTempBlockMeta(tempBlockMeta);
    } catch (BlockAlreadyExistsException | BlockDoesNotExistException | WorkerOutOfSpaceException e) {
        // we shall never reach here
        throw Throwables.propagate(e);
    }
    // Check if block is pinned on commit
    if (pinOnCreate) {
        addToPinnedInodes(BlockId.getFileId(blockId));
    }
    return loc;
}