Examples with BlockLocation alluxio.wire.BlockLocation used on opensource projects

Example 1 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class WebInterfaceBrowseServlet method doGet.

/**
   * Populates attribute fields with data from the MasterInfo associated with this servlet. Errors
   * will be displayed in an error field. Debugging can be enabled to display additional data. Will
   * eventually redirect the request to a jsp.
   *
   * @param request the {@link HttpServletRequest} object
   * @param response the {@link HttpServletResponse} object
   * @throws ServletException if the target resource throws this exception
   * @throws IOException if the target resource throws this exception
   */
@Override
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    if (SecurityUtils.isSecurityEnabled() && AuthenticatedClientUser.get() == null) {
        AuthenticatedClientUser.set(LoginUser.get().getName());
    }
    request.setAttribute("debug", Configuration.getBoolean(PropertyKey.DEBUG));
    request.setAttribute("showPermissions", Configuration.getBoolean(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_ENABLED));
    request.setAttribute("masterNodeAddress", mMaster.getRpcAddress().toString());
    request.setAttribute("invalidPathError", "");
    List<FileInfo> filesInfo;
    String requestPath = request.getParameter("path");
    if (requestPath == null || requestPath.isEmpty()) {
        requestPath = AlluxioURI.SEPARATOR;
    }
    AlluxioURI currentPath = new AlluxioURI(requestPath);
    request.setAttribute("currentPath", currentPath.toString());
    request.setAttribute("viewingOffset", 0);
    try {
        long fileId = mMaster.getFileSystemMaster().getFileId(currentPath);
        FileInfo fileInfo = mMaster.getFileSystemMaster().getFileInfo(fileId);
        UIFileInfo currentFileInfo = new UIFileInfo(fileInfo);
        if (currentFileInfo.getAbsolutePath() == null) {
            throw new FileDoesNotExistException(currentPath.toString());
        }
        request.setAttribute("currentDirectory", currentFileInfo);
        request.setAttribute("blockSizeBytes", currentFileInfo.getBlockSizeBytes());
        if (!currentFileInfo.getIsDirectory()) {
            String offsetParam = request.getParameter("offset");
            long relativeOffset = 0;
            long offset;
            try {
                if (offsetParam != null) {
                    relativeOffset = Long.parseLong(offsetParam);
                }
            } catch (NumberFormatException e) {
                relativeOffset = 0;
            }
            String endParam = request.getParameter("end");
            // relative to the end of the file.
            if (endParam == null) {
                offset = relativeOffset;
            } else {
                offset = fileInfo.getLength() - relativeOffset;
            }
            if (offset < 0) {
                offset = 0;
            } else if (offset > fileInfo.getLength()) {
                offset = fileInfo.getLength();
            }
            try {
                displayFile(new AlluxioURI(currentFileInfo.getAbsolutePath()), request, offset);
            } catch (AlluxioException e) {
                throw new IOException(e);
            }
            request.setAttribute("viewingOffset", offset);
            getServletContext().getRequestDispatcher("/viewFile.jsp").forward(request, response);
            return;
        }
        setPathDirectories(currentPath, request);
        filesInfo = mMaster.getFileSystemMaster().listStatus(currentPath, ListStatusOptions.defaults().setLoadMetadataType(LoadMetadataType.Always));
    } catch (FileDoesNotExistException e) {
        request.setAttribute("invalidPathError", "Error: Invalid Path " + e.getMessage());
        getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        return;
    } catch (InvalidPathException e) {
        request.setAttribute("invalidPathError", "Error: Invalid Path " + e.getLocalizedMessage());
        getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        return;
    } catch (IOException e) {
        request.setAttribute("invalidPathError", "Error: File " + currentPath + " is not available " + e.getMessage());
        getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        return;
    } catch (AccessControlException e) {
        request.setAttribute("invalidPathError", "Error: File " + currentPath + " cannot be accessed " + e.getMessage());
        getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        return;
    }
    List<UIFileInfo> fileInfos = new ArrayList<>(filesInfo.size());
    for (FileInfo fileInfo : filesInfo) {
        UIFileInfo toAdd = new UIFileInfo(fileInfo);
        try {
            if (!toAdd.getIsDirectory() && fileInfo.getLength() > 0) {
                FileBlockInfo blockInfo = mMaster.getFileSystemMaster().getFileBlockInfoList(new AlluxioURI(toAdd.getAbsolutePath())).get(0);
                List<String> locations = new ArrayList<>();
                // add the in-memory block locations
                for (BlockLocation location : blockInfo.getBlockInfo().getLocations()) {
                    WorkerNetAddress address = location.getWorkerAddress();
                    locations.add(address.getHost() + ":" + address.getDataPort());
                }
                // add underFS locations
                locations.addAll(blockInfo.getUfsLocations());
                toAdd.setFileLocations(locations);
            }
        } catch (FileDoesNotExistException e) {
            request.setAttribute("FileDoesNotExistException", "Error: non-existing file " + e.getMessage());
            getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
            return;
        } catch (InvalidPathException e) {
            request.setAttribute("InvalidPathException", "Error: invalid path " + e.getMessage());
            getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        } catch (AccessControlException e) {
            request.setAttribute("AccessControlException", "Error: File " + currentPath + " cannot be accessed " + e.getMessage());
            getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
            return;
        }
        fileInfos.add(toAdd);
    }
    Collections.sort(fileInfos, UIFileInfo.PATH_STRING_COMPARE);
    request.setAttribute("nTotalFile", fileInfos.size());
    // URL can not determine offset and limit, let javascript in jsp determine and redirect
    if (request.getParameter("offset") == null && request.getParameter("limit") == null) {
        getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        return;
    }
    try {
        int offset = Integer.parseInt(request.getParameter("offset"));
        int limit = Integer.parseInt(request.getParameter("limit"));
        List<UIFileInfo> sub = fileInfos.subList(offset, offset + limit);
        request.setAttribute("fileInfos", sub);
    } catch (NumberFormatException e) {
        request.setAttribute("fatalError", "Error: offset or limit parse error, " + e.getLocalizedMessage());
        getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        return;
    } catch (IndexOutOfBoundsException e) {
        request.setAttribute("fatalError", "Error: offset or offset + limit is out of bound, " + e.getLocalizedMessage());
        getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        return;
    } catch (IllegalArgumentException e) {
        request.setAttribute("fatalError", e.getLocalizedMessage());
        getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        return;
    }
    getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
}

Example 2 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class UIFileBlockInfo method addLocations.

private void addLocations(FileBlockInfo fileBlockInfo) {
    Set<String> locations = new HashSet<>();
    // add alluxio locations
    for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) {
        locations.add(location.getWorkerAddress().getHost());
    }
    // add underFS locations
    for (String location : fileBlockInfo.getUfsLocations()) {
        locations.add(HostAndPort.fromString(location).getHostText());
    }
    mLocations.addAll(locations);
}

Example 3 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class AlluxioBlockStore method getInStream.

/**
   * Gets a stream to read the data of a block. The stream is backed by Alluxio storage.
   *
   * @param blockId the block to read from
   * @param options the options
   * @return an {@link InputStream} which can be used to read the data in a streaming fashion
   * @throws IOException if the block does not exist
   */
public InputStream getInStream(long blockId, InStreamOptions options) throws IOException {
    BlockInfo blockInfo;
    try (CloseableResource<BlockMasterClient> masterClientResource = mContext.acquireBlockMasterClientResource()) {
        blockInfo = masterClientResource.get().getBlockInfo(blockId);
    } catch (AlluxioException e) {
        throw new IOException(e);
    }
    if (blockInfo.getLocations().isEmpty()) {
        throw new IOException("Block " + blockId + " is not available in Alluxio");
    }
    // for hasLocalWorker is fixed.
    for (BlockLocation location : blockInfo.getLocations()) {
        WorkerNetAddress workerNetAddress = location.getWorkerAddress();
        if (workerNetAddress.getHost().equals(mLocalHostName)) {
            // There is a local worker and the block is local.
            try {
                return StreamFactory.createLocalBlockInStream(mContext, blockId, blockInfo.getLength(), workerNetAddress, options);
            } catch (IOException e) {
                LOG.warn("Failed to open local stream for block {}: {}", blockId, e.getMessage());
                // Getting a local stream failed, do not try again
                break;
            }
        }
    }
    // No local worker/block, choose a random location. In the future we could change this to
    // only randomize among locations in the highest tier, or have the master randomize the order.
    List<BlockLocation> locations = blockInfo.getLocations();
    WorkerNetAddress workerNetAddress = locations.get(mRandom.nextInt(locations.size())).getWorkerAddress();
    return StreamFactory.createRemoteBlockInStream(mContext, blockId, blockInfo.getLength(), workerNetAddress, options);
}

Example 4 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class LocationCommand method runCommand.

@Override
protected void runCommand(AlluxioURI path, CommandLine cl) throws AlluxioException, IOException {
    URIStatus status = mFileSystem.getStatus(path);
    System.out.println(path + " with file id " + status.getFileId() + " is on nodes: ");
    AlluxioBlockStore blockStore = AlluxioBlockStore.create();
    for (long blockId : status.getBlockIds()) {
        for (BlockLocation location : blockStore.getInfo(blockId).getLocations()) {
            System.out.println(location.getWorkerAddress().getHost());
        }
    }
}

Example 5 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getInStreamLocal.

@Test
public void getInStreamLocal() throws Exception {
    WorkerNetAddress remote = new WorkerNetAddress().setHost("remote");
    WorkerNetAddress local = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL);
    // Mock away gRPC usage.
    OpenLocalBlockResponse response = OpenLocalBlockResponse.newBuilder().setPath("/tmp").build();
    when(mWorkerClient.openLocalBlock(any(StreamObserver.class))).thenAnswer(invocation -> {
        mResponseObserver = invocation.getArgument(0, StreamObserver.class);
        return mStreamObserver;
    });
    doAnswer(invocation -> {
        mResponseObserver.onNext(response);
        mResponseObserver.onCompleted();
        return null;
    }).when(mStreamObserver).onNext(any(OpenLocalBlockRequest.class));
    BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays.asList(new BlockLocation().setWorkerAddress(remote), new BlockLocation().setWorkerAddress(local)));
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info);
    assertEquals(local, mBlockStore.getInStream(BLOCK_ID, new InStreamOptions(new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), sConf)).getAddress());
}