Examples with WorkerNetAddress alluxio.wire.WorkerNetAddress used on opensource projects

Example 1 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class WebInterfaceBrowseServlet method doGet.

/**
   * Populates attribute fields with data from the MasterInfo associated with this servlet. Errors
   * will be displayed in an error field. Debugging can be enabled to display additional data. Will
   * eventually redirect the request to a jsp.
   *
   * @param request the {@link HttpServletRequest} object
   * @param response the {@link HttpServletResponse} object
   * @throws ServletException if the target resource throws this exception
   * @throws IOException if the target resource throws this exception
   */
@Override
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    if (SecurityUtils.isSecurityEnabled() && AuthenticatedClientUser.get() == null) {
        AuthenticatedClientUser.set(LoginUser.get().getName());
    }
    request.setAttribute("debug", Configuration.getBoolean(PropertyKey.DEBUG));
    request.setAttribute("showPermissions", Configuration.getBoolean(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_ENABLED));
    request.setAttribute("masterNodeAddress", mMaster.getRpcAddress().toString());
    request.setAttribute("invalidPathError", "");
    List<FileInfo> filesInfo;
    String requestPath = request.getParameter("path");
    if (requestPath == null || requestPath.isEmpty()) {
        requestPath = AlluxioURI.SEPARATOR;
    }
    AlluxioURI currentPath = new AlluxioURI(requestPath);
    request.setAttribute("currentPath", currentPath.toString());
    request.setAttribute("viewingOffset", 0);
    try {
        long fileId = mMaster.getFileSystemMaster().getFileId(currentPath);
        FileInfo fileInfo = mMaster.getFileSystemMaster().getFileInfo(fileId);
        UIFileInfo currentFileInfo = new UIFileInfo(fileInfo);
        if (currentFileInfo.getAbsolutePath() == null) {
            throw new FileDoesNotExistException(currentPath.toString());
        }
        request.setAttribute("currentDirectory", currentFileInfo);
        request.setAttribute("blockSizeBytes", currentFileInfo.getBlockSizeBytes());
        if (!currentFileInfo.getIsDirectory()) {
            String offsetParam = request.getParameter("offset");
            long relativeOffset = 0;
            long offset;
            try {
                if (offsetParam != null) {
                    relativeOffset = Long.parseLong(offsetParam);
                }
            } catch (NumberFormatException e) {
                relativeOffset = 0;
            }
            String endParam = request.getParameter("end");
            // relative to the end of the file.
            if (endParam == null) {
                offset = relativeOffset;
            } else {
                offset = fileInfo.getLength() - relativeOffset;
            }
            if (offset < 0) {
                offset = 0;
            } else if (offset > fileInfo.getLength()) {
                offset = fileInfo.getLength();
            }
            try {
                displayFile(new AlluxioURI(currentFileInfo.getAbsolutePath()), request, offset);
            } catch (AlluxioException e) {
                throw new IOException(e);
            }
            request.setAttribute("viewingOffset", offset);
            getServletContext().getRequestDispatcher("/viewFile.jsp").forward(request, response);
            return;
        }
        setPathDirectories(currentPath, request);
        filesInfo = mMaster.getFileSystemMaster().listStatus(currentPath, ListStatusOptions.defaults().setLoadMetadataType(LoadMetadataType.Always));
    } catch (FileDoesNotExistException e) {
        request.setAttribute("invalidPathError", "Error: Invalid Path " + e.getMessage());
        getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        return;
    } catch (InvalidPathException e) {
        request.setAttribute("invalidPathError", "Error: Invalid Path " + e.getLocalizedMessage());
        getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        return;
    } catch (IOException e) {
        request.setAttribute("invalidPathError", "Error: File " + currentPath + " is not available " + e.getMessage());
        getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        return;
    } catch (AccessControlException e) {
        request.setAttribute("invalidPathError", "Error: File " + currentPath + " cannot be accessed " + e.getMessage());
        getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        return;
    }
    List<UIFileInfo> fileInfos = new ArrayList<>(filesInfo.size());
    for (FileInfo fileInfo : filesInfo) {
        UIFileInfo toAdd = new UIFileInfo(fileInfo);
        try {
            if (!toAdd.getIsDirectory() && fileInfo.getLength() > 0) {
                FileBlockInfo blockInfo = mMaster.getFileSystemMaster().getFileBlockInfoList(new AlluxioURI(toAdd.getAbsolutePath())).get(0);
                List<String> locations = new ArrayList<>();
                // add the in-memory block locations
                for (BlockLocation location : blockInfo.getBlockInfo().getLocations()) {
                    WorkerNetAddress address = location.getWorkerAddress();
                    locations.add(address.getHost() + ":" + address.getDataPort());
                }
                // add underFS locations
                locations.addAll(blockInfo.getUfsLocations());
                toAdd.setFileLocations(locations);
            }
        } catch (FileDoesNotExistException e) {
            request.setAttribute("FileDoesNotExistException", "Error: non-existing file " + e.getMessage());
            getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
            return;
        } catch (InvalidPathException e) {
            request.setAttribute("InvalidPathException", "Error: invalid path " + e.getMessage());
            getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        } catch (AccessControlException e) {
            request.setAttribute("AccessControlException", "Error: File " + currentPath + " cannot be accessed " + e.getMessage());
            getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
            return;
        }
        fileInfos.add(toAdd);
    }
    Collections.sort(fileInfos, UIFileInfo.PATH_STRING_COMPARE);
    request.setAttribute("nTotalFile", fileInfos.size());
    // URL can not determine offset and limit, let javascript in jsp determine and redirect
    if (request.getParameter("offset") == null && request.getParameter("limit") == null) {
        getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        return;
    }
    try {
        int offset = Integer.parseInt(request.getParameter("offset"));
        int limit = Integer.parseInt(request.getParameter("limit"));
        List<UIFileInfo> sub = fileInfos.subList(offset, offset + limit);
        request.setAttribute("fileInfos", sub);
    } catch (NumberFormatException e) {
        request.setAttribute("fatalError", "Error: offset or limit parse error, " + e.getLocalizedMessage());
        getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        return;
    } catch (IndexOutOfBoundsException e) {
        request.setAttribute("fatalError", "Error: offset or offset + limit is out of bound, " + e.getLocalizedMessage());
        getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        return;
    } catch (IllegalArgumentException e) {
        request.setAttribute("fatalError", e.getLocalizedMessage());
        getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        return;
    }
    getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
}

Example 2 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class LocalFirstAvoidEvictionPolicyTest method getLocalFirst.

/**
   * Tests that the local host is returned first.
   */
@Test
public void getLocalFirst() {
    String localhostName = NetworkAddressUtils.getLocalHostName();
    LocalFirstAvoidEvictionPolicy policy = new LocalFirstAvoidEvictionPolicy();
    List<BlockWorkerInfo> workerInfoList = new ArrayList<>();
    workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
    workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost(localhostName).setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
    Assert.assertEquals(localhostName, policy.getWorkerForNextBlock(workerInfoList, Constants.MB).getHost());
}

Example 3 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class LocalFirstPolicyTest method getLocalFirst.

/**
   * Tests that the local host is returned first.
   */
@Test
public void getLocalFirst() {
    String localhostName = NetworkAddressUtils.getLocalHostName();
    LocalFirstPolicy policy = new LocalFirstPolicy();
    List<BlockWorkerInfo> workerInfoList = new ArrayList<>();
    workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
    workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost(localhostName).setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
    Assert.assertEquals(localhostName, policy.getWorkerForNextBlock(workerInfoList, Constants.MB).getHost());
}

Example 4 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class RemoteBlockInStreamIntegrationTest method readTest5.

/**
   * Tests {@link RemoteBlockInStream#read(byte[])}. Read from remote data server.
   */
@Test
public void readTest5() throws Exception {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
        AlluxioURI uri = new AlluxioURI(uniqPath + "/file_" + k);
        FileSystemTestUtils.createByteFile(mFileSystem, uri, mWriteAlluxio, k);
        long blockId = mFileSystem.getStatus(uri).getBlockIds().get(0);
        BlockInfo info = AlluxioBlockStore.create().getInfo(blockId);
        WorkerNetAddress workerAddr = info.getLocations().get(0).getWorkerAddress();
        RemoteBlockInStream is = RemoteBlockInStream.create(info.getBlockId(), info.getLength(), workerAddr, FileSystemContext.INSTANCE, InStreamOptions.defaults());
        byte[] ret = new byte[k];
        int start = 0;
        while (start < k) {
            int read = is.read(ret);
            Assert.assertTrue(BufferUtils.equalIncreasingByteArray(start, read, ret));
            start += read;
        }
        is.close();
        Assert.assertTrue(mFileSystem.getStatus(uri).getInMemoryPercentage() == 100);
    }
}

Example 5 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class RemoteBlockInStreamIntegrationTest method readTest4.

/**
   * Tests {@link RemoteBlockInStream#read()}. Read from remote data server.
   */
@Test
public void readTest4() throws Exception {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
        AlluxioURI uri = new AlluxioURI(uniqPath + "/file_" + k);
        FileSystemTestUtils.createByteFile(mFileSystem, uri, mWriteAlluxio, k);
        long blockId = mFileSystem.getStatus(uri).getBlockIds().get(0);
        AlluxioBlockStore blockStore = AlluxioBlockStore.create();
        BlockInfo info = blockStore.getInfo(blockId);
        WorkerNetAddress workerAddr = info.getLocations().get(0).getWorkerAddress();
        RemoteBlockInStream is = RemoteBlockInStream.create(info.getBlockId(), info.getLength(), workerAddr, FileSystemContext.INSTANCE, InStreamOptions.defaults());
        byte[] ret = new byte[k];
        int value = is.read();
        int cnt = 0;
        while (value != -1) {
            Assert.assertTrue(value >= 0);
            Assert.assertTrue(value < 256);
            ret[cnt++] = (byte) value;
            value = is.read();
        }
        Assert.assertEquals(cnt, k);
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
        is.close();
        Assert.assertTrue(mFileSystem.getStatus(uri).getInMemoryPercentage() == 100);
    }
}