Examples with FileSystem alluxio.client.file.FileSystem used on opensource projects

Example 61 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class BlockMasterIntegrityIntegrationTest method deleteInvalidBlocks.

@Test
@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.MASTER_STARTUP_BLOCK_INTEGRITY_CHECK_ENABLED, "true" })
public void deleteInvalidBlocks() throws Exception {
    AlluxioURI uri = new AlluxioURI("/test");
    int len = 10;
    FileSystem fs = mCluster.getClient();
    BlockWorker worker = mCluster.getWorkerProcess().getWorker(BlockWorker.class);
    FileSystemTestUtils.createByteFile(fs, uri, WritePType.MUST_CACHE, len);
    Assert.assertEquals(1, worker.getStoreMetaFull().getNumberOfBlocks());
    removeFileMetadata(uri);
    mCluster.stopWorkers();
    mCluster.restartMasters();
    // creates a new worker, so need to get the new BlockWorker
    mCluster.startWorkers();
    BlockWorker newWorker = mCluster.getWorkerProcess().getWorker(BlockWorker.class);
    CommonUtils.waitFor("invalid blocks to be deleted", () -> newWorker.getStoreMetaFull().getNumberOfBlocks() == 0, WaitForOptions.defaults().setTimeoutMs(2000));
}

Example 62 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class WebInterfaceDownloadServlet method downloadFile.

/**
   * This function prepares for downloading a file.
   *
   * @param path the path of the file to download
   * @param request the {@link HttpServletRequest} object
   * @param response the {@link HttpServletResponse} object
   * @throws FileDoesNotExistException if the file does not exist
   * @throws IOException if an I/O error occurs
   * @throws InvalidPathException if an invalid path is encountered
   * @throws AlluxioException if an unexpected Alluxio exception is thrown
   */
private void downloadFile(AlluxioURI path, HttpServletRequest request, HttpServletResponse response) throws FileDoesNotExistException, IOException, InvalidPathException, AlluxioException {
    FileSystem alluxioClient = FileSystem.Factory.get();
    URIStatus status = alluxioClient.getStatus(path);
    long len = status.getLength();
    String fileName = path.getName();
    response.setContentType("application/octet-stream");
    if (len <= Integer.MAX_VALUE) {
        response.setContentLength((int) len);
    } else {
        response.addHeader("Content-Length", Long.toString(len));
    }
    response.addHeader("Content-Disposition", "attachment;filename=" + fileName);
    FileInStream is = null;
    ServletOutputStream out = null;
    try {
        OpenFileOptions options = OpenFileOptions.defaults().setReadType(ReadType.NO_CACHE);
        is = alluxioClient.openFile(path, options);
        out = response.getOutputStream();
        ByteStreams.copy(is, out);
    } finally {
        if (out != null) {
            out.flush();
            out.close();
        }
        if (is != null) {
            is.close();
        }
    }
}

Example 63 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class WebInterfaceBrowseServlet method displayFile.

/**
   * This function displays 5KB of a file from a specific offset if it is in ASCII format.
   *
   * @param path the path of the file to display
   * @param request the {@link HttpServletRequest} object
   * @param offset where the file starts to display
   * @throws FileDoesNotExistException if the file does not exist
   * @throws IOException if an I/O error occurs
   * @throws InvalidPathException if an invalid path is encountered
   * @throws AlluxioException if an unexpected Alluxio exception is thrown
   */
private void displayFile(AlluxioURI path, HttpServletRequest request, long offset) throws FileDoesNotExistException, InvalidPathException, IOException, AlluxioException {
    FileSystem fs = FileSystem.Factory.get();
    String fileData;
    URIStatus status = fs.getStatus(path);
    if (status.isCompleted()) {
        OpenFileOptions options = OpenFileOptions.defaults().setReadType(ReadType.NO_CACHE);
        try (FileInStream is = fs.openFile(path, options)) {
            int len = (int) Math.min(5 * Constants.KB, status.getLength() - offset);
            byte[] data = new byte[len];
            long skipped = is.skip(offset);
            if (skipped < 0) {
                // nothing was skipped
                fileData = "Unable to traverse to offset; is file empty?";
            } else if (skipped < offset) {
                // couldn't skip all the way to offset
                fileData = "Unable to traverse to offset; is offset larger than the file?";
            } else {
                // read may not read up to len, so only convert what was read
                int read = is.read(data, 0, len);
                if (read < 0) {
                    // stream couldn't read anything, skip went to EOF?
                    fileData = "Unable to read file";
                } else {
                    fileData = WebUtils.convertByteArrayToStringWithoutEscape(data, 0, read);
                }
            }
        }
    } else {
        fileData = "The requested file is not complete yet.";
    }
    List<UIFileBlockInfo> uiBlockInfo = new ArrayList<>();
    for (FileBlockInfo fileBlockInfo : mMaster.getFileSystemMaster().getFileBlockInfoList(path)) {
        uiBlockInfo.add(new UIFileBlockInfo(fileBlockInfo));
    }
    request.setAttribute("fileBlocks", uiBlockInfo);
    request.setAttribute("fileData", fileData);
    request.setAttribute("highestTierAlias", mMaster.getBlockMaster().getGlobalStorageTierAssoc().getAlias(0));
}

Example 64 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class MultiWorkerIntegrationTest method writeLargeFile.

@Test
public void writeLargeFile() throws Exception {
    int fileSize = NUM_WORKERS * WORKER_MEMORY_SIZE_BYTES;
    AlluxioURI file = new AlluxioURI("/test");
    FileSystem fs = mResource.get().getClient();
    // Write a file large enough to fill all the memory of all the workers.
    FileSystemTestUtils.createByteFile(fs, file.getPath(), fileSize, CreateFileOptions.defaults().setWriteType(WriteType.MUST_CACHE).setLocationPolicy(new RoundRobinPolicy()));
    URIStatus status = fs.getStatus(file);
    assertEquals(100, status.getInMemoryPercentage());
    try (FileInStream inStream = fs.openFile(file)) {
        assertEquals(fileSize, IOUtils.toByteArray(inStream).length);
    }
}

Example 65 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class KeyValueSystemIntegrationTest method createMultiPartitions.

/**
   * Tests creating and opening a store with a number of keys, while each key-value pair is large
   * enough to take a separate key-value partition.
   */
@Test
public void createMultiPartitions() throws Exception {
    // Each partition is at most 1 MB
    final long maxPartitionSize = Constants.MB;
    final int numKeys = 10;
    // 4Byte key
    final int keyLength = 4;
    // 500KB value
    final int valueLength = 500 * Constants.KB;
    FileSystem fs = FileSystem.Factory.get();
    AlluxioURI storeUri = createStoreOfMultiplePartitions(numKeys, null);
    List<URIStatus> files = fs.listStatus(storeUri);
    Assert.assertEquals(numKeys, files.size());
    for (URIStatus info : files) {
        Assert.assertTrue(info.getLength() <= maxPartitionSize);
    }
    mReader = sKeyValueSystem.openStore(storeUri);
    for (int i = 0; i < numKeys; i++) {
        byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
        byte[] value = mReader.get(key);
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(i, valueLength, value));
    }
    Assert.assertNull(mReader.get(KEY1));
    Assert.assertNull(mReader.get(KEY2));
    mReader.close();
}