Examples with InStreamOptions alluxio.client.file.options.InStreamOptions used on opensource projects

Example 16 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class JobUtils method loadThroughRead.

private static void loadThroughRead(URIStatus status, FileSystemContext context, long blockId, AlluxioConfiguration conf) throws IOException {
    // This does not work for remote worker unless we have passive cache on.
    AlluxioProperties prop = context.getClusterConf().copyProperties();
    prop.set(PropertyKey.USER_FILE_PASSIVE_CACHE_ENABLED, true);
    AlluxioConfiguration config = new InstancedConfiguration(prop);
    FileSystemContext loadContext = FileSystemContext.create(config);
    AlluxioBlockStore blockStore = AlluxioBlockStore.create(loadContext);
    OpenFilePOptions openOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE).build();
    InStreamOptions inOptions = new InStreamOptions(status, openOptions, conf);
    inOptions.setUfsReadLocationPolicy(BlockLocationPolicy.Factory.create(LocalFirstPolicy.class.getCanonicalName(), conf));
    BlockInfo info = Preconditions.checkNotNull(status.getBlockInfo(blockId));
    try (InputStream inputStream = blockStore.getInStream(info, inOptions, ImmutableMap.of())) {
        while (inputStream.read(sIgnoredReadBuf) != -1) {
        }
    }
}

Example 17 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class JobUtils method loadBlock.

/**
 * Loads a block into the local worker. If the block doesn't exist in Alluxio, it will be read
 * from the UFS.
 * @param status the uriStatus
 * @param context filesystem context
 * @param blockId the id of the block to load
 * @param address specify a worker to load into
 * @param directCache Use passive-cache or direct cache request
 */
public static void loadBlock(URIStatus status, FileSystemContext context, long blockId, WorkerNetAddress address, boolean directCache) throws AlluxioException, IOException {
    AlluxioConfiguration conf = ServerConfiguration.global();
    // Explicitly specified a worker to load
    WorkerNetAddress localNetAddress = address;
    String localHostName = NetworkAddressUtils.getConnectHost(ServiceType.WORKER_RPC, conf);
    List<WorkerNetAddress> netAddress = context.getCachedWorkers().stream().map(BlockWorkerInfo::getNetAddress).filter(x -> Objects.equals(x.getHost(), localHostName)).collect(Collectors.toList());
    if (localNetAddress == null && !netAddress.isEmpty()) {
        localNetAddress = netAddress.get(0);
    }
    if (localNetAddress == null) {
        throw new NotFoundException(ExceptionMessage.NO_LOCAL_BLOCK_WORKER_LOAD_TASK.getMessage(blockId));
    }
    Set<String> pinnedLocation = status.getPinnedMediumTypes();
    if (pinnedLocation.size() > 1) {
        throw new AlluxioException(ExceptionMessage.PINNED_TO_MULTIPLE_MEDIUMTYPES.getMessage(status.getPath()));
    }
    // Only use this read local first method to load if nearest worker is clear
    if (netAddress.size() <= 1 && pinnedLocation.isEmpty() && status.isPersisted()) {
        if (directCache) {
            loadThroughCacheRequest(status, context, blockId, conf, localNetAddress);
        } else {
            loadThroughRead(status, context, blockId, conf);
        }
        return;
    }
    // TODO(bin): remove the following case when we consolidate tier and medium
    // since there is only one element in the set, we take the first element in the set
    String medium = pinnedLocation.isEmpty() ? "" : pinnedLocation.iterator().next();
    OpenFilePOptions openOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build();
    InStreamOptions inOptions = new InStreamOptions(status, openOptions, conf);
    // Set read location policy always to local first for loading blocks for job tasks
    inOptions.setUfsReadLocationPolicy(BlockLocationPolicy.Factory.create(LocalFirstPolicy.class.getCanonicalName(), conf));
    OutStreamOptions outOptions = OutStreamOptions.defaults(context.getClientContext());
    outOptions.setMediumType(medium);
    // Set write location policy always to local first for loading blocks for job tasks
    outOptions.setLocationPolicy(BlockLocationPolicy.Factory.create(LocalFirstPolicy.class.getCanonicalName(), conf));
    BlockInfo blockInfo = status.getBlockInfo(blockId);
    Preconditions.checkNotNull(blockInfo, "Can not find block %s in status %s", blockId, status);
    long blockSize = blockInfo.getLength();
    AlluxioBlockStore blockStore = AlluxioBlockStore.create(context);
    try (OutputStream outputStream = blockStore.getOutStream(blockId, blockSize, localNetAddress, outOptions)) {
        try (InputStream inputStream = blockStore.getInStream(blockId, inOptions)) {
            ByteStreams.copy(inputStream, outputStream);
        } catch (Throwable t) {
            try {
                ((Cancelable) outputStream).cancel();
            } catch (Throwable t2) {
                t.addSuppressed(t2);
            }
            throw t;
        }
    }
}

Example 18 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class BaseFileSystem method openFile.

@Override
public FileInStream openFile(AlluxioURI path, OpenFileOptions options) throws FileDoesNotExistException, IOException, AlluxioException {
    URIStatus status = getStatus(path);
    if (status.isFolder()) {
        throw new FileNotFoundException(ExceptionMessage.CANNOT_READ_DIRECTORY.getMessage(status.getName()));
    }
    InStreamOptions inStreamOptions = options.toInStreamOptions();
    return FileInStream.create(status, inStreamOptions, mFileSystemContext);
}

Example 19 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class BlockWorkerDataReaderTest method createAndCloseManyReader.

// See https://github.com/Alluxio/alluxio/issues/13255
@Test
public void createAndCloseManyReader() throws Exception {
    for (int i = 0; i < LOCK_NUM * 10; i++) {
        long blockId = i;
        mBlockWorker.createBlock(SESSION_ID, blockId, 0, Constants.MEDIUM_MEM, 1);
        mBlockWorker.commitBlock(SESSION_ID, blockId, true);
        InStreamOptions inStreamOptions = new InStreamOptions(new URIStatus(new FileInfo().setBlockIds(Collections.singletonList(blockId))), FileSystemOptions.openFileDefaults(mConf), mConf);
        mDataReaderFactory = new BlockWorkerDataReader.Factory(mBlockWorker, blockId, CHUNK_SIZE, inStreamOptions);
        DataReader dataReader = mDataReaderFactory.create(0, 100);
        dataReader.close();
    }
}

Example 20 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class RemoteReadIntegrationTest method readTest4.

/**
 * Tests the single byte read API from a remote location when the data is in an Alluxio worker.
 */
@Test
public void readTest4() throws Exception {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
        AlluxioURI uri = new AlluxioURI(uniqPath + "/file_" + k);
        FileSystemTestUtils.createByteFile(mFileSystem, uri, mWriteAlluxio, k);
        URIStatus status = mFileSystem.getStatus(uri);
        InStreamOptions options = new InStreamOptions(status, ServerConfiguration.global());
        long blockId = status.getBlockIds().get(0);
        AlluxioBlockStore blockStore = AlluxioBlockStore.create(FileSystemContext.create(ServerConfiguration.global()));
        BlockInfo info = blockStore.getInfo(blockId);
        WorkerNetAddress workerAddr = info.getLocations().get(0).getWorkerAddress();
        BlockInStream is = BlockInStream.create(mFsContext, options.getBlockInfo(blockId), workerAddr, BlockInStreamSource.REMOTE, options);
        byte[] ret = new byte[k];
        int value = is.read();
        int cnt = 0;
        while (value != -1) {
            Assert.assertTrue(value >= 0);
            Assert.assertTrue(value < 256);
            ret[cnt++] = (byte) value;
            value = is.read();
        }
        Assert.assertEquals(cnt, k);
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
        is.close();
        FileSystemUtils.waitForAlluxioPercentage(mFileSystem, uri, 100);
    }
}