Examples with AlluxioConfiguration alluxio.conf.AlluxioConfiguration used on opensource projects

Example 6 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class BaseFileSystem method openFile.

@Override
public FileInStream openFile(URIStatus status, OpenFilePOptions options) throws FileDoesNotExistException, OpenDirectoryException, FileIncompleteException, IOException, AlluxioException {
    AlluxioURI path = new AlluxioURI(status.getPath());
    if (status.isFolder()) {
        throw new OpenDirectoryException(path);
    }
    if (!status.isCompleted()) {
        throw new FileIncompleteException(path);
    }
    AlluxioConfiguration conf = mFsContext.getPathConf(path);
    OpenFilePOptions mergedOptions = FileSystemOptions.openFileDefaults(conf).toBuilder().mergeFrom(options).build();
    InStreamOptions inStreamOptions = new InStreamOptions(status, mergedOptions, conf);
    return new AlluxioFileInStream(status, inStreamOptions, mFsContext);
}

Example 7 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class BlockLocationUtils method nearest.

/**
 * @param tieredIdentity the tiered identity
 * @param addresses the candidate worker addresses
 * @param conf Alluxio configuration
 * @return the first in the pair indicates the address closest to this one. If none of the
 *         identities match, the first address is returned. the second in the pair indicates
 *         whether or not the location is local
 */
public static Optional<Pair<WorkerNetAddress, Boolean>> nearest(TieredIdentity tieredIdentity, List<WorkerNetAddress> addresses, AlluxioConfiguration conf) {
    if (conf.getBoolean(PropertyKey.WORKER_DATA_SERVER_DOMAIN_SOCKET_AS_UUID)) {
        // Determine by inspecting the file system if worker is local
        for (WorkerNetAddress addr : addresses) {
            if (NettyUtils.isDomainSocketAccessible(addr, conf)) {
                LOG.debug("Found local worker by file system inspection of path {}", addr.getDomainSocketPath());
                // Returns the first local worker and does not shuffle
                return Optional.of(new Pair<>(addr, true));
            }
        }
    }
    // Find nearest tiered identity
    Optional<TieredIdentity> nearestIdentity = TieredIdentityUtils.nearest(tieredIdentity, addresses.stream().map(addr -> addr.getTieredIdentity()).collect(Collectors.toList()), conf);
    if (!nearestIdentity.isPresent()) {
        return Optional.empty();
    }
    boolean isLocal = tieredIdentity.getTier(0).getTierName().equals(Constants.LOCALITY_NODE) && tieredIdentity.topTiersMatch(nearestIdentity.get());
    Optional<WorkerNetAddress> dataSource = addresses.stream().filter(addr -> addr.getTieredIdentity().equals(nearestIdentity.get())).findFirst();
    if (!dataSource.isPresent()) {
        return Optional.empty();
    }
    return Optional.of(new Pair<>(dataSource.get(), isLocal));
}

Example 8 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class BufferCachingGrpcDataReader method create.

/**
 * Creates an instance of {@link BufferCachingGrpcDataReader} for block reads.
 *
 * @param context the file system context
 * @param address the worker address
 * @param readRequest the read request
 * @return a new {@link BufferCachingGrpcDataReader}
 */
public static BufferCachingGrpcDataReader create(FileSystemContext context, WorkerNetAddress address, ReadRequest readRequest) throws IOException {
    AlluxioConfiguration alluxioConf = context.getClusterConf();
    int readerBufferSizeMessages = alluxioConf.getInt(PropertyKey.USER_STREAMING_READER_BUFFER_SIZE_MESSAGES);
    long dataTimeoutMs = alluxioConf.getMs(PropertyKey.USER_STREAMING_DATA_READ_TIMEOUT);
    CloseableResource<BlockWorkerClient> client = context.acquireBlockWorkerClient(address);
    String desc = "BufferCachingGrpcDataReader";
    if (LOG.isDebugEnabled()) {
        // More detailed description when debug logging is enabled
        desc = String.format("BufferCachingGrpcDataReader(request=%s,address=%s)", readRequest, address);
    }
    GrpcBlockingStream<ReadRequest, ReadResponse> stream = null;
    try {
        // Stream here cannot be GrpcDataMessagingBlockingStream
        // DataBuffer.getReadOnlyByteBuffer is used to clone a copy in SharedDataReader.readChunk.
        // getReadOnlyByteBuffer is not implemented in DataBuffer
        // returned from GrpcDataMessagingBlockingStream.
        stream = new GrpcBlockingStream<>(client.get()::readBlock, readerBufferSizeMessages, desc);
        stream.send(readRequest, dataTimeoutMs);
    } catch (Exception e) {
        if (stream != null) {
            stream.close();
        }
        client.close();
        throw e;
    }
    return new BufferCachingGrpcDataReader(address, client, dataTimeoutMs, readRequest, stream);
}

Example 9 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class BlockInStream method createRemoteBlockInStream.

/**
 * Creates a {@link BlockInStream} to read from a specific remote server. Should only be used
 * in cases where the data source and method of reading is known, ie. worker - worker
 * communication.
 *
 * @param context the file system context
 * @param blockId the block id
 * @param address the address of the gRPC data server
 * @param blockSource the source location of the block
 * @param blockSize the size of the block
 * @param ufsOptions the ufs read options
 * @return the {@link BlockInStream} created
 */
public static BlockInStream createRemoteBlockInStream(FileSystemContext context, long blockId, WorkerNetAddress address, BlockInStreamSource blockSource, long blockSize, Protocol.OpenUfsBlockOptions ufsOptions) {
    AlluxioConfiguration conf = context.getClusterConf();
    long chunkSize = conf.getBytes(PropertyKey.USER_STREAMING_READER_CHUNK_SIZE_BYTES);
    ReadRequest readRequest = ReadRequest.newBuilder().setBlockId(blockId).setOpenUfsBlockOptions(ufsOptions).setChunkSize(chunkSize).buildPartial();
    DataReader.Factory factory = new GrpcDataReader.Factory(context, address, readRequest.toBuilder());
    return new BlockInStream(factory, conf, address, blockSource, blockId, blockSize);
}

Example 10 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class BlockInStream method createLocalBlockInStream.

/**
 * Creates a {@link BlockInStream} to read from a local file.
 *
 * @param context the file system context
 * @param address the network address of the gRPC data server to read from
 * @param blockId the block ID
 * @param length the block length
 * @param options the in stream options
 * @return the {@link BlockInStream} created
 */
private static BlockInStream createLocalBlockInStream(FileSystemContext context, WorkerNetAddress address, long blockId, long length, InStreamOptions options) throws IOException {
    AlluxioConfiguration conf = context.getClusterConf();
    long chunkSize = conf.getBytes(PropertyKey.USER_LOCAL_READER_CHUNK_SIZE_BYTES);
    return new BlockInStream(new LocalFileDataReader.Factory(context, address, blockId, chunkSize, options), conf, address, BlockInStreamSource.NODE_LOCAL, blockId, length);
}