Examples with WorkerNetAddress alluxio.wire.WorkerNetAddress used on opensource projects

Example 6 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class RoundRobinPolicy method getWorker.

@Override
public WorkerNetAddress getWorker(GetWorkerOptions options) {
    WorkerNetAddress address = mBlockLocationCache.get(options.getBlockId());
    if (address != null) {
        return address;
    }
    address = getWorkerForNextBlock(options.getBlockWorkerInfos(), options.getBlockSize());
    mBlockLocationCache.put(options.getBlockId(), address);
    return address;
}

Example 7 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class AlluxioBlockStore method getInStream.

/**
   * Gets a stream to read the data of a block. The stream is backed by Alluxio storage.
   *
   * @param blockId the block to read from
   * @param options the options
   * @return an {@link InputStream} which can be used to read the data in a streaming fashion
   * @throws IOException if the block does not exist
   */
public InputStream getInStream(long blockId, InStreamOptions options) throws IOException {
    BlockInfo blockInfo;
    try (CloseableResource<BlockMasterClient> masterClientResource = mContext.acquireBlockMasterClientResource()) {
        blockInfo = masterClientResource.get().getBlockInfo(blockId);
    } catch (AlluxioException e) {
        throw new IOException(e);
    }
    if (blockInfo.getLocations().isEmpty()) {
        throw new IOException("Block " + blockId + " is not available in Alluxio");
    }
    // for hasLocalWorker is fixed.
    for (BlockLocation location : blockInfo.getLocations()) {
        WorkerNetAddress workerNetAddress = location.getWorkerAddress();
        if (workerNetAddress.getHost().equals(mLocalHostName)) {
            // There is a local worker and the block is local.
            try {
                return StreamFactory.createLocalBlockInStream(mContext, blockId, blockInfo.getLength(), workerNetAddress, options);
            } catch (IOException e) {
                LOG.warn("Failed to open local stream for block {}: {}", blockId, e.getMessage());
                // Getting a local stream failed, do not try again
                break;
            }
        }
    }
    // No local worker/block, choose a random location. In the future we could change this to
    // only randomize among locations in the highest tier, or have the master randomize the order.
    List<BlockLocation> locations = blockInfo.getLocations();
    WorkerNetAddress workerNetAddress = locations.get(mRandom.nextInt(locations.size())).getWorkerAddress();
    return StreamFactory.createRemoteBlockInStream(mContext, blockId, blockInfo.getLength(), workerNetAddress, options);
}

Example 8 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getOutStreamUsingLocationPolicy.

@Test
public void getOutStreamUsingLocationPolicy() throws Exception {
    OutStreamOptions options = OutStreamOptions.defaults().setWriteType(WriteType.MUST_CACHE).setLocationPolicy(new FileWriteLocationPolicy() {

        @Override
        public WorkerNetAddress getWorkerForNextBlock(Iterable<BlockWorkerInfo> workerInfoList, long blockSizeBytes) {
            throw new RuntimeException("policy threw exception");
        }
    });
    try {
        mBlockStore.getOutStream(BLOCK_ID, BLOCK_LENGTH, options);
        Assert.fail("An exception should have been thrown");
    } catch (Exception e) {
        Assert.assertEquals("policy threw exception", e.getMessage());
    }
}

Example 9 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class AlluxioBlockStore method getOutStream.

/**
   * Gets a stream to write data to a block based on the options. The stream can only be backed by
   * Alluxio storage.
   *
   * @param blockId the block to write
   * @param blockSize the standard block size to write, or -1 if the block already exists (and this
   *        stream is just storing the block in Alluxio again)
   * @param options the output stream option
   * @return an {@link OutputStream} which can be used to write data to the block in a
   *         streaming fashion
   * @throws IOException if the block cannot be written
   */
public OutputStream getOutStream(long blockId, long blockSize, OutStreamOptions options) throws IOException {
    WorkerNetAddress address;
    FileWriteLocationPolicy locationPolicy = Preconditions.checkNotNull(options.getLocationPolicy(), PreconditionMessage.FILE_WRITE_LOCATION_POLICY_UNSPECIFIED);
    try {
        address = locationPolicy.getWorkerForNextBlock(getWorkerInfoList(), blockSize);
    } catch (AlluxioException e) {
        throw new IOException(e);
    }
    return getOutStream(blockId, blockSize, address, options);
}

Example 10 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class FileOutStreamTest method before.

/**
   * Sets up the different contexts and clients before a test runs.
   */
@Before
public void before() throws Exception {
    GroupMappingServiceTestUtils.resetCache();
    ClientTestUtils.setSmallBufferSizes();
    // PowerMock enums and final classes
    mFileSystemContext = PowerMockito.mock(FileSystemContext.class);
    mBlockStore = PowerMockito.mock(AlluxioBlockStore.class);
    mFileSystemMasterClient = PowerMockito.mock(FileSystemMasterClient.class);
    mFactory = PowerMockito.mock(UnderFileSystemFileOutStream.Factory.class);
    PowerMockito.mockStatic(AlluxioBlockStore.class);
    PowerMockito.when(AlluxioBlockStore.create(mFileSystemContext)).thenReturn(mBlockStore);
    when(mFileSystemContext.acquireMasterClientResource()).thenReturn(new DummyCloseableResource<>(mFileSystemMasterClient));
    when(mFileSystemMasterClient.getStatus(any(AlluxioURI.class))).thenReturn(new URIStatus(new FileInfo()));
    // Worker file client mocking
    mWorkerClient = PowerMockito.mock(FileSystemWorkerClient.class);
    when(mFileSystemContext.createFileSystemWorkerClient()).thenReturn(mWorkerClient);
    when(mWorkerClient.createUfsFile(any(AlluxioURI.class), any(CreateUfsFileOptions.class))).thenReturn(UFS_FILE_ID);
    // Return sequentially increasing numbers for new block ids
    when(mFileSystemMasterClient.getNewBlockIdForFile(FILE_NAME)).thenAnswer(new Answer<Long>() {

        private long mCount = 0;

        @Override
        public Long answer(InvocationOnMock invocation) throws Throwable {
            return mCount++;
        }
    });
    // Set up out streams. When they are created, add them to outStreamMap
    final Map<Long, TestBufferedBlockOutStream> outStreamMap = new HashMap<>();
    when(mBlockStore.getOutStream(anyLong(), eq(BLOCK_LENGTH), any(OutStreamOptions.class))).thenAnswer(new Answer<BufferedBlockOutStream>() {

        @Override
        public BufferedBlockOutStream answer(InvocationOnMock invocation) throws Throwable {
            Long blockId = invocation.getArgumentAt(0, Long.class);
            if (!outStreamMap.containsKey(blockId)) {
                TestBufferedBlockOutStream newStream = new TestBufferedBlockOutStream(blockId, BLOCK_LENGTH, mFileSystemContext);
                outStreamMap.put(blockId, newStream);
            }
            return outStreamMap.get(blockId);
        }
    });
    BlockWorkerInfo workerInfo = new BlockWorkerInfo(new WorkerNetAddress().setHost("localhost").setRpcPort(1).setDataPort(2).setWebPort(3), Constants.GB, 0);
    when(mBlockStore.getWorkerInfoList()).thenReturn(Lists.newArrayList(workerInfo));
    mAlluxioOutStreamMap = outStreamMap;
    // Create an under storage stream so that we can check whether it has been flushed
    final AtomicBoolean underStorageFlushed = new AtomicBoolean(false);
    mUnderStorageOutputStream = new ByteArrayOutputStream() {

        @Override
        public void flush() {
            underStorageFlushed.set(true);
        }
    };
    mUnderStorageFlushed = underStorageFlushed;
    when(mFactory.create(any(FileSystemContext.class), any(InetSocketAddress.class), anyLong())).thenReturn(mUnderStorageOutputStream);
    // Set up underFileStorage so that we can test UnderStorageType.SYNC_PERSIST
    mUnderFileSystem = ClientMockUtils.mockUnderFileSystem();
    when(mUnderFileSystem.create(anyString())).thenReturn(mUnderStorageOutputStream);
    when(mUnderFileSystem.create(anyString(), any(CreateOptions.class))).thenReturn(mUnderStorageOutputStream);
    when(mUnderFileSystem.isDirectory(anyString())).thenReturn(true);
    OutStreamOptions options = OutStreamOptions.defaults().setBlockSizeBytes(BLOCK_LENGTH).setWriteType(WriteType.CACHE_THROUGH).setUfsPath(FILE_NAME.getPath());
    mTestStream = createTestStream(FILE_NAME, options);
}