Search in sources :

Example 16 with ByteBuffer

use of java.nio.ByteBuffer in project hadoop by apache.

the class SaslRpcClient method saslConnect.

/**
   * Do client side SASL authentication with server via the given InputStream
   * and OutputStream
   * 
   * @param inS
   *          InputStream to use
   * @param outS
   *          OutputStream to use
   * @return AuthMethod used to negotiate the connection
   * @throws IOException
   */
public AuthMethod saslConnect(IpcStreams ipcStreams) throws IOException {
    // redefined if/when a SASL negotiation starts, can be queried if the
    // negotiation fails
    authMethod = AuthMethod.SIMPLE;
    sendSaslMessage(ipcStreams.out, negotiateRequest);
    // loop until sasl is complete or a rpc error occurs
    boolean done = false;
    do {
        ByteBuffer bb = ipcStreams.readResponse();
        RpcWritable.Buffer saslPacket = RpcWritable.Buffer.wrap(bb);
        RpcResponseHeaderProto header = saslPacket.getValue(RpcResponseHeaderProto.getDefaultInstance());
        switch(header.getStatus()) {
            // might get a RPC error during 
            case ERROR:
            case FATAL:
                throw new RemoteException(header.getExceptionClassName(), header.getErrorMsg());
            default:
                break;
        }
        if (header.getCallId() != AuthProtocol.SASL.callId) {
            throw new SaslException("Non-SASL response during negotiation");
        }
        RpcSaslProto saslMessage = saslPacket.getValue(RpcSaslProto.getDefaultInstance());
        if (saslPacket.remaining() > 0) {
            throw new SaslException("Received malformed response length");
        }
        // handle sasl negotiation process
        RpcSaslProto.Builder response = null;
        switch(saslMessage.getState()) {
            case NEGOTIATE:
                {
                    // create a compatible SASL client, throws if no supported auths
                    SaslAuth saslAuthType = selectSaslClient(saslMessage.getAuthsList());
                    // define auth being attempted, caller can query if connect fails
                    authMethod = AuthMethod.valueOf(saslAuthType.getMethod());
                    byte[] responseToken = null;
                    if (authMethod == AuthMethod.SIMPLE) {
                        // switching to SIMPLE
                        // not going to wait for success ack
                        done = true;
                    } else {
                        byte[] challengeToken = null;
                        if (saslAuthType.hasChallenge()) {
                            // server provided the first challenge
                            challengeToken = saslAuthType.getChallenge().toByteArray();
                            saslAuthType = SaslAuth.newBuilder(saslAuthType).clearChallenge().build();
                        } else if (saslClient.hasInitialResponse()) {
                            challengeToken = new byte[0];
                        }
                        responseToken = (challengeToken != null) ? saslClient.evaluateChallenge(challengeToken) : new byte[0];
                    }
                    response = createSaslReply(SaslState.INITIATE, responseToken);
                    response.addAuths(saslAuthType);
                    break;
                }
            case CHALLENGE:
                {
                    if (saslClient == null) {
                        // demand a specific negotiation
                        throw new SaslException("Server sent unsolicited challenge");
                    }
                    byte[] responseToken = saslEvaluateToken(saslMessage, false);
                    response = createSaslReply(SaslState.RESPONSE, responseToken);
                    break;
                }
            case SUCCESS:
                {
                    // switch to simple
                    if (saslClient == null) {
                        authMethod = AuthMethod.SIMPLE;
                    } else {
                        saslEvaluateToken(saslMessage, true);
                    }
                    done = true;
                    break;
                }
            default:
                {
                    throw new SaslException("RPC client doesn't support SASL " + saslMessage.getState());
                }
        }
        if (response != null) {
            sendSaslMessage(ipcStreams.out, response.build());
        }
    } while (!done);
    return authMethod;
}
Also used : RpcResponseHeaderProto(org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto) SaslAuth(org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcSaslProto.SaslAuth) RpcWritable(org.apache.hadoop.ipc.RpcWritable) RemoteException(org.apache.hadoop.ipc.RemoteException) SaslException(javax.security.sasl.SaslException) ByteBuffer(java.nio.ByteBuffer) RpcSaslProto(org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcSaslProto)

Example 17 with ByteBuffer

use of java.nio.ByteBuffer in project flink by apache.

the class BufferTest method testgetNioBufferThreadSafe.

@Test
public void testgetNioBufferThreadSafe() {
    final MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(1024);
    final BufferRecycler recycler = Mockito.mock(BufferRecycler.class);
    Buffer buffer = new Buffer(segment, recycler);
    ByteBuffer buf1 = buffer.getNioBuffer();
    ByteBuffer buf2 = buffer.getNioBuffer();
    assertNotNull(buf1);
    assertNotNull(buf2);
    assertTrue("Repeated call to getNioBuffer() returns the same nio buffer", buf1 != buf2);
}
Also used : ByteBuffer(java.nio.ByteBuffer) ByteBuffer(java.nio.ByteBuffer) MemorySegment(org.apache.flink.core.memory.MemorySegment) Test(org.junit.Test)

Example 18 with ByteBuffer

use of java.nio.ByteBuffer in project flink by apache.

the class BufferFileWriterFileSegmentReaderTest method testWriteRead.

@Test
public void testWriteRead() throws IOException, InterruptedException {
    int numBuffers = 1024;
    int currentNumber = 0;
    final int minBufferSize = BUFFER_SIZE / 4;
    // Write buffers filled with ascending numbers...
    for (int i = 0; i < numBuffers; i++) {
        final Buffer buffer = createBuffer();
        int size = getNextMultipleOf(getRandomNumberInRange(minBufferSize, BUFFER_SIZE), 4);
        buffer.setSize(size);
        currentNumber = fillBufferWithAscendingNumbers(buffer, currentNumber);
        writer.writeBlock(buffer);
    }
    // Make sure that the writes are finished
    writer.close();
    // Read buffers back in...
    for (int i = 0; i < numBuffers; i++) {
        assertFalse(reader.hasReachedEndOfFile());
        reader.read();
    }
    // Wait for all requests to be finished
    final CountDownLatch sync = new CountDownLatch(1);
    final NotificationListener listener = new NotificationListener() {

        @Override
        public void onNotification() {
            sync.countDown();
        }
    };
    if (reader.registerAllRequestsProcessedListener(listener)) {
        sync.await();
    }
    assertTrue(reader.hasReachedEndOfFile());
    // Verify that the content is the same
    assertEquals("Read less buffers than written.", numBuffers, returnedFileSegments.size());
    currentNumber = 0;
    FileSegment fileSegment;
    ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
    while ((fileSegment = returnedFileSegments.poll()) != null) {
        buffer.position(0);
        buffer.limit(fileSegment.getLength());
        fileSegment.getFileChannel().read(buffer, fileSegment.getPosition());
        currentNumber = verifyBufferFilledWithAscendingNumbers(new Buffer(MemorySegmentFactory.wrap(buffer.array()), BUFFER_RECYCLER), currentNumber, fileSegment.getLength());
    }
    reader.close();
}
Also used : Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) ByteBuffer(java.nio.ByteBuffer) CountDownLatch(java.util.concurrent.CountDownLatch) ByteBuffer(java.nio.ByteBuffer) NotificationListener(org.apache.flink.runtime.util.event.NotificationListener) Test(org.junit.Test)

Example 19 with ByteBuffer

use of java.nio.ByteBuffer in project hadoop by apache.

the class ClientId method getClientId.

/**
   * Return clientId as byte[]
   */
public static byte[] getClientId() {
    UUID uuid = UUID.randomUUID();
    ByteBuffer buf = ByteBuffer.wrap(new byte[BYTE_LENGTH]);
    buf.putLong(uuid.getMostSignificantBits());
    buf.putLong(uuid.getLeastSignificantBits());
    return buf.array();
}
Also used : UUID(java.util.UUID) ByteBuffer(java.nio.ByteBuffer)

Example 20 with ByteBuffer

use of java.nio.ByteBuffer in project hadoop by apache.

the class SnappyDecompressor method decompressDirect.

int decompressDirect(ByteBuffer src, ByteBuffer dst) throws IOException {
    assert (this instanceof SnappyDirectDecompressor);
    ByteBuffer presliced = dst;
    if (dst.position() > 0) {
        presliced = dst;
        dst = dst.slice();
    }
    Buffer originalCompressed = compressedDirectBuf;
    Buffer originalUncompressed = uncompressedDirectBuf;
    int originalBufferSize = directBufferSize;
    compressedDirectBuf = src.slice();
    compressedDirectBufLen = src.remaining();
    uncompressedDirectBuf = dst;
    directBufferSize = dst.remaining();
    int n = 0;
    try {
        n = decompressBytesDirect();
        presliced.position(presliced.position() + n);
        // SNAPPY always consumes the whole buffer or throws an exception
        src.position(src.limit());
        finished = true;
    } finally {
        compressedDirectBuf = originalCompressed;
        uncompressedDirectBuf = originalUncompressed;
        compressedDirectBufLen = 0;
        directBufferSize = originalBufferSize;
    }
    return n;
}
Also used : Buffer(java.nio.Buffer) ByteBuffer(java.nio.ByteBuffer) ByteBuffer(java.nio.ByteBuffer)

Aggregations

ByteBuffer (java.nio.ByteBuffer)8537 Test (org.junit.Test)1905 IOException (java.io.IOException)972 ArrayList (java.util.ArrayList)430 File (java.io.File)216 FileChannel (java.nio.channels.FileChannel)204 MappedByteBuffer (java.nio.MappedByteBuffer)190 HashMap (java.util.HashMap)172 CharBuffer (java.nio.CharBuffer)144 ByteArrayOutputStream (java.io.ByteArrayOutputStream)138 InetSocketAddress (java.net.InetSocketAddress)137 Random (java.util.Random)119 InputStream (java.io.InputStream)115 FileInputStream (java.io.FileInputStream)114 Map (java.util.Map)113 WebSocketFrame (org.eclipse.jetty.websocket.common.WebSocketFrame)99 Test (org.testng.annotations.Test)96 IntBuffer (java.nio.IntBuffer)93 SocketChannel (java.nio.channels.SocketChannel)92 FileOutputStream (java.io.FileOutputStream)87