Examples with Metadata org.apache.beam.vendor.grpc.v1p43p2.io.grpc.Metadata used on opensource projects

Example 66 with Metadata

use of org.apache.beam.vendor.grpc.v1p43p2.io.grpc.Metadata in project beam by apache.

the class Utils method getKafkaCredentialsFromVault.

/**
 * Retrieves all credentials from HashiCorp Vault secret storage.
 *
 * @param secretStoreUrl url to the secret storage that contains a credentials for Kafka
 * @param token Vault token to access the secret storage
 * @return credentials for Kafka consumer config
 */
public static Map<String, Map<String, String>> getKafkaCredentialsFromVault(String secretStoreUrl, String token) {
    Map<String, Map<String, String>> credentialMap = new HashMap<>();
    JsonObject credentials = null;
    try {
        HttpClient client = HttpClientBuilder.create().build();
        HttpGet request = new HttpGet(secretStoreUrl);
        request.addHeader("X-Vault-Token", token);
        HttpResponse response = client.execute(request);
        String json = EntityUtils.toString(response.getEntity(), "UTF-8");
        /*
       Vault's response JSON has a specific schema, where the actual data is placed under
       {data: {data: <actual data>}}.
       Example:
         {
           "request_id": "6a0bb14b-ef24-256c-3edf-cfd52ad1d60d",
           "lease_id": "",
           "renewable": false,
           "lease_duration": 0,
           "data": {
             "data": {
               "bucket": "kafka_to_pubsub_test",
               "key_password": "secret",
               "keystore_password": "secret",
               "keystore_path": "ssl_cert/kafka.keystore.jks",
               "password": "admin-secret",
               "truststore_password": "secret",
               "truststore_path": "ssl_cert/kafka.truststore.jks",
               "username": "admin"
             },
             "metadata": {
               "created_time": "2020-10-20T11:43:11.109186969Z",
               "deletion_time": "",
               "destroyed": false,
               "version": 8
             }
           },
           "wrap_info": null,
           "warnings": null,
           "auth": null
         }
      */
        // Parse security properties from the response JSON
        credentials = JsonParser.parseString(json).getAsJsonObject().get("data").getAsJsonObject().getAsJsonObject("data");
    } catch (IOException e) {
        LOG.error("Failed to retrieve credentials from Vault.", e);
    }
    if (credentials != null) {
        // Username and password for Kafka authorization
        credentialMap.put(KAFKA_CREDENTIALS, new HashMap<>());
        if (credentials.has(USERNAME) && credentials.has(PASSWORD)) {
            credentialMap.get(KAFKA_CREDENTIALS).put(USERNAME, credentials.get(USERNAME).getAsString());
            credentialMap.get(KAFKA_CREDENTIALS).put(PASSWORD, credentials.get(PASSWORD).getAsString());
        } else {
            LOG.warn("There are no username and/or password for Kafka in Vault." + "Trying to initiate an unauthorized connection.");
        }
    }
    return credentialMap;
}

Example 67 with Metadata

use of org.apache.beam.vendor.grpc.v1p43p2.io.grpc.Metadata in project grpc-java by grpc.

the class DelayedClientCallTest method listenerEventsPropagated.

@Test
public void listenerEventsPropagated() {
    DelayedClientCall<String, Integer> delayedClientCall = new DelayedClientCall<>(callExecutor, fakeClock.getScheduledExecutorService(), Deadline.after(10, SECONDS));
    delayedClientCall.start(listener, new Metadata());
    delayedClientCall.setCall(mockRealCall);
    ArgumentCaptor<Listener<Integer>> listenerCaptor = ArgumentCaptor.forClass(null);
    verify(mockRealCall).start(listenerCaptor.capture(), any(Metadata.class));
    Listener<Integer> realCallListener = listenerCaptor.getValue();
    Metadata metadata = new Metadata();
    metadata.put(Metadata.Key.of("key", Metadata.ASCII_STRING_MARSHALLER), "value");
    realCallListener.onHeaders(metadata);
    verify(listener).onHeaders(metadata);
    realCallListener.onMessage(3);
    verify(listener).onMessage(3);
    realCallListener.onReady();
    verify(listener).onReady();
    Metadata trailer = new Metadata();
    trailer.put(Metadata.Key.of("key2", Metadata.ASCII_STRING_MARSHALLER), "value2");
    realCallListener.onClose(Status.DATA_LOSS, trailer);
    verify(listener).onClose(statusCaptor.capture(), eq(trailer));
    assertThat(statusCaptor.getValue().getCode()).isEqualTo(Status.Code.DATA_LOSS);
}

Example 68 with Metadata

use of org.apache.beam.vendor.grpc.v1p43p2.io.grpc.Metadata in project grpc-java by grpc.

the class DelayedClientCallTest method deadlineExceededWhileCallIsStartedButStillPending.

// Coverage for deadline exceeded before call started is enforced by
// AbstractInteropTest.deadlineInPast().
@Test
public void deadlineExceededWhileCallIsStartedButStillPending() {
    DelayedClientCall<String, Integer> delayedClientCall = new DelayedClientCall<>(callExecutor, fakeClock.getScheduledExecutorService(), Deadline.after(10, SECONDS));
    delayedClientCall.start(listener, new Metadata());
    fakeClock.forwardTime(10, SECONDS);
    verify(listener).onClose(statusCaptor.capture(), any(Metadata.class));
    assertThat(statusCaptor.getValue().getCode()).isEqualTo(Status.Code.DEADLINE_EXCEEDED);
}

Example 69 with Metadata

use of org.apache.beam.vendor.grpc.v1p43p2.io.grpc.Metadata in project grpc-java by grpc.

the class DelayedStreamTest method listener_allQueued.

@Test
public void listener_allQueued() {
    final Metadata headers = new Metadata();
    final InputStream message1 = mock(InputStream.class);
    final InputStream message2 = mock(InputStream.class);
    final SingleMessageProducer producer1 = new SingleMessageProducer(message1);
    final SingleMessageProducer producer2 = new SingleMessageProducer(message2);
    final Metadata trailers = new Metadata();
    final Status status = Status.UNKNOWN.withDescription("unique status");
    final InOrder inOrder = inOrder(listener);
    stream.start(listener);
    callMeMaybe(stream.setStream(new NoopClientStream() {

        @Override
        public void start(ClientStreamListener passedListener) {
            passedListener.onReady();
            passedListener.headersRead(headers);
            passedListener.messagesAvailable(producer1);
            passedListener.onReady();
            passedListener.messagesAvailable(producer2);
            passedListener.closed(status, RpcProgress.PROCESSED, trailers);
            verifyNoMoreInteractions(listener);
        }
    }));
    inOrder.verify(listener).onReady();
    inOrder.verify(listener).headersRead(headers);
    inOrder.verify(listener).messagesAvailable(producer1);
    inOrder.verify(listener).onReady();
    inOrder.verify(listener).messagesAvailable(producer2);
    inOrder.verify(listener).closed(status, RpcProgress.PROCESSED, trailers);
}

Example 70 with Metadata

use of org.apache.beam.vendor.grpc.v1p43p2.io.grpc.Metadata in project grpc-java by grpc.

the class AbstractTransportTest method basicStream.

@Test
public void basicStream() throws Exception {
    InOrder serverInOrder = inOrder(serverStreamTracerFactory);
    server.start(serverListener);
    client = newClientTransport(server);
    startTransport(client, mockClientTransportListener);
    // This attribute is available right after transport is started
    assertThat(((ConnectionClientTransport) client).getAttributes().get(GrpcAttributes.ATTR_CLIENT_EAG_ATTRS)).isSameInstanceAs(EAG_ATTRS);
    MockServerTransportListener serverTransportListener = serverListener.takeListenerOrFail(TIMEOUT_MS, TimeUnit.MILLISECONDS);
    serverTransport = serverTransportListener.transport;
    Metadata clientHeaders = new Metadata();
    clientHeaders.put(asciiKey, "client");
    clientHeaders.put(asciiKey, "dupvalue");
    clientHeaders.put(asciiKey, "dupvalue");
    clientHeaders.put(binaryKey, "äbinaryclient");
    clientHeaders.put(binaryKey, "dup,value");
    Metadata clientHeadersCopy = new Metadata();
    clientHeadersCopy.merge(clientHeaders);
    ClientStream clientStream = client.newStream(methodDescriptor, clientHeaders, callOptions, tracers);
    assertThat(((TestHeaderClientStreamTracer) clientStreamTracer1).transportAttrs).isSameInstanceAs(((ConnectionClientTransport) client).getAttributes());
    ClientStreamListenerBase clientStreamListener = new ClientStreamListenerBase();
    clientStream.start(clientStreamListener);
    StreamCreation serverStreamCreation = serverTransportListener.takeStreamOrFail(TIMEOUT_MS, TimeUnit.MILLISECONDS);
    assertTrue(clientStreamTracer1.awaitOutboundHeaders(TIMEOUT_MS, TimeUnit.MILLISECONDS));
    assertEquals(methodDescriptor.getFullMethodName(), serverStreamCreation.method);
    assertEquals(Lists.newArrayList(clientHeadersCopy.getAll(asciiKey)), Lists.newArrayList(serverStreamCreation.headers.getAll(asciiKey)));
    assertEquals(Lists.newArrayList(clientHeadersCopy.getAll(binaryKey)), Lists.newArrayList(serverStreamCreation.headers.getAll(binaryKey)));
    assertEquals(tracerKeyValue, serverStreamCreation.headers.get(tracerHeaderKey));
    ServerStream serverStream = serverStreamCreation.stream;
    ServerStreamListenerBase serverStreamListener = serverStreamCreation.listener;
    serverInOrder.verify(serverStreamTracerFactory).newServerStreamTracer(eq(methodDescriptor.getFullMethodName()), any(Metadata.class));
    assertEquals("additional attribute value", serverStream.getAttributes().get(ADDITIONAL_TRANSPORT_ATTR_KEY));
    assertNotNull(serverStream.getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR));
    assertNotNull(serverStream.getAttributes().get(Grpc.TRANSPORT_ATTR_LOCAL_ADDR));
    // This attribute is still available when the transport is connected
    assertThat(((ConnectionClientTransport) client).getAttributes().get(GrpcAttributes.ATTR_CLIENT_EAG_ATTRS)).isSameInstanceAs(EAG_ATTRS);
    serverStream.request(1);
    assertTrue(clientStreamListener.awaitOnReadyAndDrain(TIMEOUT_MS, TimeUnit.MILLISECONDS));
    assertTrue(clientStream.isReady());
    clientStream.writeMessage(methodDescriptor.streamRequest("Hello!"));
    assertThat(clientStreamTracer1.nextOutboundEvent()).isEqualTo("outboundMessage(0)");
    clientStream.flush();
    InputStream message = serverStreamListener.messageQueue.poll(TIMEOUT_MS, TimeUnit.MILLISECONDS);
    assertEquals("Hello!", methodDescriptor.parseRequest(message));
    message.close();
    assertThat(clientStreamTracer1.nextOutboundEvent()).matches("outboundMessageSent\\(0, -?[0-9]+, -?[0-9]+\\)");
    if (sizesReported()) {
        assertThat(clientStreamTracer1.getOutboundWireSize()).isGreaterThan(0L);
        assertThat(clientStreamTracer1.getOutboundUncompressedSize()).isGreaterThan(0L);
    } else {
        assertThat(clientStreamTracer1.getOutboundWireSize()).isEqualTo(0L);
        assertThat(clientStreamTracer1.getOutboundUncompressedSize()).isEqualTo(0L);
    }
    assertThat(serverStreamTracer1.nextInboundEvent()).isEqualTo("inboundMessage(0)");
    assertNull("no additional message expected", serverStreamListener.messageQueue.poll());
    clientStream.halfClose();
    assertTrue(serverStreamListener.awaitHalfClosed(TIMEOUT_MS, TimeUnit.MILLISECONDS));
    if (sizesReported()) {
        assertThat(serverStreamTracer1.getInboundWireSize()).isGreaterThan(0L);
        assertThat(serverStreamTracer1.getInboundUncompressedSize()).isGreaterThan(0L);
    } else {
        assertThat(serverStreamTracer1.getInboundWireSize()).isEqualTo(0L);
        assertThat(serverStreamTracer1.getInboundUncompressedSize()).isEqualTo(0L);
    }
    assertThat(serverStreamTracer1.nextInboundEvent()).matches("inboundMessageRead\\(0, -?[0-9]+, -?[0-9]+\\)");
    Metadata serverHeaders = new Metadata();
    serverHeaders.put(asciiKey, "server");
    serverHeaders.put(asciiKey, "dupvalue");
    serverHeaders.put(asciiKey, "dupvalue");
    serverHeaders.put(binaryKey, "äbinaryserver");
    serverHeaders.put(binaryKey, "dup,value");
    Metadata serverHeadersCopy = new Metadata();
    serverHeadersCopy.merge(serverHeaders);
    serverStream.writeHeaders(serverHeaders);
    Metadata headers = clientStreamListener.headers.get(TIMEOUT_MS, TimeUnit.MILLISECONDS);
    assertNotNull(headers);
    assertAsciiMetadataValuesEqual(serverHeadersCopy.getAll(asciiKey), headers.getAll(asciiKey));
    assertEquals(Lists.newArrayList(serverHeadersCopy.getAll(binaryKey)), Lists.newArrayList(headers.getAll(binaryKey)));
    clientStream.request(1);
    assertTrue(serverStreamListener.awaitOnReadyAndDrain(TIMEOUT_MS, TimeUnit.MILLISECONDS));
    assertTrue(serverStream.isReady());
    serverStream.writeMessage(methodDescriptor.streamResponse("Hi. Who are you?"));
    assertThat(serverStreamTracer1.nextOutboundEvent()).isEqualTo("outboundMessage(0)");
    serverStream.flush();
    message = clientStreamListener.messageQueue.poll(TIMEOUT_MS, TimeUnit.MILLISECONDS);
    assertNotNull("message expected", message);
    assertThat(serverStreamTracer1.nextOutboundEvent()).matches("outboundMessageSent\\(0, -?[0-9]+, -?[0-9]+\\)");
    if (sizesReported()) {
        assertThat(serverStreamTracer1.getOutboundWireSize()).isGreaterThan(0L);
        assertThat(serverStreamTracer1.getOutboundUncompressedSize()).isGreaterThan(0L);
    } else {
        assertThat(serverStreamTracer1.getOutboundWireSize()).isEqualTo(0L);
        assertThat(serverStreamTracer1.getOutboundUncompressedSize()).isEqualTo(0L);
    }
    assertTrue(clientStreamTracer1.getInboundHeaders());
    assertThat(clientStreamTracer1.nextInboundEvent()).isEqualTo("inboundMessage(0)");
    assertEquals("Hi. Who are you?", methodDescriptor.parseResponse(message));
    assertThat(clientStreamTracer1.nextInboundEvent()).matches("inboundMessageRead\\(0, -?[0-9]+, -?[0-9]+\\)");
    if (sizesReported()) {
        assertThat(clientStreamTracer1.getInboundWireSize()).isGreaterThan(0L);
        assertThat(clientStreamTracer1.getInboundUncompressedSize()).isGreaterThan(0L);
    } else {
        assertThat(clientStreamTracer1.getInboundWireSize()).isEqualTo(0L);
        assertThat(clientStreamTracer1.getInboundUncompressedSize()).isEqualTo(0L);
    }
    message.close();
    assertNull("no additional message expected", clientStreamListener.messageQueue.poll());
    Status status = Status.OK.withDescription("That was normal");
    Metadata trailers = new Metadata();
    trailers.put(asciiKey, "trailers");
    trailers.put(asciiKey, "dupvalue");
    trailers.put(asciiKey, "dupvalue");
    trailers.put(binaryKey, "äbinarytrailers");
    trailers.put(binaryKey, "dup,value");
    serverStream.close(status, trailers);
    assertNull(serverStreamTracer1.nextInboundEvent());
    assertNull(serverStreamTracer1.nextOutboundEvent());
    assertCodeEquals(Status.OK, serverStreamListener.status.get(TIMEOUT_MS, TimeUnit.MILLISECONDS));
    assertSame(status, serverStreamTracer1.getStatus());
    Status clientStreamStatus = clientStreamListener.status.get(TIMEOUT_MS, TimeUnit.MILLISECONDS);
    Metadata clientStreamTrailers = clientStreamListener.trailers.get(TIMEOUT_MS, TimeUnit.MILLISECONDS);
    assertSame(clientStreamTrailers, clientStreamTracer1.getInboundTrailers());
    assertSame(clientStreamStatus, clientStreamTracer1.getStatus());
    assertNull(clientStreamTracer1.nextInboundEvent());
    assertNull(clientStreamTracer1.nextOutboundEvent());
    assertEquals(status.getCode(), clientStreamStatus.getCode());
    assertEquals(status.getDescription(), clientStreamStatus.getDescription());
    assertAsciiMetadataValuesEqual(trailers.getAll(asciiKey), clientStreamTrailers.getAll(asciiKey));
    assertEquals(Lists.newArrayList(trailers.getAll(binaryKey)), Lists.newArrayList(clientStreamTrailers.getAll(binaryKey)));
}