Examples with ProducerAcks org.apache.pulsar.websocket.data.ProducerAcks used on opensource projects

Example 26 with ProducerAcks

use of org.apache.pulsar.websocket.data.ProducerAcks in project incubator-pulsar by apache.

the class TopicsTest method testProduceFailed.

@Test
public void testProduceFailed() throws Exception {
    admin.topics().createNonPartitionedTopic("persistent://" + testTenant + "/" + testNamespace + "/" + testTopicName);
    pulsar.getBrokerService().getTopic("persistent://" + testTenant + "/" + testNamespace + "/" + testTopicName, false).thenAccept(topic -> {
        try {
            PersistentTopic mockPersistentTopic = spy((PersistentTopic) topic.get());
            AtomicInteger count = new AtomicInteger();
            doAnswer(new Answer() {

                @Override
                public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
                    Topic.PublishContext publishContext = invocationOnMock.getArgument(1);
                    if (count.getAndIncrement() < 2) {
                        publishContext.completed(null, -1, -1);
                    } else {
                        publishContext.completed(new BrokerServiceException.TopicFencedException("Fake exception"), -1, -1);
                    }
                    return null;
                }
            }).when(mockPersistentTopic).publishMessage(any(), any());
            BrokerService mockBrokerService = spy(pulsar.getBrokerService());
            doReturn(CompletableFuture.completedFuture(Optional.of(mockPersistentTopic))).when(mockBrokerService).getTopic(anyString(), anyBoolean());
            doReturn(mockBrokerService).when(pulsar).getBrokerService();
            AsyncResponse asyncResponse = mock(AsyncResponse.class);
            Schema<String> schema = StringSchema.utf8();
            ProducerMessages producerMessages = new ProducerMessages();
            producerMessages.setKeySchema(ObjectMapperFactory.getThreadLocal().writeValueAsString(schema.getSchemaInfo()));
            producerMessages.setValueSchema(ObjectMapperFactory.getThreadLocal().writeValueAsString(schema.getSchemaInfo()));
            String message = "[" + "{\"key\":\"my-key\",\"payload\":\"RestProducer:1\",\"eventTime\":1603045262772,\"sequenceId\":1}," + "{\"key\":\"my-key\",\"payload\":\"RestProducer:2\",\"eventTime\":1603045262772,\"sequenceId\":2}," + "{\"key\":\"my-key\",\"payload\":\"RestProducer:3\",\"eventTime\":1603045262772,\"sequenceId\":3}," + "{\"key\":\"my-key\",\"payload\":\"RestProducer:4\",\"eventTime\":1603045262772,\"sequenceId\":4}]";
            producerMessages.setMessages(ObjectMapperFactory.getThreadLocal().readValue(message, new TypeReference<List<ProducerMessage>>() {
            }));
            // Previous request should trigger namespace bundle loading, retry produce.
            topics.produceOnPersistentTopic(asyncResponse, testTenant, testNamespace, testTopicName, false, producerMessages);
            ArgumentCaptor<Response> responseCaptor = ArgumentCaptor.forClass(Response.class);
            verify(asyncResponse, timeout(5000).times(1)).resume(responseCaptor.capture());
            Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.OK.getStatusCode());
            Object responseEntity = responseCaptor.getValue().getEntity();
            Assert.assertTrue(responseEntity instanceof ProducerAcks);
            ProducerAcks response = (ProducerAcks) responseEntity;
            Assert.assertEquals(response.getMessagePublishResults().size(), 4);
            int errorResponse = 0;
            for (int index = 0; index < response.getMessagePublishResults().size(); index++) {
                int errorCode = response.getMessagePublishResults().get(index).getErrorCode();
                if (0 == errorCode) {
                    Assert.assertEquals(Integer.parseInt(response.getMessagePublishResults().get(index).getMessageId().split(":")[2]), -1);
                    Assert.assertTrue(response.getMessagePublishResults().get(index).getMessageId().length() > 0);
                } else {
                    errorResponse++;
                    Assert.assertEquals(errorCode, 2);
                    Assert.assertEquals(response.getMessagePublishResults().get(index).getErrorMsg(), "org.apache.pulsar.broker.service.BrokerServiceException$" + "TopicFencedException: Fake exception");
                }
            }
            // Add entry start to fail after 2nd operation, we published 4 msg so expecting 2 error response.
            Assert.assertTrue(errorResponse == 2);
        } catch (Throwable e) {
            Assert.fail(e.getMessage());
        }
    }).get();
}

Example 27 with ProducerAcks

use of org.apache.pulsar.websocket.data.ProducerAcks in project incubator-pulsar by apache.

the class TopicsTest method testProduceWithJsonSchema.

@Test
public void testProduceWithJsonSchema() throws Exception {
    String topicName = "persistent://" + testTenant + "/" + testNamespace + "/" + testTopicName;
    admin.topics().createNonPartitionedTopic(topicName);
    AsyncResponse asyncResponse = mock(AsyncResponse.class);
    GenericSchema jsonSchema = GenericJsonSchema.of(JSONSchema.of(SchemaDefinition.builder().withPojo(PC.class).build()).getSchemaInfo());
    PC pc = new PC("dell", "alienware", 2021, GPU.AMD, new Seller("WA", "main street", 98004));
    PC anotherPc = new PC("asus", "rog", 2020, GPU.NVIDIA, new Seller("CA", "back street", 90232));
    Consumer consumer = pulsarClient.newConsumer(jsonSchema).topic(topicName).subscriptionName("my-sub").subscriptionType(SubscriptionType.Exclusive).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe();
    ProducerMessages producerMessages = new ProducerMessages();
    producerMessages.setValueSchema(ObjectMapperFactory.getThreadLocal().writeValueAsString(jsonSchema.getSchemaInfo()));
    String message = "[" + "{\"key\":\"my-key\",\"payload\":\"" + ObjectMapperFactory.getThreadLocal().writeValueAsString(pc).replace("\"", "\\\"") + "\",\"eventTime\":1603045262772,\"sequenceId\":1}," + "{\"key\":\"my-key\",\"payload\":\"" + ObjectMapperFactory.getThreadLocal().writeValueAsString(anotherPc).replace("\"", "\\\"") + "\",\"eventTime\":1603045262772,\"sequenceId\":2}]";
    producerMessages.setMessages(ObjectMapperFactory.getThreadLocal().readValue(message, new TypeReference<List<ProducerMessage>>() {
    }));
    topics.produceOnPersistentTopic(asyncResponse, testTenant, testNamespace, testTopicName, false, producerMessages);
    ArgumentCaptor<Response> responseCaptor = ArgumentCaptor.forClass(Response.class);
    verify(asyncResponse, timeout(5000).times(1)).resume(responseCaptor.capture());
    Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.OK.getStatusCode());
    Object responseEntity = responseCaptor.getValue().getEntity();
    Assert.assertTrue(responseEntity instanceof ProducerAcks);
    ProducerAcks response = (ProducerAcks) responseEntity;
    Assert.assertEquals(response.getMessagePublishResults().size(), 2);
    Assert.assertEquals(response.getSchemaVersion(), 0);
    for (int index = 0; index < response.getMessagePublishResults().size(); index++) {
        Assert.assertEquals(Integer.parseInt(response.getMessagePublishResults().get(index).getMessageId().split(":")[2]), -1);
        Assert.assertEquals(response.getMessagePublishResults().get(index).getErrorCode(), 0);
        Assert.assertTrue(response.getMessagePublishResults().get(index).getMessageId().length() > 0);
    }
    List<PC> expected = Arrays.asList(pc, anotherPc);
    Message<String> msg = null;
    // Assert all messages published by REST producer can be received by consumer in expected order.
    for (int i = 0; i < 2; i++) {
        msg = consumer.receive(2, TimeUnit.SECONDS);
        PC msgPc = ObjectMapperFactory.getThreadLocal().treeToValue(((GenericJsonRecord) jsonSchema.decode(msg.getData())).getJsonNode(), PC.class);
        Assert.assertEquals(msgPc.brand, expected.get(i).brand);
        Assert.assertEquals(msgPc.model, expected.get(i).model);
        Assert.assertEquals(msgPc.year, expected.get(i).year);
        Assert.assertEquals(msgPc.gpu, expected.get(i).gpu);
        Assert.assertEquals(msgPc.seller.state, expected.get(i).seller.state);
        Assert.assertEquals(msgPc.seller.street, expected.get(i).seller.street);
        Assert.assertEquals(msgPc.seller.zipCode, expected.get(i).seller.zipCode);
        Assert.assertEquals("my-key", msg.getKey());
    }
}

Example 28 with ProducerAcks

use of org.apache.pulsar.websocket.data.ProducerAcks in project pulsar by apache.

the class TopicsBase method internalPublishMessages.

private void internalPublishMessages(TopicName topicName, ProducerMessages request, List<Integer> partitionIndexes, AsyncResponse asyncResponse, Schema schema, SchemaVersion schemaVersion) {
    if (partitionIndexes.size() < 1) {
        asyncResponse.resume(new RestException(Status.INTERNAL_SERVER_ERROR, new BrokerServiceException.TopicNotFoundException("Topic not owned by current broker.")));
    }
    try {
        String producerName = (null == request.getProducerName() || request.getProducerName().isEmpty()) ? defaultProducerName : request.getProducerName();
        List<Message> messages = buildMessage(request, schema, producerName, topicName);
        List<CompletableFuture<PositionImpl>> publishResults = new ArrayList<>();
        List<ProducerAck> produceMessageResults = new ArrayList<>();
        // Try to publish messages to all partitions this broker owns in round robin mode.
        for (int index = 0; index < messages.size(); index++) {
            ProducerAck produceMessageResult = new ProducerAck();
            produceMessageResult.setMessageId(partitionIndexes.get(index % (int) partitionIndexes.size()) + "");
            produceMessageResults.add(produceMessageResult);
            publishResults.add(publishSingleMessageToPartition(topicName.getPartition(partitionIndexes.get(index % (int) partitionIndexes.size())).toString(), messages.get(index)));
        }
        FutureUtil.waitForAll(publishResults).thenRun(() -> {
            processPublishMessageResults(produceMessageResults, publishResults);
            asyncResponse.resume(Response.ok().entity(new ProducerAcks(produceMessageResults, ((LongSchemaVersion) schemaVersion).getVersion())).build());
        }).exceptionally(e -> {
            // Some message may published successfully, so till return ok with result for each individual message.
            processPublishMessageResults(produceMessageResults, publishResults);
            asyncResponse.resume(Response.ok().entity(new ProducerAcks(produceMessageResults, ((LongSchemaVersion) schemaVersion).getVersion())).build());
            return null;
        });
    } catch (Exception e) {
        if (log.isDebugEnabled()) {
            log.debug("Fail to publish messages with rest produce message request for topic  {}: {} ", topicName, e.getCause());
        }
        asyncResponse.resume(new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()));
    }
}

Example 29 with ProducerAcks

use of org.apache.pulsar.websocket.data.ProducerAcks in project pulsar by apache.

the class TopicsBase method internalPublishMessagesToPartition.

private void internalPublishMessagesToPartition(TopicName topicName, ProducerMessages request, int partition, AsyncResponse asyncResponse, Schema schema, SchemaVersion schemaVersion) {
    try {
        String producerName = (null == request.getProducerName() || request.getProducerName().isEmpty()) ? defaultProducerName : request.getProducerName();
        List<Message> messages = buildMessage(request, schema, producerName, topicName);
        List<CompletableFuture<PositionImpl>> publishResults = new ArrayList<>();
        List<ProducerAck> produceMessageResults = new ArrayList<>();
        for (int index = 0; index < messages.size(); index++) {
            ProducerAck produceMessageResult = new ProducerAck();
            produceMessageResult.setMessageId(partition + "");
            produceMessageResults.add(produceMessageResult);
            publishResults.add(publishSingleMessageToPartition(topicName.getPartition(partition).toString(), messages.get(index)));
        }
        FutureUtil.waitForAll(publishResults).thenRun(() -> {
            processPublishMessageResults(produceMessageResults, publishResults);
            asyncResponse.resume(Response.ok().entity(new ProducerAcks(produceMessageResults, ((LongSchemaVersion) schemaVersion).getVersion())).build());
        }).exceptionally(e -> {
            // Some message may published successfully, so till return ok with result for each individual message.
            processPublishMessageResults(produceMessageResults, publishResults);
            asyncResponse.resume(Response.ok().entity(new ProducerAcks(produceMessageResults, ((LongSchemaVersion) schemaVersion).getVersion())).build());
            return null;
        });
    } catch (Exception e) {
        if (log.isDebugEnabled()) {
            log.debug("Fail publish messages to single partition with rest produce message " + "request for topic  {}: {} ", topicName, e.getCause());
        }
        asyncResponse.resume(new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()));
    }
}

Example 30 with ProducerAcks

use of org.apache.pulsar.websocket.data.ProducerAcks in project pulsar by apache.

the class TopicsTest method testProduceWithAvroSchema.

@Test
public void testProduceWithAvroSchema() throws Exception {
    String topicName = "persistent://" + testTenant + "/" + testNamespace + "/" + testTopicName;
    admin.topics().createNonPartitionedTopic(topicName);
    AsyncResponse asyncResponse = mock(AsyncResponse.class);
    GenericSchemaImpl avroSchema = GenericAvroSchema.of(AvroSchema.of(SchemaDefinition.builder().withPojo(PC.class).build()).getSchemaInfo());
    PC pc = new PC("dell", "alienware", 2021, GPU.AMD, new Seller("WA", "main street", 98004));
    PC anotherPc = new PC("asus", "rog", 2020, GPU.NVIDIA, new Seller("CA", "back street", 90232));
    Consumer consumer = pulsarClient.newConsumer(avroSchema).topic(topicName).subscriptionName("my-sub").subscriptionType(SubscriptionType.Exclusive).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe();
    ProducerMessages producerMessages = new ProducerMessages();
    producerMessages.setValueSchema(ObjectMapperFactory.getThreadLocal().writeValueAsString(avroSchema.getSchemaInfo()));
    ReflectDatumWriter<PC> datumWriter = new ReflectDatumWriter(avroSchema.getAvroSchema());
    ByteArrayOutputStream outputStream1 = new ByteArrayOutputStream();
    ByteArrayOutputStream outputStream2 = new ByteArrayOutputStream();
    JsonEncoder encoder1 = EncoderFactory.get().jsonEncoder(avroSchema.getAvroSchema(), outputStream1);
    JsonEncoder encoder2 = EncoderFactory.get().jsonEncoder(avroSchema.getAvroSchema(), outputStream2);
    datumWriter.write(pc, encoder1);
    encoder1.flush();
    datumWriter.write(anotherPc, encoder2);
    encoder2.flush();
    String message = "[" + "{\"key\":\"my-key\",\"payload\":\"" + outputStream1.toString().replace("\"", "\\\"") + "\",\"eventTime\":1603045262772,\"sequenceId\":1}," + "{\"key\":\"my-key\",\"payload\":\"" + outputStream2.toString().replace("\"", "\\\"") + "\",\"eventTime\":1603045262772,\"sequenceId\":2}]";
    producerMessages.setMessages(ObjectMapperFactory.getThreadLocal().readValue(message, new TypeReference<List<ProducerMessage>>() {
    }));
    topics.produceOnPersistentTopic(asyncResponse, testTenant, testNamespace, testTopicName, false, producerMessages);
    ArgumentCaptor<Response> responseCaptor = ArgumentCaptor.forClass(Response.class);
    verify(asyncResponse, timeout(5000).times(1)).resume(responseCaptor.capture());
    Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.OK.getStatusCode());
    Object responseEntity = responseCaptor.getValue().getEntity();
    Assert.assertTrue(responseEntity instanceof ProducerAcks);
    ProducerAcks response = (ProducerAcks) responseEntity;
    Assert.assertEquals(response.getMessagePublishResults().size(), 2);
    Assert.assertEquals(response.getSchemaVersion(), 0);
    for (int index = 0; index < response.getMessagePublishResults().size(); index++) {
        Assert.assertEquals(Integer.parseInt(response.getMessagePublishResults().get(index).getMessageId().split(":")[2]), -1);
        Assert.assertEquals(response.getMessagePublishResults().get(index).getErrorCode(), 0);
        Assert.assertTrue(response.getMessagePublishResults().get(index).getMessageId().length() > 0);
    }
    List<PC> expected = Arrays.asList(pc, anotherPc);
    Message<String> msg = null;
    // Assert all messages published by REST producer can be received by consumer in expected order.
    for (int i = 0; i < 2; i++) {
        msg = consumer.receive(2, TimeUnit.SECONDS);
        GenericAvroRecord avroRecord = (GenericAvroRecord) avroSchema.decode(msg.getData());
        Assert.assertEquals(((Utf8) avroRecord.getAvroRecord().get("brand")).toString(), expected.get(i).brand);
        Assert.assertEquals(((Utf8) avroRecord.getAvroRecord().get("model")).toString(), expected.get(i).model);
        Assert.assertEquals((int) avroRecord.getAvroRecord().get("year"), expected.get(i).year);
        Assert.assertEquals(((GenericData.EnumSymbol) avroRecord.getAvroRecord().get("gpu")).toString(), expected.get(i).gpu.toString());
        Assert.assertEquals(((Utf8) ((GenericRecord) avroRecord.getAvroRecord().get("seller")).get("state")).toString(), expected.get(i).seller.state);
        Assert.assertEquals(((Utf8) ((GenericRecord) avroRecord.getAvroRecord().get("seller")).get("street")).toString(), expected.get(i).seller.street);
        Assert.assertEquals(((GenericRecord) avroRecord.getAvroRecord().get("seller")).get("zipCode"), expected.get(i).seller.zipCode);
        Assert.assertEquals("my-key", msg.getKey());
    }
}