Examples with BagheeraMessage com.mozilla.bagheera.BagheeraProto.BagheeraMessage used on opensource projects

Example 6 with BagheeraMessage

use of com.mozilla.bagheera.BagheeraProto.BagheeraMessage in project bagheera by mozilla-metrics.

the class SimpleProducer method testDeleteWithPartitions.

@Test
public void testDeleteWithPartitions() throws IOException, InterruptedException {
    // Use a ReplaySink to send messages
    String destPattern = String.format("http://localhost:%d/%s/%s/%s/partition1/partition2", BAGHEERA_PORT, SubmissionHandler.ENDPOINT_SUBMIT, TEST_NAMESPACE, ReplaySink.KEY_PLACEHOLDER);
    ReplaySink sink = new ReplaySink(destPattern, "1", "true", "true");
    sink.delete(key);
    assertEquals(1, producer.queueSize());
    BagheeraMessage message = producer.getQueue().poll();
    String payload = message.getPayload().toStringUtf8();
    assertEquals("", payload);
    // ReplaySink doesn't preserve timestamps.
    assertNotSame(timestamp, message.getTimestamp());
    assertEquals(key, message.getId());
    assertEquals(BagheeraMessage.Operation.DELETE, message.getOperation());
    // Ensure that partition information comes through.
    assertEquals(2, message.getPartitionCount());
    assertEquals("partition1", message.getPartition(0));
    assertEquals("partition2", message.getPartition(1));
}

Example 7 with BagheeraMessage

use of com.mozilla.bagheera.BagheeraProto.BagheeraMessage in project bagheera by mozilla-metrics.

the class SimpleProducer method testMessageWithApiVersion.

@Test
public void testMessageWithApiVersion() throws IOException, InterruptedException {
    // Use a ReplaySink to send messages
    String destPattern = String.format("http://localhost:%d/5.5/%s/%s/%s/partition1/partition2", BAGHEERA_PORT, SubmissionHandler.ENDPOINT_SUBMIT, TEST_NAMESPACE, ReplaySink.KEY_PLACEHOLDER);
    ReplaySink sink = new ReplaySink(destPattern, "1", "true", "true");
    sink.store(key, json.getBytes(), timestamp);
    assertEquals(1, producer.queueSize());
    BagheeraMessage message = producer.getQueue().poll();
    String payload = message.getPayload().toStringUtf8();
    assertEquals(json, payload);
    // ReplaySink doesn't preserve timestamps.
    assertNotSame(timestamp, message.getTimestamp());
    assertEquals(key, message.getId());
    // Ensure that partition information comes through.
    assertEquals(2, message.getPartitionCount());
    assertEquals("partition1", message.getPartition(0));
    assertEquals("partition2", message.getPartition(1));
    assertEquals("5.5", message.getApiVersion());
}

Example 8 with BagheeraMessage

use of com.mozilla.bagheera.BagheeraProto.BagheeraMessage in project bagheera by mozilla-metrics.

the class SimpleProducer method testMessageWithPartitions.

@Test
public void testMessageWithPartitions() throws IOException, InterruptedException {
    // Use a ReplaySink to send messages
    String destPattern = String.format("http://localhost:%d/%s/%s/%s/partition1/partition2", BAGHEERA_PORT, SubmissionHandler.ENDPOINT_SUBMIT, TEST_NAMESPACE, ReplaySink.KEY_PLACEHOLDER);
    ReplaySink sink = new ReplaySink(destPattern, "1", "true", "true");
    sink.store(key, json.getBytes(), timestamp);
    assertEquals(1, producer.queueSize());
    BagheeraMessage message = producer.getQueue().poll();
    String payload = message.getPayload().toStringUtf8();
    assertEquals(json, payload);
    // ReplaySink doesn't preserve timestamps.
    assertNotSame(timestamp, message.getTimestamp());
    assertEquals(key, message.getId());
    assertEquals(BagheeraMessage.Operation.CREATE_UPDATE, message.getOperation());
    // Ensure that partition information comes through.
    assertEquals(2, message.getPartitionCount());
    assertEquals("partition1", message.getPartition(0));
    assertEquals("partition2", message.getPartition(1));
}

Example 9 with BagheeraMessage

use of com.mozilla.bagheera.BagheeraProto.BagheeraMessage in project bagheera by mozilla-metrics.

the class ProducerTest method countMessages.

private int countMessages() throws InvalidProtocolBufferException {
    SimpleConsumer consumer = new SimpleConsumer("localhost", KAFKA_BROKER_PORT, 100, 1024);
    long offset = 0l;
    int messageCount = 0;
    for (int i = 0; i < BATCH_SIZE; i++) {
        ByteBufferMessageSet messageSet = consumer.fetch(new FetchRequest(KAFKA_TOPIC, 0, offset, 1024));
        Iterator<MessageAndOffset> iterator = messageSet.iterator();
        MessageAndOffset msgAndOff;
        while (iterator.hasNext()) {
            messageCount++;
            msgAndOff = iterator.next();
            offset = msgAndOff.offset();
            Message message2 = msgAndOff.message();
            BagheeraMessage bmsg = BagheeraMessage.parseFrom(ByteString.copyFrom(message2.payload()));
            String payload = new String(bmsg.getPayload().toByteArray());
            System.out.println(String.format("Message %d @%d: %s", messageCount, offset, payload));
        }
    }
    consumer.close();
    return messageCount;
}

Example 10 with BagheeraMessage

use of com.mozilla.bagheera.BagheeraProto.BagheeraMessage in project bagheera by mozilla-metrics.

the class SubmissionHandler method handleObsoleteDocuments.

private void handleObsoleteDocuments(BagheeraHttpRequest request, String remoteIpAddress, List<String> headers, BagheeraMessage template) {
    // According to RFC 2616, the standard for multi-valued document headers is
    // a comma-separated list:
    // http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
    // ------------------------------------------------------------------
    // Multiple message-header fields with the same field-name MAY be
    // present in a message if and only if the entire field-value for
    // that header field is defined as a comma-separated list
    // [i.e., #(values)]. It MUST be possible to combine the multiple
    // header fields into one "field-name: field-value" pair, without
    // changing the semantics of the message, by appending each
    // subsequent field-value to the first, each separated by a comma.
    // The order in which header fields with the same field-name are
    // received is therefore significant to the interpretation of the
    // combined field value, and thus a proxy MUST NOT change the order
    // of these field values when a message is forwarded.
    // ------------------------------------------------------------------
    String deleteIDs = "";
    for (String header : headers) {
        // tested in BagheeraHttpRequestTest.testSplitPerformance().
        if (header != null) {
            for (String obsoleteIdRaw : header.split(",")) {
                deleteIDs += obsoleteIdRaw.trim() + ",";
                // Use the given message as a base for creating each delete message.
                BagheeraMessage.Builder deleteBuilder = BagheeraMessage.newBuilder(template);
                deleteBuilder.setOperation(Operation.DELETE);
                deleteBuilder.setId(obsoleteIdRaw.trim());
                producer.send(deleteBuilder.build());
            }
        }
    }
    LOG.info("IP " + remoteIpAddress + " " + request.getNamespace() + " HTTP_PUT " + request.getId() + " HTTP_DELETE " + deleteIDs);
}