Examples with StreamConfig co.cask.cdap.data2.transaction.stream.StreamConfig used on opensource projects

Example 16 with StreamConfig

use of co.cask.cdap.data2.transaction.stream.StreamConfig in project cdap by caskdata.

the class StreamFetchHandler method fetch.

/**
   * Handler for the HTTP API {@code /streams/[stream_name]/events?start=[start_ts]&end=[end_ts]&limit=[event_limit]}
   * <p>
   * Responds with:
   * <ul>
   * <li>404 if stream does not exist</li>
   * <li>204 if no event in the given start/end time range exists</li>
   * <li>200 if there is are one or more events</li>
   * </ul>
   * </p>
   * <p>
   * Response body is a JSON array of the StreamEvent object.
   * </p>
   *
   * @see StreamEventTypeAdapter StreamEventTypeAdapter for the format of the StreamEvent object
   */
@GET
@Path("/{stream}/events")
public void fetch(HttpRequest request, final HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("stream") String stream, @QueryParam("start") @DefaultValue("0") String start, @QueryParam("end") @DefaultValue("9223372036854775807") String end, @QueryParam("limit") @DefaultValue("2147483647") final int limitEvents) throws Exception {
    long startTime = TimeMathParser.parseTime(start, TimeUnit.MILLISECONDS);
    long endTime = TimeMathParser.parseTime(end, TimeUnit.MILLISECONDS);
    StreamId streamId = new StreamId(namespaceId, stream);
    if (!verifyGetEventsRequest(streamId, startTime, endTime, limitEvents, responder)) {
        return;
    }
    // Make sure the user has READ permission on the stream since getConfig doesn't check for the same.
    authorizationEnforcer.enforce(streamId, authenticationContext.getPrincipal(), Action.READ);
    final StreamConfig streamConfig = streamAdmin.getConfig(streamId);
    long now = System.currentTimeMillis();
    startTime = Math.max(startTime, now - streamConfig.getTTL());
    endTime = Math.min(endTime, now);
    final long streamStartTime = startTime;
    final long streamEndTime = endTime;
    impersonator.doAs(streamId, new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            int limit = limitEvents;
            // Create the stream event reader
            try (FileReader<StreamEventOffset, Iterable<StreamFileOffset>> reader = createReader(streamConfig, streamStartTime)) {
                TimeRangeReadFilter readFilter = new TimeRangeReadFilter(streamStartTime, streamEndTime);
                List<StreamEvent> events = Lists.newArrayListWithCapacity(100);
                // Reads the first batch of events from the stream.
                int eventsRead = readEvents(reader, events, limit, readFilter);
                // If empty already, return 204 no content
                if (eventsRead <= 0) {
                    responder.sendStatus(HttpResponseStatus.NO_CONTENT);
                    return null;
                }
                // Send with chunk response, as we don't want to buffer all events in memory to determine the content-length.
                ChunkResponder chunkResponder = responder.sendChunkStart(HttpResponseStatus.OK, ImmutableMultimap.of(HttpHeaders.Names.CONTENT_TYPE, "application/json; charset=utf-8"));
                ChannelBuffer buffer = ChannelBuffers.dynamicBuffer();
                JsonWriter jsonWriter = new JsonWriter(new OutputStreamWriter(new ChannelBufferOutputStream(buffer), Charsets.UTF_8));
                // Response is an array of stream event
                jsonWriter.beginArray();
                while (limit > 0 && eventsRead > 0) {
                    limit -= eventsRead;
                    for (StreamEvent event : events) {
                        GSON.toJson(event, StreamEvent.class, jsonWriter);
                        jsonWriter.flush();
                        // If exceeded chunk size limit, send a new chunk.
                        if (buffer.readableBytes() >= CHUNK_SIZE) {
                            // If the connect is closed, sendChunk will throw IOException.
                            // No need to handle the exception as it will just propagated back to the netty-http library
                            // and it will handle it.
                            // Need to copy the buffer because the buffer will get reused and send chunk is an async operation
                            chunkResponder.sendChunk(buffer.copy());
                            buffer.clear();
                        }
                    }
                    events.clear();
                    if (limit > 0) {
                        eventsRead = readEvents(reader, events, limit, readFilter);
                    }
                }
                jsonWriter.endArray();
                jsonWriter.close();
                // Send the last chunk that still has data
                if (buffer.readable()) {
                    // No need to copy the last chunk, since the buffer will not be reused
                    chunkResponder.sendChunk(buffer);
                }
                Closeables.closeQuietly(chunkResponder);
            }
            return null;
        }
    });
}

Example 17 with StreamConfig

use of co.cask.cdap.data2.transaction.stream.StreamConfig in project cdap by caskdata.

the class ConcurrentStreamWriter method appendFile.

/**
   * Appends a new stream file to the given stream.
   *
   * @param streamId identifier of the stream
   * @param eventFile location to the new stream data file
   * @param indexFile location to the new stream index file
   * @param eventCount number of events in the given stream file
   * @param timestampCloseable a {@link TimestampCloseable} to close and return the stream file close timestamp
   * @throws IOException if failed to append the new stream file
   */
public void appendFile(StreamId streamId, Location eventFile, Location indexFile, long eventCount, TimestampCloseable timestampCloseable) throws IOException, NotFoundException {
    EventQueue eventQueue = getEventQueue(streamId);
    StreamConfig config = streamAdmin.getConfig(streamId);
    while (!eventQueue.tryAppendFile(config, eventFile, indexFile, eventCount, timestampCloseable)) {
        Thread.yield();
    }
}

Example 18 with StreamConfig

use of co.cask.cdap.data2.transaction.stream.StreamConfig in project cdap by caskdata.

the class StreamConsumerTestBase method testTTL.

@Test
public void testTTL() throws Exception {
    String stream = "testTTL";
    StreamId streamId = TEST_NAMESPACE.stream(stream);
    StreamAdmin streamAdmin = getStreamAdmin();
    // Create stream with ttl of 1 day
    final long ttl = TimeUnit.DAYS.toMillis(1);
    final long currentTime = System.currentTimeMillis();
    final long increment = TimeUnit.SECONDS.toMillis(1);
    final long approxEarliestNonExpiredTime = currentTime - TimeUnit.HOURS.toMillis(1);
    Properties streamProperties = new Properties();
    streamProperties.setProperty(Constants.Stream.TTL, Long.toString(ttl));
    streamProperties.setProperty(Constants.Stream.PARTITION_DURATION, Long.toString(ttl));
    streamAdmin.create(streamId, streamProperties);
    StreamConfig streamConfig = streamAdmin.getConfig(streamId);
    streamAdmin.configureInstances(streamId, 0L, 1);
    StreamConsumerFactory consumerFactory = getConsumerFactory();
    Assert.assertEquals(ttl, streamConfig.getTTL());
    Assert.assertEquals(ttl, streamConfig.getPartitionDuration());
    Set<StreamEvent> expectedEvents = Sets.newTreeSet(STREAM_EVENT_COMPARATOR);
    FileWriter<StreamEvent> writer = getFileWriterFactory().create(streamConfig, 0);
    try {
        // Write 10 expired messages
        writeEvents(streamConfig, "Old event ", 20, new IncrementingClock(0, 1));
        // Write 5 non-expired messages
        expectedEvents.addAll(writeEvents(streamConfig, "New event ", 12, new IncrementingClock(approxEarliestNonExpiredTime, increment)));
    } finally {
        writer.close();
    }
    // Dequeue from stream. Should only get the 5 unexpired events.
    StreamConsumer consumer = consumerFactory.create(streamId, stream, new ConsumerConfig(0L, 0, 1, DequeueStrategy.FIFO, null));
    try {
        verifyEvents(consumer, expectedEvents);
        TransactionContext txContext = createTxContext(consumer);
        txContext.start();
        try {
            // Should be no more pending events
            DequeueResult<StreamEvent> result = consumer.poll(1, 2, TimeUnit.SECONDS);
            Assert.assertTrue(result.isEmpty());
        } finally {
            txContext.finish();
        }
    } finally {
        consumer.close();
    }
}

Example 19 with StreamConfig

use of co.cask.cdap.data2.transaction.stream.StreamConfig in project cdap by caskdata.

the class StreamConsumerTestBase method testTTLMultipleEventsWithSameTimestamp.

@Test
public void testTTLMultipleEventsWithSameTimestamp() throws Exception {
    String stream = "testTTLMultipleEventsWithSameTimestamp";
    StreamId streamId = TEST_NAMESPACE.stream(stream);
    StreamAdmin streamAdmin = getStreamAdmin();
    // Create stream with ttl of 1 day
    final long ttl = TimeUnit.DAYS.toMillis(1);
    final long currentTime = System.currentTimeMillis();
    final long increment = TimeUnit.SECONDS.toMillis(1);
    final long approxEarliestNonExpiredTime = currentTime - TimeUnit.HOURS.toMillis(1);
    Properties streamProperties = new Properties();
    streamProperties.setProperty(Constants.Stream.TTL, Long.toString(ttl));
    streamProperties.setProperty(Constants.Stream.PARTITION_DURATION, Long.toString(ttl));
    streamAdmin.create(streamId, streamProperties);
    StreamConfig streamConfig = streamAdmin.getConfig(streamId);
    streamAdmin.configureInstances(streamId, 0L, 1);
    StreamConsumerFactory consumerFactory = getConsumerFactory();
    Assert.assertEquals(ttl, streamConfig.getTTL());
    Assert.assertEquals(ttl, streamConfig.getPartitionDuration());
    // Write 100 expired messages to stream with expired timestamp
    writeEvents(streamConfig, "Old event ", 10, new ConstantClock(0));
    // Write 500 non-expired messages to stream with timestamp approxEarliestNonExpiredTime..currentTime
    Set<StreamEvent> expectedEvents = Sets.newTreeSet(STREAM_EVENT_COMPARATOR);
    FileWriter<StreamEvent> writer = getFileWriterFactory().create(streamConfig, 0);
    try {
        expectedEvents.addAll(writeEvents(writer, "New event pre-flush ", 20, new IncrementingClock(approxEarliestNonExpiredTime, increment, 5)));
        writer.flush();
        expectedEvents.addAll(writeEvents(writer, "New event post-flush ", 20, new IncrementingClock(approxEarliestNonExpiredTime + 1, increment, 5)));
    } finally {
        writer.close();
    }
    StreamConsumer consumer = consumerFactory.create(streamId, stream, new ConsumerConfig(0L, 0, 1, DequeueStrategy.FIFO, null));
    verifyEvents(consumer, expectedEvents);
    TransactionContext txContext = createTxContext(consumer);
    txContext.start();
    try {
        // Should be no more pending events
        DequeueResult<StreamEvent> result = consumer.poll(1, 1, TimeUnit.SECONDS);
        Assert.assertTrue(result.isEmpty());
    } finally {
        txContext.finish();
    }
    consumer.close();
}

Example 20 with StreamConfig

use of co.cask.cdap.data2.transaction.stream.StreamConfig in project cdap by caskdata.

the class StreamConsumerTestBase method testFIFOReconfigure.

@Test
public void testFIFOReconfigure() throws Exception {
    String stream = "testReconfigure";
    StreamId streamId = TEST_NAMESPACE.stream(stream);
    StreamAdmin streamAdmin = getStreamAdmin();
    streamAdmin.create(streamId);
    StreamConfig streamConfig = streamAdmin.getConfig(streamId);
    // Writes 5 events
    writeEvents(streamConfig, "Testing ", 5);
    // Configure 3 consumers.
    streamAdmin.configureInstances(streamId, 0L, 3);
    StreamConsumerFactory consumerFactory = getConsumerFactory();
    // Starts three consumers
    List<StreamConsumer> consumers = Lists.newArrayList();
    for (int i = 0; i < 3; i++) {
        consumers.add(consumerFactory.create(streamId, "fifo.reconfigure", new ConsumerConfig(0L, i, 3, DequeueStrategy.FIFO, null)));
    }
    List<TransactionContext> txContexts = Lists.newArrayList();
    for (StreamConsumer consumer : consumers) {
        txContexts.add(createTxContext(consumer));
    }
    for (TransactionContext txContext : txContexts) {
        txContext.start();
    }
    // Consumer an item from each consumer, but only have the first one commit.
    for (int i = 0; i < consumers.size(); i++) {
        DequeueResult<StreamEvent> result = consumers.get(i).poll(1, 1, TimeUnit.SECONDS);
        Assert.assertEquals("Testing " + i, Charsets.UTF_8.decode(result.iterator().next().getBody()).toString());
        if (i == 0) {
            txContexts.get(i).finish();
        } else {
            txContexts.get(i).abort();
        }
    }
    for (StreamConsumer consumer : consumers) {
        consumer.close();
    }
    // Reconfigure to have two consumers.
    streamAdmin.configureInstances(streamId, 0L, 2);
    consumers.clear();
    for (int i = 0; i < 2; i++) {
        consumers.add(consumerFactory.create(streamId, "fifo.reconfigure", new ConsumerConfig(0L, i, 2, DequeueStrategy.FIFO, null)));
    }
    txContexts.clear();
    for (StreamConsumer consumer : consumers) {
        txContexts.add(createTxContext(consumer));
    }
    // Consumer an item from each consumer, they should see all four items.
    Set<String> messages = Sets.newTreeSet();
    boolean done;
    do {
        for (TransactionContext txContext : txContexts) {
            txContext.start();
        }
        done = true;
        for (int i = 0; i < consumers.size(); i++) {
            DequeueResult<StreamEvent> result = consumers.get(i).poll(1, 1, TimeUnit.SECONDS);
            if (result.isEmpty()) {
                continue;
            }
            done = false;
            messages.add(Charsets.UTF_8.decode(result.iterator().next().getBody()).toString());
            txContexts.get(i).finish();
        }
    } while (!done);
    Assert.assertEquals(4, messages.size());
    int count = 1;
    for (String msg : messages) {
        Assert.assertEquals("Testing " + count, msg);
        count++;
    }
    for (StreamConsumer consumer : consumers) {
        consumer.close();
    }
}