Examples with StreamEvent co.cask.cdap.api.flow.flowlet.StreamEvent used on opensource projects

Example 41 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class FlowletProgramRunner method processSpecificationFactory.

private ProcessSpecificationFactory processSpecificationFactory(final BasicFlowletContext flowletContext, final DataFabricFacade dataFabricFacade, final QueueReaderFactory queueReaderFactory, final String flowletName, final Table<Node, String, Set<QueueSpecification>> queueSpecs, final ImmutableList.Builder<ConsumerSupplier<?>> queueConsumerSupplierBuilder, final SchemaCache schemaCache) {
    return new ProcessSpecificationFactory() {

        @Override
        public <T> ProcessSpecification create(Set<String> inputNames, Schema schema, TypeToken<T> dataType, ProcessMethod<T> method, ConsumerConfig consumerConfig, int batchSize, Tick tickAnnotation) throws Exception {
            List<QueueReader<T>> queueReaders = Lists.newLinkedList();
            for (Map.Entry<Node, Set<QueueSpecification>> entry : queueSpecs.column(flowletName).entrySet()) {
                for (QueueSpecification queueSpec : entry.getValue()) {
                    final QueueName queueName = queueSpec.getQueueName();
                    if (queueSpec.getInputSchema().equals(schema) && (inputNames.contains(queueName.getSimpleName()) || inputNames.contains(FlowletDefinition.ANY_INPUT))) {
                        Node sourceNode = entry.getKey();
                        if (sourceNode.getType() == FlowletConnection.Type.STREAM) {
                            ConsumerSupplier<StreamConsumer> consumerSupplier = ConsumerSupplier.create(flowletContext.getOwners(), runtimeUsageRegistry, dataFabricFacade, queueName, consumerConfig);
                            queueConsumerSupplierBuilder.add(consumerSupplier);
                            // No decoding is needed, as a process method can only have StreamEvent as type for consuming stream
                            Function<StreamEvent, T> decoder = wrapInputDecoder(flowletContext, null, queueName, new Function<StreamEvent, T>() {

                                @Override
                                @SuppressWarnings("unchecked")
                                public T apply(StreamEvent input) {
                                    return (T) input;
                                }
                            });
                            queueReaders.add(queueReaderFactory.createStreamReader(queueName.toStreamId(), consumerSupplier, batchSize, decoder));
                        } else {
                            int numGroups = getNumGroups(Iterables.concat(queueSpecs.row(entry.getKey()).values()), queueName);
                            Function<ByteBuffer, T> decoder = wrapInputDecoder(// the producer flowlet,
                            flowletContext, // the producer flowlet,
                            entry.getKey().getName(), queueName, createInputDatumDecoder(dataType, schema, schemaCache));
                            ConsumerSupplier<QueueConsumer> consumerSupplier = ConsumerSupplier.create(flowletContext.getOwners(), runtimeUsageRegistry, dataFabricFacade, queueName, consumerConfig, numGroups);
                            queueConsumerSupplierBuilder.add(consumerSupplier);
                            queueReaders.add(queueReaderFactory.createQueueReader(consumerSupplier, batchSize, decoder));
                        }
                    }
                }
            }
            // If inputs is needed but there is no available input queue, return null
            if (!inputNames.isEmpty() && queueReaders.isEmpty()) {
                return null;
            }
            return new ProcessSpecification<>(new RoundRobinQueueReader<>(queueReaders), method, tickAnnotation);
        }
    };
}

Example 42 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class StreamClient method getEvents.

/**
 * Reads events from a stream
 *
 * @param streamId ID of the stream
 * @param start Timestamp in milliseconds or now-xs format to start reading event from (inclusive)
 * @param end Timestamp in milliseconds or now-xs format for the last event to read (exclusive)
 * @param limit Maximum number of events to read
 * @param callback Callback to invoke for each stream event read. If the callback function returns {@code false}
 *                 upon invocation, it will stops the reading
 * @throws IOException If fails to read from stream
 * @throws StreamNotFoundException If the given stream does not exists
 */
public void getEvents(StreamId streamId, String start, String end, int limit, Function<? super StreamEvent, Boolean> callback) throws IOException, StreamNotFoundException, UnauthenticatedException, UnauthorizedException {
    long startTime = TimeMathParser.parseTime(start, TimeUnit.MILLISECONDS);
    long endTime = TimeMathParser.parseTime(end, TimeUnit.MILLISECONDS);
    URL url = config.resolveNamespacedURLV3(streamId.getParent(), String.format("streams/%s/events?start=%d&end=%d&limit=%d", streamId.getStream(), startTime, endTime, limit));
    HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();
    AccessToken accessToken = config.getAccessToken();
    if (accessToken != null) {
        urlConn.setRequestProperty(HttpHeaders.AUTHORIZATION, accessToken.getTokenType() + " " + accessToken.getValue());
    }
    if (urlConn instanceof HttpsURLConnection && !config.isVerifySSLCert()) {
        try {
            HttpRequests.disableCertCheck((HttpsURLConnection) urlConn);
        } catch (Exception e) {
        // TODO: Log "Got exception while disabling SSL certificate check for request.getURL()"
        }
    }
    try {
        if (urlConn.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
            throw new UnauthenticatedException("Unauthorized status code received from the server.");
        }
        if (urlConn.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
            throw new StreamNotFoundException(streamId);
        }
        if (urlConn.getResponseCode() == HttpURLConnection.HTTP_NO_CONTENT) {
            return;
        }
        if (urlConn.getResponseCode() == HttpURLConnection.HTTP_FORBIDDEN) {
            throw new UnauthorizedException(CharStreams.toString(new InputStreamReader(urlConn.getErrorStream(), Charsets.UTF_8)));
        }
        // The response is an array of stream event object
        InputStream inputStream = urlConn.getInputStream();
        JsonReader jsonReader = new JsonReader(new InputStreamReader(inputStream, Charsets.UTF_8));
        jsonReader.beginArray();
        while (jsonReader.peek() != JsonToken.END_ARRAY) {
            Boolean result = callback.apply(GSON.<StreamEvent>fromJson(jsonReader, StreamEvent.class));
            if (result == null || !result) {
                break;
            }
        }
        drain(inputStream);
    // No need to close reader, the urlConn.disconnect in finally will close all underlying streams
    } finally {
        urlConn.disconnect();
    }
}

Example 43 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class StreamClientTestRun method testAsyncWrite.

/**
 * Tests for async write to stream.
 */
@Test
public void testAsyncWrite() throws Exception {
    StreamId streamId = namespaceId.stream("testAsync");
    streamClient.create(streamId);
    // Send 10 async writes
    int msgCount = 10;
    for (int i = 0; i < msgCount; i++) {
        streamClient.asyncSendEvent(streamId, "Testing " + i);
    }
    // Reads them back to verify. Needs to do it multiple times as the writes happens async.
    List<StreamEvent> events = Lists.newArrayList();
    Stopwatch stopwatch = new Stopwatch();
    stopwatch.start();
    while (events.size() != msgCount && stopwatch.elapsedTime(TimeUnit.SECONDS) < 10L) {
        events.clear();
        streamClient.getEvents(streamId, 0, Long.MAX_VALUE, msgCount, events);
    }
    Assert.assertEquals(msgCount, events.size());
    long lastTimestamp = 0L;
    for (int i = 0; i < msgCount; i++) {
        Assert.assertEquals("Testing " + i, Charsets.UTF_8.decode(events.get(i).getBody()).toString());
        lastTimestamp = events.get(i).getTimestamp();
    }
    // No more events
    stopwatch = new Stopwatch();
    stopwatch.start();
    events.clear();
    while (events.isEmpty() && stopwatch.elapsedTime(TimeUnit.SECONDS) < 1L) {
        events.clear();
        streamClient.getEvents(streamId, lastTimestamp + 1, Long.MAX_VALUE, msgCount, events);
    }
    Assert.assertTrue(events.isEmpty());
}

Example 44 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class StreamClientTestRun method testStreamEvents.

/**
 * Tests for the get events call
 */
@Test
public void testStreamEvents() throws IOException, BadRequestException, StreamNotFoundException, UnauthenticatedException, UnauthorizedException {
    StreamId streamId = namespaceId.stream("testEvents");
    streamClient.create(streamId);
    // Send 5000 events
    int eventCount = 5000;
    for (int i = 0; i < eventCount; i++) {
        streamClient.sendEvent(streamId, "Testing " + i);
    }
    // Read all events
    List<StreamEvent> events = streamClient.getEvents(streamId, 0, Long.MAX_VALUE, Integer.MAX_VALUE, Lists.<StreamEvent>newArrayList());
    Assert.assertEquals(eventCount, events.size());
    for (int i = 0; i < eventCount; i++) {
        Assert.assertEquals("Testing " + i, Bytes.toString(events.get(i).getBody()));
    }
    // Read first 5 only
    events.clear();
    streamClient.getEvents(streamId, 0, Long.MAX_VALUE, 5, events);
    Assert.assertEquals(5, events.size());
    // Use the 2nd and the 3rd events time as start and end time respectively
    long startTime = events.get(1).getTimestamp();
    long endTime = events.get(2).getTimestamp() + 1;
    events.clear();
    streamClient.getEvents(streamId, startTime, endTime, Integer.MAX_VALUE, events);
    // At least read the 2nd and the 3rd event. It might read more than 2 events
    // if events are written within the same timestamp.
    Assert.assertTrue(events.size() >= 2);
    int i = 1;
    for (StreamEvent event : events) {
        Assert.assertEquals("Testing " + i, Charsets.UTF_8.decode(events.get(i - 1).getBody()).toString());
        Assert.assertTrue(event.getTimestamp() >= startTime);
        Assert.assertTrue(event.getTimestamp() < endTime);
        i++;
    }
}

Example 45 with StreamEvent

use of co.cask.cdap.api.flow.flowlet.StreamEvent in project cdap by caskdata.

the class StreamClientTestRun method testDelete.

@Test
public void testDelete() throws Exception {
    StreamId streamId = namespaceId.stream("testDelete");
    streamClient.create(streamId);
    // Send an event and get it back
    String msg = "Test Delete";
    streamClient.sendEvent(streamId, msg);
    List<StreamEvent> events = Lists.newArrayList();
    streamClient.getEvents(streamId, 0, Long.MAX_VALUE, Integer.MAX_VALUE, events);
    Assert.assertEquals(1, events.size());
    Assert.assertEquals(msg, Charsets.UTF_8.decode(events.get(0).getBody()).toString());
    // Delete the stream
    streamClient.delete(streamId);
    // Try to get info, it should throw a StreamNotFoundException
    try {
        streamClient.getConfig(streamId);
        Assert.fail();
    } catch (StreamNotFoundException e) {
    // Expected
    }
    // Try to get events, it should throw a StreamNotFoundException
    try {
        streamClient.getEvents(streamId, 0, Long.MAX_VALUE, Integer.MAX_VALUE, events);
        Assert.fail();
    } catch (StreamNotFoundException e) {
    // Expected
    }
    // Create the stream again, it should returns empty events
    streamClient.create(streamId);
    events.clear();
    streamClient.getEvents(streamId, 0, Long.MAX_VALUE, Integer.MAX_VALUE, events);
    Assert.assertTrue(events.isEmpty());
}