Examples with StreamClient co.cask.cdap.client.StreamClient used on opensource projects

Example 1 with StreamClient

use of co.cask.cdap.client.StreamClient in project cdap by caskdata.

the class AppFabricTestBase method initializeAndStartServices.

protected static void initializeAndStartServices(CConfiguration cConf, @Nullable SConfiguration sConf) throws Exception {
    injector = Guice.createInjector(Modules.override(new AppFabricTestModule(cConf, sConf)).with(new AbstractModule() {

        @Override
        protected void configure() {
            // needed because we set Kerberos to true in DefaultNamespaceAdminTest
            bind(UGIProvider.class).to(CurrentUGIProvider.class);
        }
    }));
    messagingService = injector.getInstance(MessagingService.class);
    if (messagingService instanceof Service) {
        ((Service) messagingService).startAndWait();
    }
    txManager = injector.getInstance(TransactionManager.class);
    txManager.startAndWait();
    dsOpService = injector.getInstance(DatasetOpExecutor.class);
    dsOpService.startAndWait();
    remoteSysOpService = injector.getInstance(RemoteSystemOperationsService.class);
    remoteSysOpService.startAndWait();
    datasetService = injector.getInstance(DatasetService.class);
    datasetService.startAndWait();
    appFabricServer = injector.getInstance(AppFabricServer.class);
    appFabricServer.startAndWait();
    DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class);
    ServiceDiscovered appFabricHttpDiscovered = discoveryClient.discover(Constants.Service.APP_FABRIC_HTTP);
    EndpointStrategy endpointStrategy = new RandomEndpointStrategy(appFabricHttpDiscovered);
    port = endpointStrategy.pick(1, TimeUnit.SECONDS).getSocketAddress().getPort();
    txClient = injector.getInstance(TransactionSystemClient.class);
    metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
    metricsCollectionService.startAndWait();
    metricsService = injector.getInstance(MetricsQueryService.class);
    metricsService.startAndWait();
    streamService = injector.getInstance(StreamService.class);
    streamService.startAndWait();
    serviceStore = injector.getInstance(ServiceStore.class);
    serviceStore.startAndWait();
    metadataService = injector.getInstance(MetadataService.class);
    metadataService.startAndWait();
    locationFactory = getInjector().getInstance(LocationFactory.class);
    streamClient = new StreamClient(getClientConfig(discoveryClient, Constants.Service.STREAMS));
    datasetClient = new DatasetClient(getClientConfig(discoveryClient, Constants.Service.DATASET_MANAGER));
    createNamespaces();
}

Example 2 with StreamClient

use of co.cask.cdap.client.StreamClient in project cdap-ingest by caskdata.

the class PollingListenerImpl method getStreamWriterForPipe.

/**
 * create StreamWriter for pipe
 *
 * @param pipeConf the pipe configuration
 * @return the pipe's streamWriter
 * @throws java.io.IOException streamWriter creation failed
 */
private StreamWriter getStreamWriterForPipe(PipeConfiguration pipeConf) throws IOException {
    StreamClient client = pipeConf.getSinkConfiguration().getStreamClient();
    String streamName = pipeConf.getSinkConfiguration().getStreamName();
    try {
        client.create(streamName);
        StreamWriter writer = client.createWriter(streamName);
        return writer;
    } catch (IOException e) {
        throw new IOException(String.format("Cannot create/get client stream by name: %s: %s", streamName, e));
    }
}

Example 3 with StreamClient

use of co.cask.cdap.client.StreamClient in project cdap-ingest by caskdata.

the class CdapFlumeIT method createStream.

private void createStream() throws Exception {
    RestStreamClient.Builder builder = RestStreamClient.builder(streamReader.getCdapHost(), streamReader.getCdapPort()).ssl(streamReader.getSsl());
    if (streamReader.getAuthClientPropertiesPath() != null) {
        AuthenticationClient authClient = streamReader.createAuthClient();
        builder.authClient(authClient);
    }
    StreamClient streamClient = builder.build();
    streamClient.create(streamName);
}

Example 4 with StreamClient

use of co.cask.cdap.client.StreamClient in project cdap by caskdata.

the class GenerateClientUsageExample method streamClient.

public void streamClient() throws Exception {
    // Construct the client used to interact with CDAP
    StreamClient streamClient = new StreamClient(clientConfig);
    // Fetch the stream list
    List streams = streamClient.list(NamespaceId.DEFAULT);
    // Create a stream, using the Purchase example
    StreamId streamId = NamespaceId.DEFAULT.stream("purchases");
    streamClient.create(streamId);
    // Fetch a stream's properties
    StreamProperties config = streamClient.getConfig(streamId);
    // Send events to a stream
    streamClient.sendEvent(streamId, "Tom bought 5 apples for $10");
    // Read all events from a stream (results in events)
    List<StreamEvent> events = Lists.newArrayList();
    streamClient.getEvents(streamId, 0, Long.MAX_VALUE, Integer.MAX_VALUE, events);
    // Read first 5 events from a stream (results in events)
    events = Lists.newArrayList();
    streamClient.getEvents(streamId, 0, Long.MAX_VALUE, 5, events);
    // Read 2nd and 3rd events from a stream, after first calling getEvents
    long startTime = events.get(1).getTimestamp();
    long endTime = events.get(2).getTimestamp() + 1;
    events.clear();
    streamClient.getEvents(streamId, startTime, endTime, Integer.MAX_VALUE, events);
    // Write asynchronously to a stream
    streamId = NamespaceId.DEFAULT.stream("testAsync");
    events = Lists.newArrayList();
    streamClient.create(streamId);
    // Send 10 async writes
    int msgCount = 10;
    for (int i = 0; i < msgCount; i++) {
        streamClient.asyncSendEvent(streamId, "Testing " + i);
    }
    // Read them back; need to read it multiple times as the writes happen asynchronously
    while (events.size() != msgCount) {
        events.clear();
        streamClient.getEvents(streamId, 0, Long.MAX_VALUE, msgCount, events);
    }
    // Check that there are no more events
    events.clear();
    while (events.isEmpty()) {
        events.clear();
        streamClient.getEvents(streamId, 0, Long.MAX_VALUE, msgCount, events);
    }
// End write asynchronously
}

Example 5 with StreamClient

use of co.cask.cdap.client.StreamClient in project cdap-ingest by caskdata.

the class FileTailerIT method mockMetricsProcessor.

private void mockMetricsProcessor(PipeManager manager) throws IOException, NoSuchMethodException, InvocationTargetException, IllegalAccessException, NoSuchFieldException {
    List<Pipe> pipeList = new ArrayList<Pipe>();
    StreamClient client = null;
    StreamWriter writer = null;
    try {
        Method method1 = manager.getClass().getDeclaredMethod("getPipeConfigs");
        method1.setAccessible(true);
        List<PipeConfiguration> pipeConfList = (List<PipeConfiguration>) method1.invoke(manager);
        for (PipeConfiguration pipeConf : pipeConfList) {
            FileTailerQueue queue = new FileTailerQueue(pipeConf.getQueueSize());
            client = pipeConf.getSinkConfiguration().getStreamClient();
            String streamName = pipeConf.getSinkConfiguration().getStreamName();
            Method method2 = manager.getClass().getDeclaredMethod("getStreamWriterForPipe", StreamClient.class, String.class);
            method2.setAccessible(true);
            writer = (StreamWriter) method2.invoke(manager, client, streamName);
            FileTailerStateProcessor stateProcessor = new FileTailerStateProcessorImpl(pipeConf.getDaemonDir(), pipeConf.getStateFile());
            FileTailerMetricsProcessor metricsProcessor = new FileTailerMetricsProcessor(pipeConf.getDaemonDir(), pipeConf.getStatisticsFile(), pipeConf.getStatisticsSleepInterval(), pipeConf.getPipeName(), pipeConf.getSourceConfiguration().getFileName()) {

                @Override
                public void onReadEventMetric(int eventSize) {
                    super.onReadEventMetric(eventSize);
                    read.incrementAndGet();
                }

                @Override
                public void onIngestEventMetric(int latency) {
                    super.onIngestEventMetric(latency);
                    ingest.incrementAndGet();
                }
            };
            pipeList.add(new Pipe(new LogTailer(pipeConf, queue, stateProcessor, metricsProcessor, null), new FileTailerSink(queue, writer, SinkStrategy.LOADBALANCE, stateProcessor, metricsProcessor, null, pipeConf.getSinkConfiguration().getPackSize()), metricsProcessor));
            client = null;
            writer = null;
        }
        Field field = manager.getClass().getDeclaredField("serviceManager");
        field.setAccessible(true);
        Field modifiersField = Field.class.getDeclaredField("modifiers");
        modifiersField.setAccessible(true);
        modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
        field.set(manager, new ServiceManager(pipeList));
    } finally {
        if (client != null) {
            client.close();
        }
        if (writer != null) {
            writer.close();
        }
    }
}