Examples with Stream co.cask.cdap.api.data.stream.Stream used on opensource projects

Example 26 with Stream

use of co.cask.cdap.api.data.stream.Stream in project cdap by caskdata.

the class PurchaseApp method configure.

@Override
public void configure() {
    setName(APP_NAME);
    setDescription("Purchase history application");
    // Ingest data into the Application via a Stream
    addStream(new Stream("purchaseStream"));
    // Store processed data in a Dataset
    createDataset("frequentCustomers", KeyValueTable.class, DatasetProperties.builder().setDescription("Store frequent customers").build());
    // Store user profiles in a Dataset
    createDataset("userProfiles", KeyValueTable.class, DatasetProperties.builder().setDescription("Store user profiles").build());
    // Process events in realtime using a Flow
    addFlow(new PurchaseFlow());
    // Specify a MapReduce to run on the acquired data
    addMapReduce(new PurchaseHistoryBuilder());
    // Run a Workflow that uses the MapReduce to run on the acquired data
    addWorkflow(new PurchaseHistoryWorkflow());
    // Retrieve the processed data using a Service
    addService(new PurchaseHistoryService());
    // Store and retrieve user profile data using a Service
    addService(UserProfileServiceHandler.SERVICE_NAME, new UserProfileServiceHandler());
    // Provide a Service to Application components
    addService(new CatalogLookupService());
    // Schedule the workflow
    schedule(buildSchedule("DailySchedule", ProgramType.WORKFLOW, "PurchaseHistoryWorkflow").withConcurrency(1).triggerByTime("0 4 * * *"));
    // Schedule the workflow based on the data coming in the purchaseStream stream
    scheduleWorkflow(Schedules.builder("DataSchedule").setDescription("Schedule execution when 1 MB or more of data is ingested in the purchaseStream").setMaxConcurrentRuns(1).createDataSchedule(Schedules.Source.STREAM, "purchaseStream", 1), "PurchaseHistoryWorkflow");
    createDataset("history", PurchaseHistoryStore.class, PurchaseHistoryStore.properties("History dataset"));
    try {
        createDataset("purchases", ObjectMappedTable.class, ObjectMappedTableProperties.builder().setType(Purchase.class).setDescription("Store purchases").build());
    } catch (UnsupportedTypeException e) {
        // because PurchaseHistory and Purchase are actual classes.
        throw new RuntimeException(e);
    }
}

Example 27 with Stream

use of co.cask.cdap.api.data.stream.Stream in project cdap by caskdata.

the class ClicksAndViews method configure.

@Override
public void configure() {
    setName(NAME);
    setDescription("Example Clicks and Views Application");
    // Process the "clicks" and "views" streams using MapReduce
    addMapReduce(new ClicksAndViewsMapReduce());
    addStream(new Stream(VIEWS, "Stores the views that happen for each ad"));
    addStream(new Stream(CLICKS, "Stores the clicks that happen for each view"));
    createDataset(JOINED, PartitionedFileSet.class, PartitionedFileSetProperties.builder().setPartitioning(Partitioning.builder().addLongField("runtime").build()).setOutputFormat(TextOutputFormat.class).setEnableExploreOnCreate(true).setExploreFormat("text").setExploreFormatProperty("delimiter", "\t").setExploreSchema("viewId BIGINT, requestBeginTime BIGINT, adId BIGINT, referrer STRING, " + "userCookie STRING, ip STRING, numClicks INT").setDescription("Stores the joined views, which additionally keeps track of how many clicks each view had").build());
}

Example 28 with Stream

use of co.cask.cdap.api.data.stream.Stream in project cdap by caskdata.

the class HighPassFilterApp method configure.

@Override
public void configure() {
    setName(NAME);
    setDescription("Application for filtering numbers. Test runtimeargs.");
    addStream(new Stream("inputvalue"));
    createDataset("counter", KeyValueTable.class);
    addFlow(new FilterFlow());
    addService(new BasicService("Count", new CountHandler()));
}

Example 29 with Stream

use of co.cask.cdap.api.data.stream.Stream in project cdap by caskdata.

the class JoinMultiStreamApp method configure.

@Override
public void configure() {
    setName("JoinMulti");
    setDescription("JoinMulti");
    addStream(new Stream("s1"));
    addStream(new Stream("s2"));
    addStream(new Stream("s3"));
    createDataset("mytable", KeyValueTable.class);
    addFlow(new JoinMultiFlow());
    addService(new BasicService("QueryService", new QueryHandler()));
}

Example 30 with Stream

use of co.cask.cdap.api.data.stream.Stream in project cdap by caskdata.

the class TestFlowStreamIntegrationAcrossNSApp method configure.

@Override
public void configure() {
    setName("TestFlowStreamIntegrationAcrossNSApp");
    setDescription("Application for testing batch stream from another namespace");
    addStream(new Stream("s1"));
    addFlow(new StreamTestFlow());
}