Examples with StreamManager co.cask.cdap.test.StreamManager used on opensource projects

Example 21 with StreamManager

use of co.cask.cdap.test.StreamManager in project cdap by caskdata.

the class AuthorizationTest method testWorkerStreamAuth.

@Test
@Category(SlowTests.class)
public void testWorkerStreamAuth() throws Exception {
    createAuthNamespace();
    Authorizer authorizer = getAuthorizer();
    ApplicationManager appManager = deployApplication(AUTH_NAMESPACE, StreamAuthApp.class);
    // After deploy, change Alice from ALL to ADMIN on the namespace
    authorizer.revoke(AUTH_NAMESPACE, ALICE, EnumSet.allOf(Action.class));
    authorizer.grant(AUTH_NAMESPACE, ALICE, EnumSet.of(Action.ADMIN));
    WorkerManager workerManager = appManager.getWorkerManager(StreamAuthApp.WORKER);
    workerManager.start();
    workerManager.waitForRun(ProgramRunStatus.COMPLETED, 60, TimeUnit.SECONDS);
    StreamId streamId = AUTH_NAMESPACE.stream(StreamAuthApp.STREAM);
    StreamManager streamManager = getStreamManager(AUTH_NAMESPACE.stream(StreamAuthApp.STREAM));
    Assert.assertEquals(5, streamManager.getEvents(0, Long.MAX_VALUE, Integer.MAX_VALUE).size());
    // Now revoke write permission for Alice on that stream (revoke ALL and then grant everything other than WRITE)
    authorizer.revoke(streamId, ALICE, EnumSet.allOf(Action.class));
    authorizer.grant(streamId, ALICE, EnumSet.of(Action.READ, Action.ADMIN, Action.EXECUTE));
    workerManager.start();
    workerManager.waitForRuns(ProgramRunStatus.FAILED, 1, 60, TimeUnit.SECONDS);
    // Give permissions back so that we can fetch the stream events
    authorizer.grant(streamId, ALICE, EnumSet.allOf(Action.class));
    Assert.assertEquals(5, streamManager.getEvents(0, Long.MAX_VALUE, Integer.MAX_VALUE).size());
    appManager.delete();
    assertNoAccess(AUTH_NAMESPACE.app(StreamAuthApp.APP));
}

Example 22 with StreamManager

use of co.cask.cdap.test.StreamManager in project cdap by caskdata.

the class PurchaseAppTest method test.

@Test
public void test() throws Exception {
    // Deploy the PurchaseApp application
    ApplicationManager appManager = deployApplication(PurchaseApp.class);
    // Start PurchaseFlow
    FlowManager flowManager = appManager.getFlowManager("PurchaseFlow").start();
    // Send stream events to the "purchaseStream" Stream
    StreamManager streamManager = getStreamManager("purchaseStream");
    streamManager.send("bob bought 3 apples for $30");
    streamManager.send("joe bought 1 apple for $100");
    streamManager.send("joe bought 10 pineapples for $20");
    streamManager.send("cat bought 3 bottles for $12");
    streamManager.send("cat bought 2 pops for $14");
    try {
        // Wait for the last Flowlet processing 5 events, or at most 15 seconds
        RuntimeMetrics metrics = flowManager.getFlowletMetrics("collector");
        metrics.waitForProcessed(5, 15, TimeUnit.SECONDS);
    } finally {
        flowManager.stop();
    }
    ServiceManager userProfileServiceManager = getUserProfileServiceManager(appManager);
    // Add customer's profile information
    URL userProfileUrl = new URL(userProfileServiceManager.getServiceURL(15, TimeUnit.SECONDS), UserProfileServiceHandler.USER_ENDPOINT);
    HttpURLConnection userProfileConnection = (HttpURLConnection) userProfileUrl.openConnection();
    String userProfileJson = "{'id' : 'joe', 'firstName': 'joe', 'lastName':'bernard', 'categories': ['fruits']}";
    try {
        userProfileConnection.setDoOutput(true);
        userProfileConnection.setRequestMethod("POST");
        userProfileConnection.getOutputStream().write(userProfileJson.getBytes(Charsets.UTF_8));
        Assert.assertEquals(HttpURLConnection.HTTP_OK, userProfileConnection.getResponseCode());
    } finally {
        userProfileConnection.disconnect();
    }
    // Test service to retrieve customer's profile information
    userProfileUrl = new URL(userProfileServiceManager.getServiceURL(15, TimeUnit.SECONDS), UserProfileServiceHandler.USER_ENDPOINT + "/joe");
    userProfileConnection = (HttpURLConnection) userProfileUrl.openConnection();
    Assert.assertEquals(HttpURLConnection.HTTP_OK, userProfileConnection.getResponseCode());
    String customerJson;
    try {
        customerJson = new String(ByteStreams.toByteArray(userProfileConnection.getInputStream()), Charsets.UTF_8);
    } finally {
        userProfileConnection.disconnect();
    }
    UserProfile profileFromService = GSON.fromJson(customerJson, UserProfile.class);
    Assert.assertEquals(profileFromService.getFirstName(), "joe");
    Assert.assertEquals(profileFromService.getLastName(), "bernard");
    // Run PurchaseHistoryWorkflow which will process the data
    MapReduceManager mapReduceManager = appManager.getMapReduceManager(PurchaseHistoryBuilder.class.getSimpleName()).start();
    mapReduceManager.waitForRun(ProgramRunStatus.COMPLETED, 3, TimeUnit.MINUTES);
    // Start PurchaseHistoryService
    ServiceManager purchaseHistoryServiceManager = appManager.getServiceManager(PurchaseHistoryService.SERVICE_NAME).start();
    // Wait for service startup
    purchaseHistoryServiceManager.waitForStatus(true);
    // Test service to retrieve a customer's purchase history
    URL url = new URL(purchaseHistoryServiceManager.getServiceURL(15, TimeUnit.SECONDS), "history/joe");
    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
    Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
    String historyJson;
    try {
        historyJson = new String(ByteStreams.toByteArray(conn.getInputStream()), Charsets.UTF_8);
    } finally {
        conn.disconnect();
    }
    PurchaseHistory history = GSON.fromJson(historyJson, PurchaseHistory.class);
    Assert.assertEquals("joe", history.getCustomer());
    Assert.assertEquals(2, history.getPurchases().size());
    UserProfile profileFromPurchaseHistory = history.getUserProfile();
    Assert.assertEquals(profileFromPurchaseHistory.getFirstName(), "joe");
    Assert.assertEquals(profileFromPurchaseHistory.getLastName(), "bernard");
}

Example 23 with StreamManager

use of co.cask.cdap.test.StreamManager in project cdap by caskdata.

the class HelloWorldTest method test.

@Test
public void test() throws Exception {
    // Deploy the HelloWorld application
    ApplicationManager appManager = deployApplication(HelloWorld.class);
    // Start WhoFlow
    FlowManager flowManager = appManager.getFlowManager("WhoFlow").start();
    Assert.assertTrue(flowManager.isRunning());
    // Send stream events to the "who" Stream
    StreamManager streamManager = getStreamManager("who");
    streamManager.send("1");
    streamManager.send("2");
    streamManager.send("3");
    streamManager.send("4");
    streamManager.send("5");
    try {
        // Wait for the last Flowlet processing 5 events, or at most 5 seconds
        RuntimeMetrics metrics = flowManager.getFlowletMetrics("saver");
        metrics.waitForProcessed(5, 5, TimeUnit.SECONDS);
    } finally {
        flowManager.stop();
        Assert.assertFalse(flowManager.isRunning());
    }
    // Start Greeting service and use it
    ServiceManager serviceManager = appManager.getServiceManager(HelloWorld.Greeting.SERVICE_NAME).start();
    // Wait service startup
    serviceManager.waitForStatus(true);
    URL url = new URL(serviceManager.getServiceURL(), "greet");
    HttpURLConnection connection = (HttpURLConnection) url.openConnection();
    Assert.assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
    String response;
    try {
        response = new String(ByteStreams.toByteArray(connection.getInputStream()), Charsets.UTF_8);
    } finally {
        connection.disconnect();
    }
    Assert.assertEquals("Hello 5!", response);
}

Example 24 with StreamManager

use of co.cask.cdap.test.StreamManager in project cdap by caskdata.

the class ClicksAndViewsMapReduceTest method testClicksAndViews.

@Test
public void testClicksAndViews() throws Exception {
    ApplicationManager applicationManager = deployApplication(ClicksAndViews.class);
    // write each of the views to the VIEWS stream
    StreamManager viewsStream = getStreamManager(ClicksAndViews.VIEWS);
    for (String view : VIEWS) {
        viewsStream.send(view);
    }
    // send clicks for viewIds 1,2,5
    StreamManager clicksStream = getStreamManager(ClicksAndViews.CLICKS);
    for (Integer click : CLICKS) {
        clicksStream.send(createClick(click));
    }
    MapReduceManager mapReduceManager = applicationManager.getMapReduceManager(ClicksAndViewsMapReduce.NAME).start(ImmutableMap.of("output.partition.key.runtime", Integer.toString(OUTPUT_PARTITION_RUNTIME)));
    mapReduceManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES);
    List<String> joinedViews = new ArrayList<>();
    for (int i = 0; i < VIEWS.size(); i++) {
        joinedViews.add(createJoinedView(VIEWS.get(i), Collections.frequency(CLICKS, i)));
    }
    Set<String> expectedJoinedViews = ImmutableSet.copyOf(joinedViews);
    Assert.assertEquals(expectedJoinedViews, getDataFromFile());
    Assert.assertEquals(expectedJoinedViews, getDataFromExplore());
}

Example 25 with StreamManager

use of co.cask.cdap.test.StreamManager in project cdap by caskdata.

the class DataPipelineTest method testSinglePhaseWithSparkSink.

private void testSinglePhaseWithSparkSink() throws Exception {
    /*
     * source1 ---|
     *            |--> sparksink
     * source2 ---|
     */
    ETLBatchConfig etlConfig = ETLBatchConfig.builder("* * * * *").addStage(new ETLStage("source1", MockSource.getPlugin("messages1", SpamMessage.SCHEMA))).addStage(new ETLStage("source2", MockSource.getPlugin("messages2", SpamMessage.SCHEMA))).addStage(new ETLStage("customsink", new ETLPlugin(NaiveBayesTrainer.PLUGIN_NAME, SparkSink.PLUGIN_TYPE, ImmutableMap.of("fileSetName", "modelFileSet", "path", "output", "fieldToClassify", SpamMessage.TEXT_FIELD, "predictionField", SpamMessage.SPAM_PREDICTION_FIELD), null))).addConnection("source1", "customsink").addConnection("source2", "customsink").build();
    AppRequest<ETLBatchConfig> appRequest = new AppRequest<>(APP_ARTIFACT, etlConfig);
    ApplicationId appId = NamespaceId.DEFAULT.app("SparkSinkApp");
    ApplicationManager appManager = deployApplication(appId.toId(), appRequest);
    // set up five spam messages and five non-spam messages to be used for classification
    List<StructuredRecord> messagesToWrite = new ArrayList<>();
    messagesToWrite.add(new SpamMessage("buy our clothes", 1.0).toStructuredRecord());
    messagesToWrite.add(new SpamMessage("sell your used books to us", 1.0).toStructuredRecord());
    messagesToWrite.add(new SpamMessage("earn money for free", 1.0).toStructuredRecord());
    messagesToWrite.add(new SpamMessage("this is definitely not spam", 1.0).toStructuredRecord());
    messagesToWrite.add(new SpamMessage("you won the lottery", 1.0).toStructuredRecord());
    // write records to source1
    DataSetManager<Table> inputManager = getDataset(NamespaceId.DEFAULT.dataset("messages1"));
    MockSource.writeInput(inputManager, messagesToWrite);
    messagesToWrite.clear();
    messagesToWrite.add(new SpamMessage("how was your day", 0.0).toStructuredRecord());
    messagesToWrite.add(new SpamMessage("what are you up to", 0.0).toStructuredRecord());
    messagesToWrite.add(new SpamMessage("this is a genuine message", 0.0).toStructuredRecord());
    messagesToWrite.add(new SpamMessage("this is an even more genuine message", 0.0).toStructuredRecord());
    messagesToWrite.add(new SpamMessage("could you send me the report", 0.0).toStructuredRecord());
    // write records to source2
    inputManager = getDataset(NamespaceId.DEFAULT.dataset("messages2"));
    MockSource.writeInput(inputManager, messagesToWrite);
    // ingest in some messages to be classified
    StreamManager textsToClassify = getStreamManager(NaiveBayesTrainer.TEXTS_TO_CLASSIFY);
    textsToClassify.send("how are you doing today");
    textsToClassify.send("free money money");
    textsToClassify.send("what are you doing today");
    textsToClassify.send("genuine report");
    // manually trigger the pipeline
    WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME);
    workflowManager.start();
    workflowManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES);
    DataSetManager<KeyValueTable> classifiedTexts = getDataset(NaiveBayesTrainer.CLASSIFIED_TEXTS);
    Assert.assertEquals(0.0d, Bytes.toDouble(classifiedTexts.get().read("how are you doing today")), 0.01d);
    // only 'free money money' should be predicated as spam
    Assert.assertEquals(1.0d, Bytes.toDouble(classifiedTexts.get().read("free money money")), 0.01d);
    Assert.assertEquals(0.0d, Bytes.toDouble(classifiedTexts.get().read("what are you doing today")), 0.01d);
    Assert.assertEquals(0.0d, Bytes.toDouble(classifiedTexts.get().read("genuine report")), 0.01d);
    validateMetric(5, appId, "source1.records.out");
    validateMetric(5, appId, "source2.records.out");
    validateMetric(10, appId, "customsink.records.in");
}