Examples with StreamManager co.cask.cdap.test.StreamManager used on opensource projects

Example 6 with StreamManager

use of co.cask.cdap.test.StreamManager in project cdap by caskdata.

the class SparkPageRankAppTest method test.

@Test
public void test() throws Exception {
    // Deploy the SparkPageRankApp
    ApplicationManager appManager = deployApplication(SparkPageRankApp.class);
    // Send a stream events to the Stream
    StreamManager streamManager = getStreamManager(SparkPageRankApp.BACKLINK_URL_STREAM);
    streamManager.send(Joiner.on(" ").join(URL_1, URL_2));
    streamManager.send(Joiner.on(" ").join(URL_1, URL_3));
    streamManager.send(Joiner.on(" ").join(URL_2, URL_1));
    streamManager.send(Joiner.on(" ").join(URL_3, URL_1));
    // Start service
    ServiceManager serviceManager = appManager.getServiceManager(SparkPageRankApp.SERVICE_HANDLERS).start();
    // Wait for service to start since the Spark program needs it
    serviceManager.waitForStatus(true);
    // Start the SparkPageRankProgram
    SparkManager sparkManager = appManager.getSparkManager(SparkPageRankApp.PageRankSpark.class.getSimpleName()).start();
    sparkManager.waitForRun(ProgramRunStatus.COMPLETED, 60, TimeUnit.SECONDS);
    // Run RanksCounter which will count the number of pages for a pr
    MapReduceManager mapReduceManager = appManager.getMapReduceManager(SparkPageRankApp.RanksCounter.class.getSimpleName()).start();
    mapReduceManager.waitForRun(ProgramRunStatus.COMPLETED, 3, TimeUnit.MINUTES);
    //Query for rank
    URL url = new URL(serviceManager.getServiceURL(15, TimeUnit.SECONDS), SparkPageRankApp.SparkPageRankServiceHandler.RANKS_PATH);
    HttpRequest request = HttpRequest.post(url).withBody(("{\"" + SparkPageRankApp.SparkPageRankServiceHandler.URL_KEY + "\":\"" + URL_1 + "\"}")).build();
    HttpResponse response = HttpRequests.execute(request);
    Assert.assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode());
    Assert.assertEquals(RANK, response.getResponseBodyAsString());
    // Request total pages for a page rank and verify it
    url = new URL(serviceManager.getServiceURL(15, TimeUnit.SECONDS), SparkPageRankApp.SparkPageRankServiceHandler.TOTAL_PAGES_PATH + "/" + RANK);
    response = HttpRequests.execute(HttpRequest.get(url).build());
    Assert.assertEquals(TOTAL_PAGES, response.getResponseBodyAsString());
}

Example 7 with StreamManager

use of co.cask.cdap.test.StreamManager in project cdap by caskdata.

the class SpamClassifierTest method ingestTrainingData.

private void ingestTrainingData() throws IOException {
    StreamManager streamManager = getStreamManager(SpamClassifier.STREAM);
    try (BufferedReader reader = new BufferedReader(new InputStreamReader(getClass().getResourceAsStream("/trainingData.txt"), "UTF-8"))) {
        String line = reader.readLine();
        while (line != null) {
            streamManager.send(line);
            line = reader.readLine();
        }
    }
}

Example 8 with StreamManager

use of co.cask.cdap.test.StreamManager in project cdap by caskdata.

the class WikipediaPipelineAppTest method createTestData.

private void createTestData() throws Exception {
    StreamManager likesStreamManager = getStreamManager(WikipediaPipelineApp.PAGE_TITLES_STREAM);
    String like1 = GSON.toJson(new StreamToDataset.PageTitleToDatasetMapper.Page("Metallica", "103636093053996", "012-02-11T22:41:49+0000"));
    String like2 = GSON.toJson(new StreamToDataset.PageTitleToDatasetMapper.Page("grunge", "58417452552", "2012-02-07T21:29:53+0000"));
    likesStreamManager.send(like1);
    likesStreamManager.send(like2);
    StreamManager rawWikipediaStreamManager = getStreamManager(WikipediaPipelineApp.RAW_WIKIPEDIA_STREAM);
    String data1 = "{\"batchcomplete\":\"\",\"query\":{\"normalized\":[{\"from\":\"metallica\",\"to\":\"Metallica\"}]" + ",\"pages\":{\"18787\":{\"pageid\":18787,\"ns\":0,\"title\":\"Metallica\",\"revisions\":[{\"contentformat\":" + "\"text/x-wiki\",\"contentmodel\":\"wikitext\",\"*\":\"{{Other uses}}{{pp-semi|small=yes}}{{pp-move-indef|" + "small=yes}}{{Use mdy dates|date=April 2013}}{{Infobox musical artist|name = Metallica|image = Metallica at " + "The O2 Arena London 2008.jpg|caption = Metallica in [[London]] in 2008. From left to right: [[Kirk Hammett]], " + "[[Lars Ulrich]], [[James Hetfield]] and [[Robert Trujillo]]\"}]}}}}";
    String data2 = "{\"batchcomplete\":\"\",\"query\":{\"pages\":{\"51580\":{\"pageid\":51580,\"ns\":0," + "\"title\":\"Grunge\",\"revisions\":[{\"contentformat\":\"text/x-wiki\",\"contentmodel\":\"wikitext\"," + "\"*\":\"{{About|the music genre}}{{Infobox music genre| name  = Grunge| bgcolor = crimson| color = white| " + "stylistic_origins = {{nowrap|[[Alternative rock]], [[hardcore punk]],}} [[Heavy metal music|heavy metal]], " + "[[punk rock]], [[hard rock]], [[noise rock]]| cultural_origins  = Mid-1980s, [[Seattle|Seattle, Washington]], " + "[[United States]]| instruments = [[Electric guitar]], [[bass guitar]], [[Drum kit|drums]], " + "[[Singing|vocals]]| derivatives = [[Post-grunge]], [[nu metal]]| subgenrelist = | subgenres = | fusiongenres" + "      = | regional_scenes   = [[Music of Washington (state)|Washington state]]| other_topics      = * " + "[[Alternative metal]]* [[Generation X]]* [[Grunge speak|grunge speak hoax]]* [[timeline of alternative " + "rock]]}}'''Grunge''' (sometimes referred to as the '''Seattle sound''') is a subgenre of [[alternative rock]]" + " that emerged during the mid-1980s in the American state of [[Washington (state)|Washington]], particularly " + "in [[Seattle]].  The early grunge movement revolved around Seattle's [[independent record label]] " + "[[Sub Pop]], but by the early 1990s its popularity had spread, with grunge acts in California and other " + "parts of the U.S. building strong followings and signing major record deals.Grunge became commercially " + "successful in the first half of the 1990s, due mainly to the release of [[Nirvana (band)|Nirvana]]'s " + "''[[Nevermind]]'', [[Pearl Jam]]'s ''[[Ten (Pearl Jam album)|Ten]]'', [[Soundgarden]]'s " + "''[[Badmotorfinger]]'', [[Alice in Chains]]' ''[[Dirt (Alice in Chains album)|Dirt]]'', and " + "[[Stone Temple Pilots]]' ''[[Core (Stone Temple Pilots album)|Core]]''.\"}]}}}}";
    rawWikipediaStreamManager.send(data1);
    rawWikipediaStreamManager.send(data2);
    waitForStreamToBePopulated(likesStreamManager, 2);
    waitForStreamToBePopulated(rawWikipediaStreamManager, 2);
}

Example 9 with StreamManager

use of co.cask.cdap.test.StreamManager in project cdap by caskdata.

the class UserProfilesTest method testUserProfiles.

@Test
public void testUserProfiles() throws Exception {
    // deploy the app
    ApplicationManager applicationManager = deployApplication(UserProfiles.class);
    // run the service and the flow
    FlowManager flowManager = applicationManager.getFlowManager("ActivityFlow").start();
    ServiceManager serviceManager = applicationManager.getServiceManager("UserProfileService").start();
    serviceManager.waitForStatus(true);
    URL serviceURL = serviceManager.getServiceURL();
    // create a user through the service
    String userJson = new Gson().toJson(ImmutableMap.of("id", "1234", "name", "joe", "email", "joe@bla.ck"));
    HttpURLConnection connection = (HttpURLConnection) new URL(serviceURL, "profiles/1234").openConnection();
    try {
        connection.setDoOutput(true);
        connection.setRequestMethod("PUT");
        connection.getOutputStream().write(userJson.getBytes(Charsets.UTF_8));
        Assert.assertEquals(HttpURLConnection.HTTP_CREATED, connection.getResponseCode());
    } finally {
        connection.disconnect();
    }
    // read the user through the dataset
    DataSetManager<Table> tableManager = getDataset("profiles");
    Row row = tableManager.get().get(new Get("1234"));
    Assert.assertEquals("1234", row.getString("id"));
    Assert.assertEquals("joe", row.getString("name"));
    Assert.assertEquals("joe@bla.ck", row.getString("email"));
    Assert.assertNull(row.getLong("login"));
    Assert.assertNull(row.getLong("active"));
    // update email address through service
    connection = (HttpURLConnection) new URL(serviceURL, "profiles/1234/email").openConnection();
    try {
        connection.setDoOutput(true);
        connection.setRequestMethod("PUT");
        connection.getOutputStream().write("joe@black.com".getBytes(Charsets.UTF_8));
        Assert.assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
    } finally {
        connection.disconnect();
    }
    // verify the updated email address
    tableManager.flush();
    row = tableManager.get().get(new Get("1234"));
    Assert.assertEquals("1234", row.getString("id"));
    Assert.assertEquals("joe", row.getString("name"));
    Assert.assertEquals("joe@black.com", row.getString("email"));
    Assert.assertNull(row.getLong("login"));
    Assert.assertNull(row.getLong("active"));
    // send a login event
    long loginTime = System.currentTimeMillis();
    connection = (HttpURLConnection) new URL(serviceURL, "profiles/1234/lastLogin").openConnection();
    try {
        connection.setDoOutput(true);
        connection.setRequestMethod("PUT");
        connection.getOutputStream().write(Long.toString(loginTime).getBytes(Charsets.UTF_8));
        Assert.assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
    } finally {
        connection.disconnect();
    }
    // verify the login time through the dataset
    tableManager.flush();
    row = tableManager.get().get(new Get("1234"));
    Assert.assertEquals("1234", row.getString("id"));
    Assert.assertEquals("joe", row.getString("name"));
    Assert.assertEquals("joe@black.com", row.getString("email"));
    Assert.assertEquals(new Long(loginTime), row.getLong("login"));
    Assert.assertNull(row.getLong("active"));
    // send an event to the stream
    long activeTime = System.currentTimeMillis();
    StreamManager streamManager = getStreamManager("events");
    streamManager.send(new Gson().toJson(new Event(activeTime, "1234", "/some/path")));
    try {
        // Wait for the last Flowlet processing 1 events, or at most 5 seconds
        RuntimeMetrics metrics = flowManager.getFlowletMetrics("updater");
        metrics.waitForProcessed(1, 5, TimeUnit.SECONDS);
    } finally {
        flowManager.stop();
        Assert.assertFalse(flowManager.isRunning());
    }
    // verify the last active time for the user
    tableManager.flush();
    row = tableManager.get().get(new Get("1234"));
    Assert.assertEquals("1234", row.getString("id"));
    Assert.assertEquals("joe", row.getString("name"));
    Assert.assertEquals("joe@black.com", row.getString("email"));
    Assert.assertEquals(new Long(loginTime), row.getLong("login"));
    Assert.assertEquals(new Long(activeTime), row.getLong("active"));
    // delete the user
    connection = (HttpURLConnection) new URL(serviceURL, "profiles/1234").openConnection();
    try {
        connection.setRequestMethod("DELETE");
        Assert.assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
    } finally {
        connection.disconnect();
    }
    // verify the user is gone
    tableManager.flush();
    row = tableManager.get().get(new Get("1234"));
    Assert.assertTrue(row.isEmpty());
    // stop the service and the flow
    serviceManager.stop();
}

Example 10 with StreamManager

use of co.cask.cdap.test.StreamManager in project cdap by caskdata.

the class AuthorizationTest method testFlowStreamAuth.

@Test
@Category(SlowTests.class)
public void testFlowStreamAuth() throws Exception {
    createAuthNamespace();
    Authorizer authorizer = getAuthorizer();
    ApplicationManager appManager = deployApplication(AUTH_NAMESPACE, StreamAuthApp.class);
    // After deploy, change Alice from ALL to ADMIN on the namespace
    authorizer.revoke(AUTH_NAMESPACE, ALICE, EnumSet.allOf(Action.class));
    authorizer.grant(AUTH_NAMESPACE, ALICE, EnumSet.of(Action.ADMIN));
    final FlowManager flowManager = appManager.getFlowManager(StreamAuthApp.FLOW);
    StreamId streamId = AUTH_NAMESPACE.stream(StreamAuthApp.STREAM);
    StreamManager streamManager = getStreamManager(AUTH_NAMESPACE.stream(StreamAuthApp.STREAM));
    StreamManager streamManager2 = getStreamManager(AUTH_NAMESPACE.stream(StreamAuthApp.STREAM2));
    streamManager.send("Auth");
    flowManager.start();
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            DataSetManager<KeyValueTable> kvTable = getDataset(AUTH_NAMESPACE.dataset(StreamAuthApp.KVTABLE));
            return kvTable.get().read("Auth") != null;
        }
    }, 5, TimeUnit.SECONDS);
    flowManager.stop();
    flowManager.waitForRun(ProgramRunStatus.KILLED, 60, TimeUnit.SECONDS);
    // Now revoke read permission for Alice on that stream (revoke ALL and then grant everything other than READ)
    authorizer.revoke(streamId, ALICE, EnumSet.allOf(Action.class));
    authorizer.grant(streamId, ALICE, EnumSet.of(Action.WRITE, Action.ADMIN, Action.EXECUTE));
    streamManager.send("Security");
    streamManager2.send("Safety");
    try {
        flowManager.start();
    } catch (RuntimeException e) {
        Assert.assertTrue(e.getCause() instanceof UnauthorizedException);
    }
    authorizer.grant(streamId, ALICE, ImmutableSet.of(Action.READ));
    flowManager.start();
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            DataSetManager<KeyValueTable> kvTable = getDataset(AUTH_NAMESPACE.dataset(StreamAuthApp.KVTABLE));
            return kvTable.get().read("Security") != null;
        }
    }, 5, TimeUnit.SECONDS);
    authorizer.revoke(streamId, ALICE, ImmutableSet.of(Action.READ));
    TimeUnit.MILLISECONDS.sleep(10);
    flowManager.stop();
    flowManager.waitForRuns(ProgramRunStatus.KILLED, 2, 5, TimeUnit.SECONDS);
    appManager.delete();
}