Search in sources :

Example 1 with FlowManager

use of co.cask.cdap.test.FlowManager in project cdap by caskdata.

the class WordCountTest method testWordCount.

@Test
public void testWordCount() throws Exception {
    WordCount.WordCountConfig config = new WordCount.WordCountConfig("words", "stats", "counts", "unique", "assoc");
    // Deploy the Application
    ApplicationManager appManager = deployApplication(WordCount.class, config);
    // validate that the wordCount table is empty, and that it has no entry for "world"
    DataSetManager<KeyValueTable> datasetManager = getDataset(config.getWordCountTable());
    KeyValueTable wordCounts = datasetManager.get();
    Assert.assertNull(wordCounts.read("world"));
    // Start the Flow
    FlowManager flowManager = appManager.getFlowManager("WordCounter").start();
    // Send a few events to the stream
    StreamManager streamManager = getStreamManager("words");
    streamManager.send("hello world");
    streamManager.send("a wonderful world");
    streamManager.send("the world says hello");
    // Wait for the events to be processed, or at most 5 seconds
    RuntimeMetrics metrics = flowManager.getFlowletMetrics("associator");
    metrics.waitForProcessed(3, 5, TimeUnit.SECONDS);
    // start a new transaction so that the "wordCounts" dataset sees changes made by the flow
    datasetManager.flush();
    Assert.assertEquals(3L, Bytes.toLong(wordCounts.read("world")));
    // Start RetrieveCounts service
    ServiceManager serviceManager = appManager.getServiceManager(RetrieveCounts.SERVICE_NAME).start();
    // Wait service startup
    serviceManager.waitForStatus(true);
    // First verify global statistics
    String response = requestService(new URL(serviceManager.getServiceURL(15, TimeUnit.SECONDS), "stats"));
    Map<String, String> map = new Gson().fromJson(response, STRING_MAP_TYPE);
    Assert.assertEquals("9", map.get("totalWords"));
    Assert.assertEquals("6", map.get("uniqueWords"));
    Assert.assertEquals(((double) 42) / 9, Double.valueOf(map.get("averageLength")), 0.001);
    // Now verify statistics for a specific word
    response = requestService(new URL(serviceManager.getServiceURL(15, TimeUnit.SECONDS), "count/world"));
    Map<String, Object> omap = new Gson().fromJson(response, OBJECT_MAP_TYPE);
    Assert.assertEquals("world", omap.get("word"));
    Assert.assertEquals(3.0, omap.get("count"));
    // The associations are a map within the map
    @SuppressWarnings("unchecked") Map<String, Double> assocs = (Map<String, Double>) omap.get("assocs");
    Assert.assertEquals(2.0, assocs.get("hello"), 0.000001);
    Assert.assertTrue(assocs.containsKey("hello"));
}
Also used : FlowManager(co.cask.cdap.test.FlowManager) ApplicationManager(co.cask.cdap.test.ApplicationManager) RuntimeMetrics(co.cask.cdap.api.metrics.RuntimeMetrics) Gson(com.google.gson.Gson) URL(java.net.URL) KeyValueTable(co.cask.cdap.api.dataset.lib.KeyValueTable) StreamManager(co.cask.cdap.test.StreamManager) ServiceManager(co.cask.cdap.test.ServiceManager) Map(java.util.Map) Test(org.junit.Test)

Example 2 with FlowManager

use of co.cask.cdap.test.FlowManager in project cdap by caskdata.

the class HelloWorldTest method test.

@Test
public void test() throws Exception {
    // Deploy the HelloWorld application
    ApplicationManager appManager = deployApplication(HelloWorld.class);
    // Start WhoFlow
    FlowManager flowManager = appManager.getFlowManager("WhoFlow").start();
    Assert.assertTrue(flowManager.isRunning());
    // Send stream events to the "who" Stream
    StreamManager streamManager = getStreamManager("who");
    streamManager.send("1");
    streamManager.send("2");
    streamManager.send("3");
    streamManager.send("4");
    streamManager.send("5");
    try {
        // Wait for the last Flowlet processing 5 events, or at most 5 seconds
        RuntimeMetrics metrics = flowManager.getFlowletMetrics("saver");
        metrics.waitForProcessed(5, 5, TimeUnit.SECONDS);
    } finally {
        flowManager.stop();
        Assert.assertFalse(flowManager.isRunning());
    }
    // Start Greeting service and use it
    ServiceManager serviceManager = appManager.getServiceManager(HelloWorld.Greeting.SERVICE_NAME).start();
    // Wait service startup
    serviceManager.waitForStatus(true);
    URL url = new URL(serviceManager.getServiceURL(), "greet");
    HttpURLConnection connection = (HttpURLConnection) url.openConnection();
    Assert.assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
    String response;
    try {
        response = new String(ByteStreams.toByteArray(connection.getInputStream()), Charsets.UTF_8);
    } finally {
        connection.disconnect();
    }
    Assert.assertEquals("Hello 5!", response);
}
Also used : FlowManager(co.cask.cdap.test.FlowManager) ApplicationManager(co.cask.cdap.test.ApplicationManager) HttpURLConnection(java.net.HttpURLConnection) StreamManager(co.cask.cdap.test.StreamManager) RuntimeMetrics(co.cask.cdap.api.metrics.RuntimeMetrics) ServiceManager(co.cask.cdap.test.ServiceManager) URL(java.net.URL) Test(org.junit.Test)

Example 3 with FlowManager

use of co.cask.cdap.test.FlowManager in project cdap by caskdata.

the class UserProfilesTest method testUserProfiles.

@Test
public void testUserProfiles() throws Exception {
    // deploy the app
    ApplicationManager applicationManager = deployApplication(UserProfiles.class);
    // run the service and the flow
    FlowManager flowManager = applicationManager.getFlowManager("ActivityFlow").start();
    ServiceManager serviceManager = applicationManager.getServiceManager("UserProfileService").start();
    serviceManager.waitForStatus(true);
    URL serviceURL = serviceManager.getServiceURL();
    // create a user through the service
    String userJson = new Gson().toJson(ImmutableMap.of("id", "1234", "name", "joe", "email", "joe@bla.ck"));
    HttpURLConnection connection = (HttpURLConnection) new URL(serviceURL, "profiles/1234").openConnection();
    try {
        connection.setDoOutput(true);
        connection.setRequestMethod("PUT");
        connection.getOutputStream().write(userJson.getBytes(Charsets.UTF_8));
        Assert.assertEquals(HttpURLConnection.HTTP_CREATED, connection.getResponseCode());
    } finally {
        connection.disconnect();
    }
    // read the user through the dataset
    DataSetManager<Table> tableManager = getDataset("profiles");
    Row row = tableManager.get().get(new Get("1234"));
    Assert.assertEquals("1234", row.getString("id"));
    Assert.assertEquals("joe", row.getString("name"));
    Assert.assertEquals("joe@bla.ck", row.getString("email"));
    Assert.assertNull(row.getLong("login"));
    Assert.assertNull(row.getLong("active"));
    // update email address through service
    connection = (HttpURLConnection) new URL(serviceURL, "profiles/1234/email").openConnection();
    try {
        connection.setDoOutput(true);
        connection.setRequestMethod("PUT");
        connection.getOutputStream().write("joe@black.com".getBytes(Charsets.UTF_8));
        Assert.assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
    } finally {
        connection.disconnect();
    }
    // verify the updated email address
    tableManager.flush();
    row = tableManager.get().get(new Get("1234"));
    Assert.assertEquals("1234", row.getString("id"));
    Assert.assertEquals("joe", row.getString("name"));
    Assert.assertEquals("joe@black.com", row.getString("email"));
    Assert.assertNull(row.getLong("login"));
    Assert.assertNull(row.getLong("active"));
    // send a login event
    long loginTime = System.currentTimeMillis();
    connection = (HttpURLConnection) new URL(serviceURL, "profiles/1234/lastLogin").openConnection();
    try {
        connection.setDoOutput(true);
        connection.setRequestMethod("PUT");
        connection.getOutputStream().write(Long.toString(loginTime).getBytes(Charsets.UTF_8));
        Assert.assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
    } finally {
        connection.disconnect();
    }
    // verify the login time through the dataset
    tableManager.flush();
    row = tableManager.get().get(new Get("1234"));
    Assert.assertEquals("1234", row.getString("id"));
    Assert.assertEquals("joe", row.getString("name"));
    Assert.assertEquals("joe@black.com", row.getString("email"));
    Assert.assertEquals(new Long(loginTime), row.getLong("login"));
    Assert.assertNull(row.getLong("active"));
    // send an event to the stream
    long activeTime = System.currentTimeMillis();
    StreamManager streamManager = getStreamManager("events");
    streamManager.send(new Gson().toJson(new Event(activeTime, "1234", "/some/path")));
    try {
        // Wait for the last Flowlet processing 1 events, or at most 5 seconds
        RuntimeMetrics metrics = flowManager.getFlowletMetrics("updater");
        metrics.waitForProcessed(1, 5, TimeUnit.SECONDS);
    } finally {
        flowManager.stop();
        Assert.assertFalse(flowManager.isRunning());
    }
    // verify the last active time for the user
    tableManager.flush();
    row = tableManager.get().get(new Get("1234"));
    Assert.assertEquals("1234", row.getString("id"));
    Assert.assertEquals("joe", row.getString("name"));
    Assert.assertEquals("joe@black.com", row.getString("email"));
    Assert.assertEquals(new Long(loginTime), row.getLong("login"));
    Assert.assertEquals(new Long(activeTime), row.getLong("active"));
    // delete the user
    connection = (HttpURLConnection) new URL(serviceURL, "profiles/1234").openConnection();
    try {
        connection.setRequestMethod("DELETE");
        Assert.assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
    } finally {
        connection.disconnect();
    }
    // verify the user is gone
    tableManager.flush();
    row = tableManager.get().get(new Get("1234"));
    Assert.assertTrue(row.isEmpty());
    // stop the service and the flow
    serviceManager.stop();
}
Also used : FlowManager(co.cask.cdap.test.FlowManager) ApplicationManager(co.cask.cdap.test.ApplicationManager) Table(co.cask.cdap.api.dataset.table.Table) RuntimeMetrics(co.cask.cdap.api.metrics.RuntimeMetrics) Gson(com.google.gson.Gson) URL(java.net.URL) HttpURLConnection(java.net.HttpURLConnection) ServiceManager(co.cask.cdap.test.ServiceManager) StreamManager(co.cask.cdap.test.StreamManager) Get(co.cask.cdap.api.dataset.table.Get) Row(co.cask.cdap.api.dataset.table.Row) Test(org.junit.Test)

Example 4 with FlowManager

use of co.cask.cdap.test.FlowManager in project cdap by caskdata.

the class AuthorizationTest method testFlowStreamAuth.

@Test
@Category(SlowTests.class)
public void testFlowStreamAuth() throws Exception {
    createAuthNamespace();
    Authorizer authorizer = getAuthorizer();
    ApplicationManager appManager = deployApplication(AUTH_NAMESPACE, StreamAuthApp.class);
    // After deploy, change Alice from ALL to ADMIN on the namespace
    authorizer.revoke(AUTH_NAMESPACE, ALICE, EnumSet.allOf(Action.class));
    authorizer.grant(AUTH_NAMESPACE, ALICE, EnumSet.of(Action.ADMIN));
    final FlowManager flowManager = appManager.getFlowManager(StreamAuthApp.FLOW);
    StreamId streamId = AUTH_NAMESPACE.stream(StreamAuthApp.STREAM);
    StreamManager streamManager = getStreamManager(AUTH_NAMESPACE.stream(StreamAuthApp.STREAM));
    StreamManager streamManager2 = getStreamManager(AUTH_NAMESPACE.stream(StreamAuthApp.STREAM2));
    streamManager.send("Auth");
    flowManager.start();
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            DataSetManager<KeyValueTable> kvTable = getDataset(AUTH_NAMESPACE.dataset(StreamAuthApp.KVTABLE));
            return kvTable.get().read("Auth") != null;
        }
    }, 5, TimeUnit.SECONDS);
    flowManager.stop();
    flowManager.waitForRun(ProgramRunStatus.KILLED, 60, TimeUnit.SECONDS);
    // Now revoke read permission for Alice on that stream (revoke ALL and then grant everything other than READ)
    authorizer.revoke(streamId, ALICE, EnumSet.allOf(Action.class));
    authorizer.grant(streamId, ALICE, EnumSet.of(Action.WRITE, Action.ADMIN, Action.EXECUTE));
    streamManager.send("Security");
    streamManager2.send("Safety");
    try {
        flowManager.start();
    } catch (RuntimeException e) {
        Assert.assertTrue(e.getCause() instanceof UnauthorizedException);
    }
    authorizer.grant(streamId, ALICE, ImmutableSet.of(Action.READ));
    flowManager.start();
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            DataSetManager<KeyValueTable> kvTable = getDataset(AUTH_NAMESPACE.dataset(StreamAuthApp.KVTABLE));
            return kvTable.get().read("Security") != null;
        }
    }, 5, TimeUnit.SECONDS);
    authorizer.revoke(streamId, ALICE, ImmutableSet.of(Action.READ));
    TimeUnit.MILLISECONDS.sleep(10);
    flowManager.stop();
    flowManager.waitForRuns(ProgramRunStatus.KILLED, 2, 5, TimeUnit.SECONDS);
    appManager.delete();
}
Also used : FlowManager(co.cask.cdap.test.FlowManager) ApplicationManager(co.cask.cdap.test.ApplicationManager) Action(co.cask.cdap.proto.security.Action) StreamId(co.cask.cdap.proto.id.StreamId) TimeoutException(java.util.concurrent.TimeoutException) UnauthorizedException(co.cask.cdap.security.spi.authorization.UnauthorizedException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) StreamManager(co.cask.cdap.test.StreamManager) InMemoryAuthorizer(co.cask.cdap.security.authorization.InMemoryAuthorizer) Authorizer(co.cask.cdap.security.spi.authorization.Authorizer) UnauthorizedException(co.cask.cdap.security.spi.authorization.UnauthorizedException) DataSetManager(co.cask.cdap.test.DataSetManager) Category(org.junit.experimental.categories.Category) Test(org.junit.Test)

Example 5 with FlowManager

use of co.cask.cdap.test.FlowManager in project cdap by caskdata.

the class TestFrameworkTestRun method testFlowletInitAndSetInstances.

@Test(timeout = 60000L)
public void testFlowletInitAndSetInstances() throws Exception {
    ApplicationManager appManager = deployApplication(testSpace, DataSetInitApp.class);
    FlowManager flowManager = appManager.getFlowManager("DataSetFlow").start();
    RuntimeMetrics flowletMetrics = flowManager.getFlowletMetrics("Consumer");
    flowletMetrics.waitForProcessed(1, 5, TimeUnit.SECONDS);
    String generator = "Generator";
    Assert.assertEquals(1, flowManager.getFlowletInstances(generator));
    // Now change generator to 3 instances
    flowManager.setFlowletInstances(generator, 3);
    Assert.assertEquals(3, flowManager.getFlowletInstances(generator));
    // Now should have 3 processed from the consumer flowlet
    flowletMetrics.waitForProcessed(3, 10, TimeUnit.SECONDS);
    // Now reset to 1 instances
    flowManager.setFlowletInstances(generator, 1);
    Assert.assertEquals(1, flowManager.getFlowletInstances(generator));
    // Shouldn't have new item
    TimeUnit.SECONDS.sleep(3);
    Assert.assertEquals(3, flowletMetrics.getProcessed());
    // Now set to 2 instances again. Since there is a new instance, expect one new item emitted
    flowManager.setFlowletInstances(generator, 2);
    Assert.assertEquals(2, flowManager.getFlowletInstances(generator));
    flowletMetrics.waitForProcessed(4, 10, TimeUnit.SECONDS);
    flowManager.stop();
    DataSetManager<Table> dataSetManager = getDataset(testSpace.dataset("conf"));
    Table confTable = dataSetManager.get();
    Assert.assertEquals("generator", confTable.get(new Get("key", "column")).getString("column"));
    dataSetManager.flush();
}
Also used : FlowManager(co.cask.cdap.test.FlowManager) ApplicationManager(co.cask.cdap.test.ApplicationManager) KeyValueTable(co.cask.cdap.api.dataset.lib.KeyValueTable) Table(co.cask.cdap.api.dataset.table.Table) RuntimeMetrics(co.cask.cdap.api.metrics.RuntimeMetrics) Get(co.cask.cdap.api.dataset.table.Get) Test(org.junit.Test)

Aggregations

FlowManager (co.cask.cdap.test.FlowManager)24 ApplicationManager (co.cask.cdap.test.ApplicationManager)22 Test (org.junit.Test)22 RuntimeMetrics (co.cask.cdap.api.metrics.RuntimeMetrics)17 StreamManager (co.cask.cdap.test.StreamManager)17 ServiceManager (co.cask.cdap.test.ServiceManager)13 KeyValueTable (co.cask.cdap.api.dataset.lib.KeyValueTable)9 URL (java.net.URL)9 HttpURLConnection (java.net.HttpURLConnection)6 Category (org.junit.experimental.categories.Category)6 Gson (com.google.gson.Gson)5 MapReduceManager (co.cask.cdap.test.MapReduceManager)4 SparkManager (co.cask.cdap.test.SparkManager)4 Get (co.cask.cdap.api.dataset.table.Get)3 Table (co.cask.cdap.api.dataset.table.Table)3 Action (co.cask.cdap.proto.security.Action)2 WorkerManager (co.cask.cdap.test.WorkerManager)2 WorkflowManager (co.cask.cdap.test.WorkflowManager)2 HashMap (java.util.HashMap)2 Map (java.util.Map)2