Search in sources :

Example 1 with RuntimeMetrics

use of co.cask.cdap.api.metrics.RuntimeMetrics in project cdap by caskdata.

the class FlowStreamIntegrationTestRun method submitAndVerifyFlowProgram.

private void submitAndVerifyFlowProgram(FlowManager flowManager) throws Exception {
    flowManager.start();
    RuntimeMetrics flowletMetrics = flowManager.getFlowletMetrics("StreamReader");
    flowletMetrics.waitForProcessed(1, 10, TimeUnit.SECONDS);
    if (flowletMetrics.getException() > 0) {
        Assert.fail("StreamReader test failed");
    }
}
Also used : RuntimeMetrics(co.cask.cdap.api.metrics.RuntimeMetrics)

Example 2 with RuntimeMetrics

use of co.cask.cdap.api.metrics.RuntimeMetrics in project cdap by caskdata.

the class GenerateClientUsageExample method metricsClient.

public void metricsClient() throws Exception {
    // Construct the client used to interact with CDAP
    MetricsClient metricsClient = new MetricsClient(clientConfig);
    // Fetch the total number of events that have been processed by a flowlet
    RuntimeMetrics metric = metricsClient.getFlowletMetrics(new NamespaceId("user").app("HelloWorld").flow("someFlow").flowlet("process.events.processed"));
}
Also used : MetricsClient(co.cask.cdap.client.MetricsClient) RuntimeMetrics(co.cask.cdap.api.metrics.RuntimeMetrics) NamespaceId(co.cask.cdap.proto.id.NamespaceId)

Example 3 with RuntimeMetrics

use of co.cask.cdap.api.metrics.RuntimeMetrics in project cdap by caskdata.

the class UserProfilesTest method testUserProfiles.

@Test
public void testUserProfiles() throws Exception {
    // deploy the app
    ApplicationManager applicationManager = deployApplication(UserProfiles.class);
    // run the service and the flow
    FlowManager flowManager = applicationManager.getFlowManager("ActivityFlow").start();
    ServiceManager serviceManager = applicationManager.getServiceManager("UserProfileService").start();
    serviceManager.waitForStatus(true);
    URL serviceURL = serviceManager.getServiceURL();
    // create a user through the service
    String userJson = new Gson().toJson(ImmutableMap.of("id", "1234", "name", "joe", "email", "joe@bla.ck"));
    HttpURLConnection connection = (HttpURLConnection) new URL(serviceURL, "profiles/1234").openConnection();
    try {
        connection.setDoOutput(true);
        connection.setRequestMethod("PUT");
        connection.getOutputStream().write(userJson.getBytes(Charsets.UTF_8));
        Assert.assertEquals(HttpURLConnection.HTTP_CREATED, connection.getResponseCode());
    } finally {
        connection.disconnect();
    }
    // read the user through the dataset
    DataSetManager<Table> tableManager = getDataset("profiles");
    Row row = tableManager.get().get(new Get("1234"));
    Assert.assertEquals("1234", row.getString("id"));
    Assert.assertEquals("joe", row.getString("name"));
    Assert.assertEquals("joe@bla.ck", row.getString("email"));
    Assert.assertNull(row.getLong("login"));
    Assert.assertNull(row.getLong("active"));
    // update email address through service
    connection = (HttpURLConnection) new URL(serviceURL, "profiles/1234/email").openConnection();
    try {
        connection.setDoOutput(true);
        connection.setRequestMethod("PUT");
        connection.getOutputStream().write("joe@black.com".getBytes(Charsets.UTF_8));
        Assert.assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
    } finally {
        connection.disconnect();
    }
    // verify the updated email address
    tableManager.flush();
    row = tableManager.get().get(new Get("1234"));
    Assert.assertEquals("1234", row.getString("id"));
    Assert.assertEquals("joe", row.getString("name"));
    Assert.assertEquals("joe@black.com", row.getString("email"));
    Assert.assertNull(row.getLong("login"));
    Assert.assertNull(row.getLong("active"));
    // send a login event
    long loginTime = System.currentTimeMillis();
    connection = (HttpURLConnection) new URL(serviceURL, "profiles/1234/lastLogin").openConnection();
    try {
        connection.setDoOutput(true);
        connection.setRequestMethod("PUT");
        connection.getOutputStream().write(Long.toString(loginTime).getBytes(Charsets.UTF_8));
        Assert.assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
    } finally {
        connection.disconnect();
    }
    // verify the login time through the dataset
    tableManager.flush();
    row = tableManager.get().get(new Get("1234"));
    Assert.assertEquals("1234", row.getString("id"));
    Assert.assertEquals("joe", row.getString("name"));
    Assert.assertEquals("joe@black.com", row.getString("email"));
    Assert.assertEquals(new Long(loginTime), row.getLong("login"));
    Assert.assertNull(row.getLong("active"));
    // send an event to the stream
    long activeTime = System.currentTimeMillis();
    StreamManager streamManager = getStreamManager("events");
    streamManager.send(new Gson().toJson(new Event(activeTime, "1234", "/some/path")));
    try {
        // Wait for the last Flowlet processing 1 events, or at most 5 seconds
        RuntimeMetrics metrics = flowManager.getFlowletMetrics("updater");
        metrics.waitForProcessed(1, 5, TimeUnit.SECONDS);
    } finally {
        flowManager.stop();
        Assert.assertFalse(flowManager.isRunning());
    }
    // verify the last active time for the user
    tableManager.flush();
    row = tableManager.get().get(new Get("1234"));
    Assert.assertEquals("1234", row.getString("id"));
    Assert.assertEquals("joe", row.getString("name"));
    Assert.assertEquals("joe@black.com", row.getString("email"));
    Assert.assertEquals(new Long(loginTime), row.getLong("login"));
    Assert.assertEquals(new Long(activeTime), row.getLong("active"));
    // delete the user
    connection = (HttpURLConnection) new URL(serviceURL, "profiles/1234").openConnection();
    try {
        connection.setRequestMethod("DELETE");
        Assert.assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
    } finally {
        connection.disconnect();
    }
    // verify the user is gone
    tableManager.flush();
    row = tableManager.get().get(new Get("1234"));
    Assert.assertTrue(row.isEmpty());
    // stop the service and the flow
    serviceManager.stop();
}
Also used : FlowManager(co.cask.cdap.test.FlowManager) ApplicationManager(co.cask.cdap.test.ApplicationManager) Table(co.cask.cdap.api.dataset.table.Table) RuntimeMetrics(co.cask.cdap.api.metrics.RuntimeMetrics) Gson(com.google.gson.Gson) URL(java.net.URL) HttpURLConnection(java.net.HttpURLConnection) ServiceManager(co.cask.cdap.test.ServiceManager) StreamManager(co.cask.cdap.test.StreamManager) Get(co.cask.cdap.api.dataset.table.Get) Row(co.cask.cdap.api.dataset.table.Row) Test(org.junit.Test)

Example 4 with RuntimeMetrics

use of co.cask.cdap.api.metrics.RuntimeMetrics in project cdap by caskdata.

the class WordCountTest method testWordCount.

@Test
public void testWordCount() throws Exception {
    WordCount.WordCountConfig config = new WordCount.WordCountConfig("words", "stats", "counts", "unique", "assoc");
    // Deploy the Application
    ApplicationManager appManager = deployApplication(WordCount.class, config);
    // validate that the wordCount table is empty, and that it has no entry for "world"
    DataSetManager<KeyValueTable> datasetManager = getDataset(config.getWordCountTable());
    KeyValueTable wordCounts = datasetManager.get();
    Assert.assertNull(wordCounts.read("world"));
    // Start the Flow
    FlowManager flowManager = appManager.getFlowManager("WordCounter").start();
    // Send a few events to the stream
    StreamManager streamManager = getStreamManager("words");
    streamManager.send("hello world");
    streamManager.send("a wonderful world");
    streamManager.send("the world says hello");
    // Wait for the events to be processed, or at most 5 seconds
    RuntimeMetrics metrics = flowManager.getFlowletMetrics("associator");
    metrics.waitForProcessed(3, 5, TimeUnit.SECONDS);
    // start a new transaction so that the "wordCounts" dataset sees changes made by the flow
    datasetManager.flush();
    Assert.assertEquals(3L, Bytes.toLong(wordCounts.read("world")));
    // Start RetrieveCounts service
    ServiceManager serviceManager = appManager.getServiceManager(RetrieveCounts.SERVICE_NAME).start();
    // Wait service startup
    serviceManager.waitForStatus(true);
    // First verify global statistics
    String response = requestService(new URL(serviceManager.getServiceURL(15, TimeUnit.SECONDS), "stats"));
    Map<String, String> map = new Gson().fromJson(response, STRING_MAP_TYPE);
    Assert.assertEquals("9", map.get("totalWords"));
    Assert.assertEquals("6", map.get("uniqueWords"));
    Assert.assertEquals(((double) 42) / 9, Double.valueOf(map.get("averageLength")), 0.001);
    // Now verify statistics for a specific word
    response = requestService(new URL(serviceManager.getServiceURL(15, TimeUnit.SECONDS), "count/world"));
    Map<String, Object> omap = new Gson().fromJson(response, OBJECT_MAP_TYPE);
    Assert.assertEquals("world", omap.get("word"));
    Assert.assertEquals(3.0, omap.get("count"));
    // The associations are a map within the map
    @SuppressWarnings("unchecked") Map<String, Double> assocs = (Map<String, Double>) omap.get("assocs");
    Assert.assertEquals(2.0, assocs.get("hello"), 0.000001);
    Assert.assertTrue(assocs.containsKey("hello"));
}
Also used : FlowManager(co.cask.cdap.test.FlowManager) ApplicationManager(co.cask.cdap.test.ApplicationManager) RuntimeMetrics(co.cask.cdap.api.metrics.RuntimeMetrics) Gson(com.google.gson.Gson) URL(java.net.URL) KeyValueTable(co.cask.cdap.api.dataset.lib.KeyValueTable) StreamManager(co.cask.cdap.test.StreamManager) ServiceManager(co.cask.cdap.test.ServiceManager) Map(java.util.Map) Test(org.junit.Test)

Example 5 with RuntimeMetrics

use of co.cask.cdap.api.metrics.RuntimeMetrics in project cdap by caskdata.

the class TestFrameworkTestRun method testFlowletMetricsReset.

@Test(timeout = 60000L)
public void testFlowletMetricsReset() throws Exception {
    ApplicationManager appManager = deployApplication(DataSetInitApp.class);
    FlowManager flowManager = appManager.getFlowManager("DataSetFlow").start();
    RuntimeMetrics flowletMetrics = flowManager.getFlowletMetrics("Consumer");
    flowletMetrics.waitForProcessed(1, 5, TimeUnit.SECONDS);
    flowManager.stop();
    Assert.assertEquals(1, flowletMetrics.getProcessed());
    getMetricsManager().resetAll();
    // check the metrics were deleted after reset
    Assert.assertEquals(0, flowletMetrics.getProcessed());
}
Also used : FlowManager(co.cask.cdap.test.FlowManager) ApplicationManager(co.cask.cdap.test.ApplicationManager) RuntimeMetrics(co.cask.cdap.api.metrics.RuntimeMetrics) Test(org.junit.Test)

Aggregations

RuntimeMetrics (co.cask.cdap.api.metrics.RuntimeMetrics)21 ApplicationManager (co.cask.cdap.test.ApplicationManager)17 FlowManager (co.cask.cdap.test.FlowManager)17 Test (org.junit.Test)16 ServiceManager (co.cask.cdap.test.ServiceManager)12 StreamManager (co.cask.cdap.test.StreamManager)12 URL (java.net.URL)10 KeyValueTable (co.cask.cdap.api.dataset.lib.KeyValueTable)6 Gson (com.google.gson.Gson)6 HttpURLConnection (java.net.HttpURLConnection)6 Category (org.junit.experimental.categories.Category)5 Get (co.cask.cdap.api.dataset.table.Get)2 Table (co.cask.cdap.api.dataset.table.Table)2 MapReduceManager (co.cask.cdap.test.MapReduceManager)2 SparkManager (co.cask.cdap.test.SparkManager)2 HttpRequest (co.cask.common.http.HttpRequest)2 HttpResponse (co.cask.common.http.HttpResponse)2 Map (java.util.Map)2 KeyValue (co.cask.cdap.api.dataset.lib.KeyValue)1 Row (co.cask.cdap.api.dataset.table.Row)1