Examples with StreamId co.cask.cdap.proto.id.StreamId used on opensource projects

Example 11 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class StreamClientTestRun method testAsyncWrite.

/**
   * Tests for async write to stream.
   */
@Test
public void testAsyncWrite() throws Exception {
    StreamId streamId = namespaceId.stream("testAsync");
    streamClient.create(streamId);
    // Send 10 async writes
    int msgCount = 10;
    for (int i = 0; i < msgCount; i++) {
        streamClient.asyncSendEvent(streamId, "Testing " + i);
    }
    // Reads them back to verify. Needs to do it multiple times as the writes happens async.
    List<StreamEvent> events = Lists.newArrayList();
    Stopwatch stopwatch = new Stopwatch();
    stopwatch.start();
    while (events.size() != msgCount && stopwatch.elapsedTime(TimeUnit.SECONDS) < 10L) {
        events.clear();
        streamClient.getEvents(streamId, 0, Long.MAX_VALUE, msgCount, events);
    }
    Assert.assertEquals(msgCount, events.size());
    long lastTimestamp = 0L;
    for (int i = 0; i < msgCount; i++) {
        Assert.assertEquals("Testing " + i, Charsets.UTF_8.decode(events.get(i).getBody()).toString());
        lastTimestamp = events.get(i).getTimestamp();
    }
    // No more events
    stopwatch = new Stopwatch();
    stopwatch.start();
    events.clear();
    while (events.isEmpty() && stopwatch.elapsedTime(TimeUnit.SECONDS) < 1L) {
        events.clear();
        streamClient.getEvents(streamId, lastTimestamp + 1, Long.MAX_VALUE, msgCount, events);
    }
    Assert.assertTrue(events.isEmpty());
}

Example 12 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class StreamViewClientTest method testAll.

@Test
public void testAll() throws Exception {
    NamespaceId namespace = NamespaceId.DEFAULT;
    StreamId stream = namespace.stream("foo");
    StreamViewId view1 = stream.view("view1");
    LOG.info("Creating stream {}", stream);
    streamClient.create(stream);
    try {
        LOG.info("Sending events to stream {}", stream);
        streamClient.sendEvent(stream, "a,b,c");
        streamClient.sendEvent(stream, "d,e,f");
        streamClient.sendEvent(stream, "g,h,i");
        LOG.info("Verifying that no views exist yet");
        Assert.assertEquals(ImmutableList.of(), streamViewClient.list(stream));
        try {
            streamViewClient.get(view1);
            Assert.fail();
        } catch (NotFoundException e) {
            Assert.assertEquals(view1, e.getObject());
        }
        FormatSpecification format = new FormatSpecification("csv", Schema.recordOf("foo", Schema.Field.of("one", Schema.of(Schema.Type.STRING)), Schema.Field.of("two", Schema.of(Schema.Type.STRING)), Schema.Field.of("three", Schema.of(Schema.Type.STRING))));
        ViewSpecification viewSpecification = new ViewSpecification(format, "firsttable");
        LOG.info("Creating view {} with config {}", view1, GSON.toJson(viewSpecification));
        Assert.assertEquals(true, streamViewClient.createOrUpdate(view1, viewSpecification));
        LOG.info("Verifying that view {} has been created", view1);
        Assert.assertEquals(new ViewDetail(view1.getView(), viewSpecification), streamViewClient.get(view1));
        Assert.assertEquals(ImmutableList.of(view1.getView()), streamViewClient.list(stream));
        FormatSpecification newFormat = new FormatSpecification("csv", Schema.recordOf("foo", Schema.Field.of("one", Schema.of(Schema.Type.STRING)), Schema.Field.of("two", Schema.of(Schema.Type.STRING)), Schema.Field.of("three", Schema.of(Schema.Type.STRING))));
        ViewSpecification newViewSpecification = new ViewSpecification(newFormat, "firsttable");
        LOG.info("Updating view {} with config {}", view1, GSON.toJson(newViewSpecification));
        Assert.assertEquals(false, streamViewClient.createOrUpdate(view1, newViewSpecification));
        LOG.info("Verifying that view {} has been updated", view1);
        Assert.assertEquals(new ViewDetail(view1.getView(), newViewSpecification), streamViewClient.get(view1));
        Assert.assertEquals(ImmutableList.of(view1.getView()), streamViewClient.list(stream));
        ExploreExecutionResult executionResult = queryClient.execute(view1.getParent().getParent(), "select one,two,three from firsttable").get();
        Assert.assertNotNull(executionResult.getResultSchema());
        Assert.assertEquals(3, executionResult.getResultSchema().size());
        Assert.assertEquals("one", executionResult.getResultSchema().get(0).getName());
        Assert.assertEquals("two", executionResult.getResultSchema().get(1).getName());
        Assert.assertEquals("three", executionResult.getResultSchema().get(2).getName());
        List<QueryResult> results = Lists.newArrayList(executionResult);
        Assert.assertNotNull(results);
        Assert.assertEquals(3, results.size());
        Assert.assertEquals("a", results.get(0).getColumns().get(0));
        Assert.assertEquals("b", results.get(0).getColumns().get(1));
        Assert.assertEquals("c", results.get(0).getColumns().get(2));
        Assert.assertEquals("d", results.get(1).getColumns().get(0));
        Assert.assertEquals("e", results.get(1).getColumns().get(1));
        Assert.assertEquals("f", results.get(1).getColumns().get(2));
        Assert.assertEquals("g", results.get(2).getColumns().get(0));
        Assert.assertEquals("h", results.get(2).getColumns().get(1));
        Assert.assertEquals("i", results.get(2).getColumns().get(2));
        LOG.info("Deleting view {}", view1);
        streamViewClient.delete(view1);
        LOG.info("Verifying that view {] has been deleted", view1);
        try {
            streamViewClient.get(view1);
            Assert.fail();
        } catch (NotFoundException e) {
            Assert.assertEquals(view1, e.getObject());
        }
        Assert.assertEquals(ImmutableList.of(), streamViewClient.list(stream));
    } finally {
        streamClient.delete(stream);
    }
    // test deleting stream with a view
    LOG.info("Creating stream {}", stream);
    streamClient.create(stream);
    try {
        FormatSpecification format = new FormatSpecification("csv", Schema.recordOf("foo", Schema.Field.of("one", Schema.of(Schema.Type.STRING)), Schema.Field.of("two", Schema.of(Schema.Type.STRING)), Schema.Field.of("three", Schema.of(Schema.Type.STRING))));
        ViewSpecification viewSpecification = new ViewSpecification(format, "firsttable");
        LOG.info("Creating view {} with config {}", view1, GSON.toJson(viewSpecification));
        Assert.assertEquals(true, streamViewClient.createOrUpdate(view1, viewSpecification));
    } finally {
        streamClient.delete(stream);
    }
}

Example 13 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class QueryClientTest method testAll.

@Test
public void testAll() throws Exception {
    NamespaceId namespace = new NamespaceId("queryClientTestNamespace");
    NamespaceId otherNamespace = new NamespaceId("queryClientOtherNamespace");
    namespaceClient.create(new NamespaceMeta.Builder().setName(namespace).build());
    ApplicationId app = namespace.app(FakeApp.NAME);
    FlowId flow = app.flow(FakeFlow.NAME);
    DatasetId dataset = namespace.dataset(FakeApp.DS_NAME);
    appClient.deploy(namespace, createAppJarFile(FakeApp.class));
    try {
        programClient.start(flow);
        assertProgramRunning(programClient, flow);
        StreamId stream = namespace.stream(FakeApp.STREAM_NAME);
        streamClient.sendEvent(stream, "bob:123");
        streamClient.sendEvent(stream, "joe:321");
        Thread.sleep(3000);
        executeBasicQuery(namespace, FakeApp.DS_NAME);
        exploreClient.disableExploreDataset(dataset).get();
        try {
            queryClient.execute(namespace, "select * from " + FakeApp.DS_NAME).get();
            Assert.fail("Explore Query should have thrown an ExecutionException since explore is disabled");
        } catch (ExecutionException e) {
        // ignored
        }
        exploreClient.enableExploreDataset(dataset).get();
        executeBasicQuery(namespace, FakeApp.DS_NAME);
        try {
            queryClient.execute(otherNamespace, "show tables").get();
            Assert.fail("Explore Query should have thrown an ExecutionException since the database should not exist");
        } catch (ExecutionException e) {
        // expected
        }
    } finally {
        programClient.stop(flow);
        assertProgramStopped(programClient, flow);
        try {
            appClient.delete(app);
        } catch (Exception e) {
            LOG.error("Error deleting app {} during test cleanup.", e);
        }
    }
}

Example 14 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class UsageHandlerTestRun method testSparkUsage.

@Test
public void testSparkUsage() throws Exception {
    final ApplicationId app = NamespaceId.DEFAULT.app(AllProgramsApp.NAME);
    final ProgramId program = app.spark(AllProgramsApp.NoOpSpark.NAME);
    final StreamId stream = NamespaceId.DEFAULT.stream(AllProgramsApp.STREAM_NAME);
    final DatasetId dataset = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME);
    Assert.assertEquals(0, getAppStreamUsage(app).size());
    Assert.assertEquals(0, getProgramStreamUsage(program).size());
    Assert.assertEquals(0, getStreamProgramUsage(stream).size());
    Assert.assertEquals(0, getAppDatasetUsage(app).size());
    Assert.assertEquals(0, getDatasetProgramUsage(dataset).size());
    deployApp(AllProgramsApp.class);
    try {
        // the program will run and stop by itself.
        startProgram(program);
        waitState(program, ProgramStatus.STOPPED);
        Assert.assertTrue(getAppStreamUsage(app).contains(stream));
        Assert.assertTrue(getProgramStreamUsage(program).contains(stream));
        Assert.assertTrue(getStreamProgramUsage(stream).contains(program));
        Assert.assertTrue(getProgramDatasetUsage(program).contains(dataset));
        Assert.assertTrue(getAppDatasetUsage(app).contains(dataset));
        Assert.assertTrue(getDatasetProgramUsage(dataset).contains(program));
    } finally {
        deleteApp(app);
        Assert.assertEquals(0, getAppStreamUsage(app).size());
        Assert.assertEquals(0, getProgramStreamUsage(program).size());
        Assert.assertEquals(0, getStreamProgramUsage(stream).size());
        Assert.assertEquals(0, getAppDatasetUsage(app).size());
        Assert.assertEquals(0, getDatasetProgramUsage(dataset).size());
    }
    deployApp(AllProgramsApp.class);
    try {
        // the program will run and stop by itself.
        startProgram(program);
        waitState(program, ProgramStatus.STOPPED);
        Assert.assertTrue(getAppStreamUsage(app).contains(stream));
        Assert.assertTrue(getProgramStreamUsage(program).contains(stream));
        Assert.assertTrue(getStreamProgramUsage(stream).contains(program));
        Assert.assertTrue(getProgramDatasetUsage(program).contains(dataset));
        Assert.assertTrue(getAppDatasetUsage(app).contains(dataset));
        Assert.assertTrue(getDatasetProgramUsage(dataset).contains(program));
    } finally {
        deleteApp(app);
        Assert.assertEquals(0, getAppStreamUsage(app).size());
        Assert.assertEquals(0, getProgramStreamUsage(program).size());
        Assert.assertEquals(0, getStreamProgramUsage(stream).size());
        Assert.assertEquals(0, getAppDatasetUsage(app).size());
        Assert.assertEquals(0, getDatasetProgramUsage(dataset).size());
    }
}

Example 15 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class UsageHandlerTestRun method testMapReduceUsage.

@Test
public void testMapReduceUsage() throws Exception {
    final ApplicationId app = NamespaceId.DEFAULT.app(AllProgramsApp.NAME);
    final ProgramId program = app.mr(AllProgramsApp.NoOpMR.NAME);
    final StreamId stream = NamespaceId.DEFAULT.stream(AllProgramsApp.STREAM_NAME);
    final DatasetId dataset = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME);
    Assert.assertEquals(0, getAppStreamUsage(app).size());
    Assert.assertEquals(0, getProgramStreamUsage(program).size());
    Assert.assertEquals(0, getStreamProgramUsage(stream).size());
    Assert.assertEquals(0, getAppDatasetUsage(app).size());
    Assert.assertEquals(0, getDatasetProgramUsage(dataset).size());
    deployApp(AllProgramsApp.class);
    // now that we only support dynamic dataset instantiation in initialize (and not in configure as before),
    // we must run the mapreduce program to register its usage
    startProgram(program);
    waitState(program, ProgramStatus.STOPPED);
    try {
        Assert.assertTrue(getAppStreamUsage(app).contains(stream));
        Assert.assertTrue(getProgramStreamUsage(program).contains(stream));
        Assert.assertTrue(getStreamProgramUsage(stream).contains(program));
        Assert.assertTrue(getProgramDatasetUsage(program).contains(dataset));
        Assert.assertTrue(getAppDatasetUsage(app).contains(dataset));
        Assert.assertTrue(getDatasetProgramUsage(dataset).contains(program));
    } finally {
        deleteApp(app);
        Assert.assertEquals(0, getAppStreamUsage(app).size());
        Assert.assertEquals(0, getProgramStreamUsage(program).size());
        Assert.assertEquals(0, getStreamProgramUsage(stream).size());
        Assert.assertEquals(0, getAppDatasetUsage(app).size());
        Assert.assertEquals(0, getDatasetProgramUsage(dataset).size());
    }
}