Examples with StreamManager co.cask.cdap.test.StreamManager used on opensource projects

Example 1 with StreamManager

use of co.cask.cdap.test.StreamManager in project cdap by caskdata.

the class SparkTestRun method testScalaSparkCrossNSStream.

@Test
public void testScalaSparkCrossNSStream() throws Exception {
    // create a namespace for stream and create the stream in it
    NamespaceMeta crossNSStreamMeta = new NamespaceMeta.Builder().setName("streamSpaceForSpark").build();
    getNamespaceAdmin().create(crossNSStreamMeta);
    StreamManager streamManager = getStreamManager(crossNSStreamMeta.getNamespaceId().stream("testStream"));
    // create a namespace for dataset and add the dataset instance in it
    NamespaceMeta crossNSDatasetMeta = new NamespaceMeta.Builder().setName("crossNSDataset").build();
    getNamespaceAdmin().create(crossNSDatasetMeta);
    addDatasetInstance(crossNSDatasetMeta.getNamespaceId().dataset("count"), "keyValueTable");
    // write something to the stream
    streamManager.createStream();
    for (int i = 0; i < 50; i++) {
        streamManager.send(String.valueOf(i));
    }
    // deploy the spark app in another namespace (default)
    ApplicationManager applicationManager = deploy(SparkAppUsingObjectStore.class);
    Map<String, String> args = ImmutableMap.of(ScalaCrossNSProgram.STREAM_NAMESPACE(), crossNSStreamMeta.getNamespaceId().getNamespace(), ScalaCrossNSProgram.DATASET_NAMESPACE(), crossNSDatasetMeta.getNamespaceId().getNamespace(), ScalaCrossNSProgram.DATASET_NAME(), "count");
    SparkManager sparkManager = applicationManager.getSparkManager(ScalaCrossNSProgram.class.getSimpleName()).start(args);
    sparkManager.waitForRun(ProgramRunStatus.COMPLETED, 1, TimeUnit.MINUTES);
    // get the dataset from the other namespace where we expect it to exist and compare the data
    DataSetManager<KeyValueTable> countManager = getDataset(crossNSDatasetMeta.getNamespaceId().dataset("count"));
    KeyValueTable results = countManager.get();
    for (int i = 0; i < 50; i++) {
        byte[] key = String.valueOf(i).getBytes(Charsets.UTF_8);
        Assert.assertArrayEquals(key, results.read(key));
    }
}

Example 2 with StreamManager

use of co.cask.cdap.test.StreamManager in project cdap by caskdata.

the class SparkTestRun method testStreamFormatSpec.

@Test
public void testStreamFormatSpec() throws Exception {
    ApplicationManager appManager = deploy(TestSparkApp.class);
    StreamManager stream = getStreamManager("PeopleStream");
    stream.send("Old Man,50");
    stream.send("Baby,1");
    stream.send("Young Guy,18");
    stream.send("Small Kid,5");
    stream.send("Legal Drinker,21");
    Map<String, String> outputArgs = new HashMap<>();
    FileSetArguments.setOutputPath(outputArgs, "output");
    Map<String, String> runtimeArgs = new HashMap<>();
    runtimeArgs.putAll(RuntimeArguments.addScope(Scope.DATASET, "PeopleFileSet", outputArgs));
    runtimeArgs.put("stream.name", "PeopleStream");
    runtimeArgs.put("output.dataset", "PeopleFileSet");
    runtimeArgs.put("sql.statement", "SELECT name, age FROM people WHERE age >= 21");
    List<String> programs = Arrays.asList(ScalaStreamFormatSpecSpark.class.getSimpleName(), StreamFormatSpecSpark.class.getSimpleName());
    for (String sparkProgramName : programs) {
        // Clean the output before starting
        DataSetManager<FileSet> fileSetManager = getDataset("PeopleFileSet");
        Location outputDir = fileSetManager.get().getLocation("output");
        outputDir.delete(true);
        SparkManager sparkManager = appManager.getSparkManager(sparkProgramName);
        sparkManager.start(runtimeArgs);
        sparkManager.waitForRun(ProgramRunStatus.COMPLETED, 180, TimeUnit.SECONDS);
        // Find the output part file. There is only one because the program repartition to 1
        Location outputFile = Iterables.find(outputDir.list(), new Predicate<Location>() {

            @Override
            public boolean apply(Location input) {
                return input.getName().startsWith("part-r-");
            }
        });
        // Verify the result
        List<String> lines = CharStreams.readLines(CharStreams.newReaderSupplier(Locations.newInputSupplier(outputFile), Charsets.UTF_8));
        Map<String, Integer> result = new HashMap<>();
        for (String line : lines) {
            String[] parts = line.split(":");
            result.put(parts[0], Integer.parseInt(parts[1]));
        }
        Assert.assertEquals(ImmutableMap.of("Old Man", 50, "Legal Drinker", 21), result);
    }
}

Example 3 with StreamManager

use of co.cask.cdap.test.StreamManager in project cdap by caskdata.

the class AuthorizationTest method testCrossNSFlowlet.

@Test
public void testCrossNSFlowlet() throws Exception {
    createAuthNamespace();
    ApplicationId appId = AUTH_NAMESPACE.app(CrossNsDatasetAccessApp.APP_NAME);
    StreamId streamId = AUTH_NAMESPACE.stream(CrossNsDatasetAccessApp.STREAM_NAME);
    Map<EntityId, Set<Action>> neededPrivileges = ImmutableMap.<EntityId, Set<Action>>builder().put(appId, EnumSet.of(Action.ADMIN)).put(AUTH_NAMESPACE.artifact(CrossNsDatasetAccessApp.class.getSimpleName(), "1.0-SNAPSHOT"), EnumSet.of(Action.ADMIN)).put(streamId, EnumSet.of(Action.ADMIN)).build();
    setUpPrivilegeAndRegisterForDeletion(ALICE, neededPrivileges);
    ProgramId programId = appId.flow(CrossNsDatasetAccessApp.FLOW_NAME);
    cleanUpEntities.add(programId);
    // grant bob execute on program and READ/WRITE on stream
    grantAndAssertSuccess(programId, BOB, EnumSet.of(Action.EXECUTE));
    grantAndAssertSuccess(streamId, BOB, EnumSet.of(Action.WRITE, Action.READ));
    ApplicationManager appManager = deployApplication(AUTH_NAMESPACE, CrossNsDatasetAccessApp.class);
    // switch to BOB
    SecurityRequestContext.setUserId(BOB.getName());
    // Send data to stream as BOB this ensures that BOB can write to a stream in auth namespace
    StreamManager streamManager = getStreamManager(AUTH_NAMESPACE.stream(CrossNsDatasetAccessApp.STREAM_NAME));
    for (int i = 0; i < 10; i++) {
        streamManager.send(String.valueOf(i).getBytes());
    }
    // switch to back to ALICE
    SecurityRequestContext.setUserId(ALICE.getName());
    final FlowManager flowManager = appManager.getFlowManager(CrossNsDatasetAccessApp.FLOW_NAME);
    testSystemDatasetAccessFromFlowlet(flowManager);
    testCrossNSDatasetAccessFromFlowlet(flowManager);
}

Example 4 with StreamManager

use of co.cask.cdap.test.StreamManager in project cdap by caskdata.

the class AuthorizationTest method testFlowStreamAuth.

@Test
@Category(SlowTests.class)
public void testFlowStreamAuth() throws Exception {
    createAuthNamespace();
    Authorizer authorizer = getAuthorizer();
    // set up privilege to deploy the app
    setUpPrivilegeToDeployStreamAuthApp();
    StreamId streamId1 = AUTH_NAMESPACE.stream(StreamAuthApp.STREAM);
    StreamId streamId2 = AUTH_NAMESPACE.stream(StreamAuthApp.STREAM2);
    Map<EntityId, Set<Action>> additionalPrivileges = ImmutableMap.<EntityId, Set<Action>>builder().put(streamId1, EnumSet.of(Action.READ, Action.WRITE)).put(streamId2, EnumSet.of(Action.READ, Action.WRITE)).put(AUTH_NAMESPACE.dataset(StreamAuthApp.KVTABLE), EnumSet.of(Action.READ, Action.WRITE)).put(AUTH_NAMESPACE.app(StreamAuthApp.APP).flow(StreamAuthApp.FLOW), EnumSet.of(Action.EXECUTE)).build();
    setUpPrivilegeAndRegisterForDeletion(ALICE, additionalPrivileges);
    ApplicationManager appManager = deployApplication(AUTH_NAMESPACE, StreamAuthApp.class);
    final FlowManager flowManager = appManager.getFlowManager(StreamAuthApp.FLOW);
    StreamManager streamManager = getStreamManager(streamId1);
    StreamManager streamManager2 = getStreamManager(streamId2);
    streamManager.send("Auth");
    flowManager.start();
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            DataSetManager<KeyValueTable> kvTable = getDataset(AUTH_NAMESPACE.dataset(StreamAuthApp.KVTABLE));
            return kvTable.get().read("Auth") != null;
        }
    }, 5, TimeUnit.SECONDS);
    flowManager.stop();
    flowManager.waitForRun(ProgramRunStatus.KILLED, 60, TimeUnit.SECONDS);
    // Now revoke the privileges for ALICE on the stream and grant her ADMIN and WRITE
    authorizer.revoke(Authorizable.fromEntityId(streamId1), ALICE, EnumSet.allOf(Action.class));
    authorizer.grant(Authorizable.fromEntityId(streamId1), ALICE, EnumSet.of(Action.WRITE, Action.ADMIN));
    streamManager.send("Security");
    streamManager2.send("Safety");
    try {
        flowManager.start();
    } catch (UnauthorizedException e) {
    // Expected
    }
    flowManager.waitForStatus(false);
    authorizer.grant(Authorizable.fromEntityId(streamId1), ALICE, ImmutableSet.of(Action.READ));
    flowManager.start();
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            DataSetManager<KeyValueTable> kvTable = getDataset(AUTH_NAMESPACE.dataset(StreamAuthApp.KVTABLE));
            return kvTable.get().read("Security") != null;
        }
    }, 5, TimeUnit.SECONDS);
    TimeUnit.MILLISECONDS.sleep(10);
    flowManager.stop();
    flowManager.waitForRuns(ProgramRunStatus.KILLED, 2, 5, TimeUnit.SECONDS);
    appManager.delete();
}

Example 5 with StreamManager

use of co.cask.cdap.test.StreamManager in project cdap by caskdata.

the class AuthorizationTest method testWorkerStreamAuth.

@Test
@Category(SlowTests.class)
public void testWorkerStreamAuth() throws Exception {
    createAuthNamespace();
    Authorizer authorizer = getAuthorizer();
    setUpPrivilegeToDeployStreamAuthApp();
    StreamId streamId = AUTH_NAMESPACE.stream(StreamAuthApp.STREAM);
    Map<EntityId, Set<Action>> additionalPrivileges = ImmutableMap.<EntityId, Set<Action>>builder().put(streamId, EnumSet.of(Action.READ, Action.WRITE)).put(AUTH_NAMESPACE.app(StreamAuthApp.APP).worker(StreamAuthApp.WORKER), EnumSet.of(Action.EXECUTE)).build();
    setUpPrivilegeAndRegisterForDeletion(ALICE, additionalPrivileges);
    ApplicationManager appManager = deployApplication(AUTH_NAMESPACE, StreamAuthApp.class);
    WorkerManager workerManager = appManager.getWorkerManager(StreamAuthApp.WORKER);
    workerManager.start();
    workerManager.waitForRun(ProgramRunStatus.COMPLETED, 60, TimeUnit.SECONDS);
    StreamManager streamManager = getStreamManager(AUTH_NAMESPACE.stream(StreamAuthApp.STREAM));
    Assert.assertEquals(5, streamManager.getEvents(0, Long.MAX_VALUE, Integer.MAX_VALUE).size());
    // Now revoke write permission for Alice on that stream
    authorizer.revoke(Authorizable.fromEntityId(streamId), ALICE, EnumSet.of(Action.WRITE));
    workerManager.start();
    workerManager.waitForRuns(ProgramRunStatus.FAILED, 1, 60, TimeUnit.SECONDS);
    Assert.assertEquals(5, streamManager.getEvents(0, Long.MAX_VALUE, Integer.MAX_VALUE).size());
    appManager.delete();
}