Examples with StreamProperties co.cask.cdap.proto.StreamProperties used on opensource projects

Example 1 with StreamProperties

use of co.cask.cdap.proto.StreamProperties in project cdap by caskdata.

the class WorkflowHttpHandlerTest method testStreamSizeSchedules.

@Test
public void testStreamSizeSchedules() throws Exception {
    // Steps for the test:
    // 1. Deploy the app
    // 2. Verify the schedules
    // 3. Ingest data in the stream
    // 4. Verify the history after waiting a while
    // 5. Suspend the schedule
    // 6. Ingest data in the stream
    // 7. Verify there are no runs after the suspend by looking at the history
    // 8. Resume the schedule
    // 9. Verify there are runs after the resume by looking at the history
    String appName = "AppWithStreamSizeSchedule";
    String sampleSchedule1 = "SampleSchedule1";
    String sampleSchedule2 = "SampleSchedule2";
    String workflowName = "SampleWorkflow";
    String streamName = "stream";
    Id.Program programId = Id.Program.from(TEST_NAMESPACE2, appName, ProgramType.WORKFLOW, workflowName);
    StringBuilder longStringBuilder = new StringBuilder();
    for (int i = 0; i < 10000; i++) {
        longStringBuilder.append("dddddddddd");
    }
    String longString = longStringBuilder.toString();
    // deploy app with schedule in namespace 2
    HttpResponse response = deploy(AppWithStreamSizeSchedule.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
    Assert.assertEquals(200, response.getStatusLine().getStatusCode());
    Assert.assertEquals(200, resumeSchedule(TEST_NAMESPACE2, appName, sampleSchedule1));
    Assert.assertEquals(200, resumeSchedule(TEST_NAMESPACE2, appName, sampleSchedule2));
    // get schedules
    List<ScheduleDetail> schedules = getSchedules(TEST_NAMESPACE2, appName, workflowName);
    Assert.assertEquals(2, schedules.size());
    String scheduleName1 = schedules.get(0).getName();
    String scheduleName2 = schedules.get(1).getName();
    Assert.assertNotNull(scheduleName1);
    Assert.assertFalse(scheduleName1.isEmpty());
    // Change notification threshold for stream
    response = doPut(String.format("/v3/namespaces/%s/streams/%s/properties", TEST_NAMESPACE2, streamName), "{'notification.threshold.mb': 1}");
    Assert.assertEquals(200, response.getStatusLine().getStatusCode());
    response = doGet(String.format("/v3/namespaces/%s/streams/%s", TEST_NAMESPACE2, streamName));
    String json = EntityUtils.toString(response.getEntity());
    StreamProperties properties = new Gson().fromJson(json, StreamProperties.class);
    Assert.assertEquals(1, properties.getNotificationThresholdMB().intValue());
    // Ingest over 1MB of data in stream
    for (int i = 0; i < 12; ++i) {
        response = doPost(String.format("/v3/namespaces/%s/streams/%s", TEST_NAMESPACE2, streamName), longString);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
    }
    // Only schedule 1 should get executed
    verifyProgramRuns(programId, "completed");
    //Check schedule status
    assertSchedule(programId, scheduleName1, true, 30, TimeUnit.SECONDS);
    assertSchedule(programId, scheduleName2, true, 30, TimeUnit.SECONDS);
    Assert.assertEquals(200, suspendSchedule(TEST_NAMESPACE2, appName, scheduleName1));
    Assert.assertEquals(200, suspendSchedule(TEST_NAMESPACE2, appName, scheduleName2));
    //check paused state
    assertSchedule(programId, scheduleName1, false, 30, TimeUnit.SECONDS);
    assertSchedule(programId, scheduleName2, false, 30, TimeUnit.SECONDS);
    int workflowRuns = getProgramRuns(programId, "completed").size();
    // Should still be one
    Assert.assertEquals(1, workflowRuns);
    // Sleep for some time and verify there are no more scheduled jobs after the suspend.
    for (int i = 0; i < 12; ++i) {
        response = doPost(String.format("/v3/namespaces/%s/streams/%s", TEST_NAMESPACE2, streamName), longString);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
    }
    TimeUnit.SECONDS.sleep(5);
    int workflowRunsAfterSuspend = getProgramRuns(programId, "completed").size();
    Assert.assertEquals(workflowRuns, workflowRunsAfterSuspend);
    Assert.assertEquals(200, resumeSchedule(TEST_NAMESPACE2, appName, scheduleName1));
    //check scheduled state
    assertSchedule(programId, scheduleName1, true, 30, TimeUnit.SECONDS);
    // an additional run should execute and complete after resuming the schedule
    assertRunHistory(programId, "completed", 1 + workflowRunsAfterSuspend, 60, TimeUnit.SECONDS);
    //Check status of a non existing schedule
    try {
        assertSchedule(programId, "invalid", true, 2, TimeUnit.SECONDS);
        Assert.fail();
    } catch (Exception e) {
    // expected
    }
    Assert.assertEquals(200, suspendSchedule(TEST_NAMESPACE2, appName, scheduleName1));
    //check paused state
    assertSchedule(programId, scheduleName1, false, 30, TimeUnit.SECONDS);
    //Schedule operations using invalid namespace
    try {
        assertSchedule(Id.Program.from(TEST_NAMESPACE1, appName, ProgramType.WORKFLOW, workflowName), scheduleName1, true, 2, TimeUnit.SECONDS);
        Assert.fail();
    } catch (Exception e) {
    // expected
    }
    Assert.assertEquals(404, suspendSchedule(TEST_NAMESPACE1, appName, scheduleName1));
    Assert.assertEquals(404, resumeSchedule(TEST_NAMESPACE1, appName, scheduleName1));
    // Wait until any running jobs just before suspend call completes.
    TimeUnit.SECONDS.sleep(2);
}

Example 2 with StreamProperties

use of co.cask.cdap.proto.StreamProperties in project cdap by caskdata.

the class CreateStreamCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
    String streamId = arguments.get(ArgumentName.NEW_STREAM.toString());
    StreamProperties streamProperties = null;
    if (arguments.hasArgument(ArgumentName.LOCAL_FILE_PATH.toString())) {
        File file = new File(arguments.get(ArgumentName.LOCAL_FILE_PATH.toString()));
        try (Reader reader = Files.newReader(file, Charsets.UTF_8)) {
            streamProperties = GSON.fromJson(reader, StreamProperties.class);
        } catch (FileNotFoundException e) {
            throw new IllegalArgumentException("Not a file: " + file);
        } catch (Exception e) {
            throw new IllegalArgumentException("Stream properties are malformed.", e);
        }
    }
    streamClient.create(cliConfig.getCurrentNamespace().stream(streamId), streamProperties);
    output.printf("Successfully created stream with ID '%s'\n", streamId);
}

Example 3 with StreamProperties

use of co.cask.cdap.proto.StreamProperties in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testSystemMetadataRetrieval.

@Test
public void testSystemMetadataRetrieval() throws Exception {
    appClient.deploy(NamespaceId.DEFAULT, createAppJarFile(AllProgramsApp.class));
    // verify stream system metadata
    StreamId streamId = NamespaceId.DEFAULT.stream(AllProgramsApp.STREAM_NAME);
    Set<String> streamSystemTags = getTags(streamId, MetadataScope.SYSTEM);
    Assert.assertEquals(ImmutableSet.of(AbstractSystemMetadataWriter.EXPLORE_TAG), streamSystemTags);
    Map<String, String> streamSystemProperties = getProperties(streamId, MetadataScope.SYSTEM);
    // Verify create time exists, and is within the past hour
    Assert.assertTrue("Expected creation time to exist but it does not", streamSystemProperties.containsKey(AbstractSystemMetadataWriter.CREATION_TIME_KEY));
    long createTime = Long.parseLong(streamSystemProperties.get(AbstractSystemMetadataWriter.CREATION_TIME_KEY));
    Assert.assertTrue("Stream create time should be within the last hour - " + createTime, createTime > System.currentTimeMillis() - TimeUnit.HOURS.toMillis(1));
    Assert.assertEquals(ImmutableMap.of(AbstractSystemMetadataWriter.SCHEMA_KEY, Schema.recordOf("stringBody", Schema.Field.of("body", Schema.of(Schema.Type.STRING))).toString(), AbstractSystemMetadataWriter.TTL_KEY, String.valueOf(Long.MAX_VALUE), AbstractSystemMetadataWriter.DESCRIPTION_KEY, "test stream", AbstractSystemMetadataWriter.CREATION_TIME_KEY, String.valueOf(createTime), AbstractSystemMetadataWriter.ENTITY_NAME_KEY, streamId.getEntityName()), streamSystemProperties);
    // Update stream properties and verify metadata got updated (except creation time and description)
    long newTtl = 100000L;
    streamClient.setStreamProperties(streamId, new StreamProperties(newTtl, null, null));
    streamSystemProperties = getProperties(streamId, MetadataScope.SYSTEM);
    Assert.assertEquals(ImmutableMap.of(AbstractSystemMetadataWriter.SCHEMA_KEY, Schema.recordOf("stringBody", Schema.Field.of("body", Schema.of(Schema.Type.STRING))).toString(), AbstractSystemMetadataWriter.TTL_KEY, String.valueOf(newTtl * 1000), AbstractSystemMetadataWriter.DESCRIPTION_KEY, "test stream", AbstractSystemMetadataWriter.CREATION_TIME_KEY, String.valueOf(createTime), AbstractSystemMetadataWriter.ENTITY_NAME_KEY, streamId.getEntityName()), streamSystemProperties);
    Set<MetadataRecord> streamSystemMetadata = getMetadata(streamId, MetadataScope.SYSTEM);
    Assert.assertEquals(ImmutableSet.of(new MetadataRecord(streamId, MetadataScope.SYSTEM, streamSystemProperties, streamSystemTags)), streamSystemMetadata);
    // create view and verify view system metadata
    StreamViewId view = new StreamViewId(streamId.getNamespace(), streamId.getStream(), "view");
    Schema viewSchema = Schema.recordOf("record", Schema.Field.of("viewBody", Schema.nullableOf(Schema.of(Schema.Type.BYTES))));
    streamViewClient.createOrUpdate(view, new ViewSpecification(new FormatSpecification("format", viewSchema)));
    ImmutableSet<String> viewUserTags = ImmutableSet.of("viewTag");
    addTags(view, viewUserTags);
    Assert.assertEquals(ImmutableSet.of(new MetadataRecord(view, MetadataScope.USER, ImmutableMap.<String, String>of(), viewUserTags), new MetadataRecord(view, MetadataScope.SYSTEM, ImmutableMap.of(AbstractSystemMetadataWriter.ENTITY_NAME_KEY, view.getEntityName(), AbstractSystemMetadataWriter.SCHEMA_KEY, viewSchema.toString()), ImmutableSet.of(AllProgramsApp.STREAM_NAME))), removeCreationTime(getMetadata(view)));
    // verify dataset system metadata
    DatasetId datasetInstance = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME);
    Set<String> dsSystemTags = getTags(datasetInstance, MetadataScope.SYSTEM);
    Assert.assertEquals(ImmutableSet.of(DatasetSystemMetadataWriter.BATCH_TAG, AbstractSystemMetadataWriter.EXPLORE_TAG), dsSystemTags);
    Map<String, String> dsSystemProperties = getProperties(datasetInstance, MetadataScope.SYSTEM);
    // Verify create time exists, and is within the past hour
    Assert.assertTrue("Expected creation time to exist but it does not", dsSystemProperties.containsKey(AbstractSystemMetadataWriter.CREATION_TIME_KEY));
    createTime = Long.parseLong(dsSystemProperties.get(AbstractSystemMetadataWriter.CREATION_TIME_KEY));
    Assert.assertTrue("Dataset create time should be within the last hour - " + createTime, createTime > System.currentTimeMillis() - TimeUnit.HOURS.toMillis(1));
    // Now remove create time and assert all other system properties
    Assert.assertEquals(ImmutableMap.of("type", KeyValueTable.class.getName(), AbstractSystemMetadataWriter.DESCRIPTION_KEY, "test dataset", AbstractSystemMetadataWriter.CREATION_TIME_KEY, String.valueOf(createTime), AbstractSystemMetadataWriter.ENTITY_NAME_KEY, datasetInstance.getEntityName()), dsSystemProperties);
    //Update properties, and make sure that system metadata gets updated (except create time)
    datasetClient.update(datasetInstance, TableProperties.builder().setTTL(100000L).build().getProperties());
    dsSystemProperties = getProperties(datasetInstance, MetadataScope.SYSTEM);
    Assert.assertEquals(ImmutableMap.of("type", KeyValueTable.class.getName(), AbstractSystemMetadataWriter.DESCRIPTION_KEY, "test dataset", AbstractSystemMetadataWriter.TTL_KEY, "100000", AbstractSystemMetadataWriter.CREATION_TIME_KEY, String.valueOf(createTime), AbstractSystemMetadataWriter.ENTITY_NAME_KEY, datasetInstance.getEntityName()), dsSystemProperties);
    // verify artifact metadata
    ArtifactId artifactId = getArtifactId();
    Assert.assertEquals(ImmutableSet.of(new MetadataRecord(artifactId, MetadataScope.SYSTEM, ImmutableMap.of(AbstractSystemMetadataWriter.ENTITY_NAME_KEY, artifactId.getEntityName()), ImmutableSet.<String>of())), removeCreationTime(getMetadata(artifactId, MetadataScope.SYSTEM)));
    // verify app system metadata
    ApplicationId app = NamespaceId.DEFAULT.app(AllProgramsApp.NAME);
    Assert.assertEquals(ImmutableMap.builder().put(ProgramType.FLOW.getPrettyName() + MetadataDataset.KEYVALUE_SEPARATOR + AllProgramsApp.NoOpFlow.NAME, AllProgramsApp.NoOpFlow.NAME).put(ProgramType.MAPREDUCE.getPrettyName() + MetadataDataset.KEYVALUE_SEPARATOR + AllProgramsApp.NoOpMR.NAME, AllProgramsApp.NoOpMR.NAME).put(ProgramType.MAPREDUCE.getPrettyName() + MetadataDataset.KEYVALUE_SEPARATOR + AllProgramsApp.NoOpMR2.NAME, AllProgramsApp.NoOpMR2.NAME).put(ProgramType.SERVICE.getPrettyName() + MetadataDataset.KEYVALUE_SEPARATOR + AllProgramsApp.NoOpService.NAME, AllProgramsApp.NoOpService.NAME).put(ProgramType.SPARK.getPrettyName() + MetadataDataset.KEYVALUE_SEPARATOR + AllProgramsApp.NoOpSpark.NAME, AllProgramsApp.NoOpSpark.NAME).put(ProgramType.WORKER.getPrettyName() + MetadataDataset.KEYVALUE_SEPARATOR + AllProgramsApp.NoOpWorker.NAME, AllProgramsApp.NoOpWorker.NAME).put(ProgramType.WORKFLOW.getPrettyName() + MetadataDataset.KEYVALUE_SEPARATOR + AllProgramsApp.NoOpWorkflow.NAME, AllProgramsApp.NoOpWorkflow.NAME).put(AbstractSystemMetadataWriter.ENTITY_NAME_KEY, app.getEntityName()).put(AbstractSystemMetadataWriter.VERSION_KEY, ApplicationId.DEFAULT_VERSION).put(AbstractSystemMetadataWriter.DESCRIPTION_KEY, AllProgramsApp.DESCRIPTION).put("schedule" + MetadataDataset.KEYVALUE_SEPARATOR + AllProgramsApp.SCHEDULE_NAME, AllProgramsApp.SCHEDULE_NAME + MetadataDataset.KEYVALUE_SEPARATOR + AllProgramsApp.SCHEDULE_DESCRIPTION).build(), removeCreationTime(getProperties(app, MetadataScope.SYSTEM)));
    Assert.assertEquals(ImmutableSet.of(AllProgramsApp.class.getSimpleName()), getTags(app, MetadataScope.SYSTEM));
    // verify program system metadata
    assertProgramSystemMetadata(app.flow(AllProgramsApp.NoOpFlow.NAME), "Realtime", AllProgramsApp.NoOpFlow.DESCRIPTION);
    assertProgramSystemMetadata(app.worker(AllProgramsApp.NoOpWorker.NAME), "Realtime", null);
    assertProgramSystemMetadata(app.service(AllProgramsApp.NoOpService.NAME), "Realtime", null);
    assertProgramSystemMetadata(app.mr(AllProgramsApp.NoOpMR.NAME), "Batch", null);
    assertProgramSystemMetadata(app.spark(AllProgramsApp.NoOpSpark.NAME), "Batch", null);
    assertProgramSystemMetadata(app.workflow(AllProgramsApp.NoOpWorkflow.NAME), "Batch", AllProgramsApp.NoOpWorkflow.DESCRIPTION);
    // update dataset properties to add the workflow.local.dataset property to it.
    datasetClient.update(datasetInstance, ImmutableMap.of(Constants.AppFabric.WORKFLOW_LOCAL_DATASET_PROPERTY, "true"));
    dsSystemTags = getTags(datasetInstance, MetadataScope.SYSTEM);
    Assert.assertEquals(ImmutableSet.of(DatasetSystemMetadataWriter.BATCH_TAG, AbstractSystemMetadataWriter.EXPLORE_TAG, DatasetSystemMetadataWriter.LOCAL_DATASET_TAG), dsSystemTags);
}

Example 4 with StreamProperties

use of co.cask.cdap.proto.StreamProperties in project cdap by caskdata.

the class StreamAdminTest method testAuditPublish.

@Test
public void testAuditPublish() throws Exception {
    grantAndAssertSuccess(FOO_NAMESPACE, USER, EnumSet.allOf(Action.class));
    // clear existing all messages
    getInMemoryAuditPublisher().popMessages();
    final List<AuditMessage> expectedMessages = new ArrayList<>();
    StreamAdmin streamAdmin = getStreamAdmin();
    StreamId stream1 = FOO_NAMESPACE.stream("stream1");
    streamAdmin.create(stream1);
    expectedMessages.add(new AuditMessage(0, stream1, "", AuditType.CREATE, AuditPayload.EMPTY_PAYLOAD));
    StreamId stream2 = FOO_NAMESPACE.stream("stream2");
    streamAdmin.create(stream2);
    expectedMessages.add(new AuditMessage(0, stream2, "", AuditType.CREATE, AuditPayload.EMPTY_PAYLOAD));
    streamAdmin.truncate(stream1);
    expectedMessages.add(new AuditMessage(0, stream1, "", AuditType.TRUNCATE, AuditPayload.EMPTY_PAYLOAD));
    streamAdmin.updateConfig(stream1, new StreamProperties(100L, new FormatSpecification("f", null), 100));
    expectedMessages.add(new AuditMessage(0, stream1, "", AuditType.UPDATE, AuditPayload.EMPTY_PAYLOAD));
    ProgramRunId run = new ProgramId("ns1", "app", ProgramType.FLOW, "flw").run(RunIds.generate().getId());
    streamAdmin.addAccess(run, stream1, AccessType.READ);
    expectedMessages.add(new AuditMessage(0, stream1, "", AuditType.ACCESS, new AccessPayload(co.cask.cdap.proto.audit.payload.access.AccessType.READ, run)));
    streamAdmin.drop(stream1);
    expectedMessages.add(new AuditMessage(0, stream1, "", AuditType.DELETE, AuditPayload.EMPTY_PAYLOAD));
    streamAdmin.dropAllInNamespace(FOO_NAMESPACE);
    expectedMessages.add(new AuditMessage(0, stream2, "", AuditType.DELETE, AuditPayload.EMPTY_PAYLOAD));
    // Ignore audit messages for system namespace (creation of system datasets, etc)
    final String systemNs = NamespaceId.SYSTEM.getNamespace();
    final Iterable<AuditMessage> actualMessages = Iterables.filter(getInMemoryAuditPublisher().popMessages(), new Predicate<AuditMessage>() {

        @Override
        public boolean apply(AuditMessage input) {
            return !(input.getEntityId() instanceof NamespacedEntityId && ((NamespacedEntityId) input.getEntityId()).getNamespace().equals(systemNs));
        }
    });
    Assert.assertEquals(expectedMessages, Lists.newArrayList(actualMessages));
}

Example 5 with StreamProperties

use of co.cask.cdap.proto.StreamProperties in project cdap by caskdata.

the class StreamAdminTest method testOwner.

@Test
public void testOwner() throws Exception {
    // crate a stream with owner
    StreamAdmin streamAdmin = getStreamAdmin();
    OwnerAdmin ownerAdmin = getOwnerAdmin();
    grantAndAssertSuccess(FOO_NAMESPACE, USER, ImmutableSet.of(Action.WRITE));
    StreamId stream = FOO_NAMESPACE.stream("stream");
    Properties properties = new Properties();
    String ownerPrincipal = "user/somehost@somekdc.net";
    properties.put(Constants.Security.PRINCIPAL, ownerPrincipal);
    streamAdmin.create(stream, properties);
    Assert.assertTrue(streamAdmin.exists(stream));
    // Check that the owner information got stored in owner store
    Assert.assertTrue(ownerAdmin.exists(stream));
    // also verify that we are able to get owner information back in properties
    Assert.assertEquals(ownerPrincipal, streamAdmin.getProperties(stream).getOwnerPrincipal());
    // updating stream owner should fail
    try {
        streamAdmin.updateConfig(stream, new StreamProperties(1L, null, null, null, "user/somekdc.net"));
        Assert.fail();
    } catch (UnauthorizedException e) {
    // expected
    }
    // trying to create same stream with different owner should fail
    properties.put(Constants.Security.PRINCIPAL, "someOtherUser/someHost@somekdc.net");
    try {
        streamAdmin.create(stream, properties);
        Assert.fail("Should have failed to add the same stream with different owner");
    } catch (UnauthorizedException e) {
    // expected
    }
    // ensure that the previous owner still exists
    Assert.assertEquals(ownerPrincipal, streamAdmin.getProperties(stream).getOwnerPrincipal());
    // drop the stream which should also delete the owner info
    streamAdmin.drop(stream);
    Assert.assertFalse(ownerAdmin.exists(stream));
}