Examples with StreamProperties co.cask.cdap.proto.StreamProperties used on opensource projects

Example 11 with StreamProperties

use of co.cask.cdap.proto.StreamProperties in project cdap by caskdata.

the class StreamAdminTest method testAuditPublish.

@Test
public void testAuditPublish() throws Exception {
    grantAndAssertSuccess(FOO_NAMESPACE, USER, EnumSet.allOf(Action.class));
    // clear existing all messages
    getInMemoryAuditPublisher().popMessages();
    final List<AuditMessage> expectedMessages = new ArrayList<>();
    StreamAdmin streamAdmin = getStreamAdmin();
    StreamId stream1 = FOO_NAMESPACE.stream("stream1");
    streamAdmin.create(stream1);
    expectedMessages.add(new AuditMessage(0, stream1, "", AuditType.CREATE, AuditPayload.EMPTY_PAYLOAD));
    StreamId stream2 = FOO_NAMESPACE.stream("stream2");
    streamAdmin.create(stream2);
    expectedMessages.add(new AuditMessage(0, stream2, "", AuditType.CREATE, AuditPayload.EMPTY_PAYLOAD));
    streamAdmin.truncate(stream1);
    expectedMessages.add(new AuditMessage(0, stream1, "", AuditType.TRUNCATE, AuditPayload.EMPTY_PAYLOAD));
    streamAdmin.updateConfig(stream1, new StreamProperties(100L, new FormatSpecification("f", null), 100));
    expectedMessages.add(new AuditMessage(0, stream1, "", AuditType.UPDATE, AuditPayload.EMPTY_PAYLOAD));
    ProgramRunId run = new ProgramId("ns1", "app", ProgramType.FLOW, "flw").run(RunIds.generate().getId());
    streamAdmin.addAccess(run, stream1, AccessType.READ);
    expectedMessages.add(new AuditMessage(0, stream1, "", AuditType.ACCESS, new AccessPayload(co.cask.cdap.proto.audit.payload.access.AccessType.READ, run)));
    streamAdmin.drop(stream1);
    expectedMessages.add(new AuditMessage(0, stream1, "", AuditType.DELETE, AuditPayload.EMPTY_PAYLOAD));
    streamAdmin.dropAllInNamespace(FOO_NAMESPACE);
    expectedMessages.add(new AuditMessage(0, stream2, "", AuditType.DELETE, AuditPayload.EMPTY_PAYLOAD));
    // Ignore audit messages for system namespace (creation of system datasets, etc)
    final String systemNs = NamespaceId.SYSTEM.getNamespace();
    final Iterable<AuditMessage> actualMessages = Iterables.filter(getInMemoryAuditPublisher().popMessages(), new Predicate<AuditMessage>() {

        @Override
        public boolean apply(AuditMessage input) {
            return !(input.getEntityId() instanceof NamespacedEntityId && ((NamespacedEntityId) input.getEntityId()).getNamespace().equals(systemNs));
        }
    });
    Assert.assertEquals(expectedMessages, Lists.newArrayList(actualMessages));
}

Example 12 with StreamProperties

use of co.cask.cdap.proto.StreamProperties in project cdap by caskdata.

the class StreamAdminTest method testOwner.

@Test
public void testOwner() throws Exception {
    // crate a stream with owner
    StreamAdmin streamAdmin = getStreamAdmin();
    OwnerAdmin ownerAdmin = getOwnerAdmin();
    grantAndAssertSuccess(FOO_NAMESPACE, USER, ImmutableSet.of(Action.WRITE));
    StreamId stream = FOO_NAMESPACE.stream("stream");
    Properties properties = new Properties();
    String ownerPrincipal = "user/somehost@somekdc.net";
    properties.put(Constants.Security.PRINCIPAL, ownerPrincipal);
    streamAdmin.create(stream, properties);
    Assert.assertTrue(streamAdmin.exists(stream));
    // Check that the owner information got stored in owner store
    Assert.assertTrue(ownerAdmin.exists(stream));
    // also verify that we are able to get owner information back in properties
    Assert.assertEquals(ownerPrincipal, streamAdmin.getProperties(stream).getOwnerPrincipal());
    // updating stream owner should fail
    try {
        streamAdmin.updateConfig(stream, new StreamProperties(1L, null, null, null, "user/somekdc.net"));
        Assert.fail();
    } catch (UnauthorizedException e) {
    // expected
    }
    // trying to create same stream with different owner should fail
    properties.put(Constants.Security.PRINCIPAL, "someOtherUser/someHost@somekdc.net");
    try {
        streamAdmin.create(stream, properties);
        Assert.fail("Should have failed to add the same stream with different owner");
    } catch (UnauthorizedException e) {
    // expected
    }
    // ensure that the previous owner still exists
    Assert.assertEquals(ownerPrincipal, streamAdmin.getProperties(stream).getOwnerPrincipal());
    // drop the stream which should also delete the owner info
    streamAdmin.drop(stream);
    Assert.assertFalse(ownerAdmin.exists(stream));
}

Example 13 with StreamProperties

use of co.cask.cdap.proto.StreamProperties in project cdap by caskdata.

the class FileStreamAdmin method updateConfig.

@Override
public void updateConfig(final StreamId streamId, final StreamProperties properties) throws Exception {
    Location streamLocation;
    // User should have admin access on the stream to update its configuration
    ensureAccess(streamId, Action.ADMIN);
    streamLocation = impersonator.doAs(streamId.getParent(), new Callable<Location>() {

        @Override
        public Location call() throws Exception {
            return getStreamLocation(streamId);
        }
    });
    Preconditions.checkArgument(streamLocation.isDirectory(), "Stream '%s' does not exist.", streamId);
    SecurityUtil.verifyOwnerPrincipal(streamId, properties.getOwnerPrincipal(), ownerAdmin);
    streamCoordinatorClient.updateProperties(streamId, new Callable<CoordinatorStreamProperties>() {

        @Override
        public CoordinatorStreamProperties call() throws Exception {
            StreamProperties oldProperties = updateProperties(streamId, properties);
            FormatSpecification format = properties.getFormat();
            if (format != null) {
                // if the schema has changed, we need to recreate the hive table.
                // Changes in format and settings don't require
                // a hive change, as they are just properties used by the stream storage handler.
                Schema currSchema = oldProperties.getFormat().getSchema();
                Schema newSchema = format.getSchema();
                if (!Objects.equals(currSchema, newSchema)) {
                    alterExploreStream(streamId, false, null);
                    alterExploreStream(streamId, true, format);
                }
            }
            publishAudit(streamId, AuditType.UPDATE);
            return new CoordinatorStreamProperties(properties.getTTL(), properties.getFormat(), properties.getNotificationThresholdMB(), null, properties.getDescription(), properties.getOwnerPrincipal());
        }
    });
}

Example 14 with StreamProperties

use of co.cask.cdap.proto.StreamProperties in project cdap by caskdata.

the class FileStreamAdmin method updateProperties.

private StreamProperties updateProperties(StreamId streamId, StreamProperties properties) throws Exception {
    StreamConfig config = getConfig(streamId);
    StreamConfig.Builder builder = StreamConfig.builder(config);
    if (properties.getTTL() != null) {
        builder.setTTL(properties.getTTL());
    }
    if (properties.getFormat() != null) {
        builder.setFormatSpec(properties.getFormat());
    }
    if (properties.getNotificationThresholdMB() != null) {
        builder.setNotificationThreshold(properties.getNotificationThresholdMB());
    }
    // update stream description
    String description = properties.getDescription();
    if (description != null) {
        streamMetaStore.addStream(streamId, description);
    }
    final StreamConfig newConfig = builder.build();
    impersonator.doAs(streamId, new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            writeConfig(newConfig);
            return null;
        }
    });
    // Update system metadata for stream
    SystemMetadataWriter systemMetadataWriter = new StreamSystemMetadataWriter(metadataStore, streamId, newConfig, description);
    systemMetadataWriter.write();
    return new StreamProperties(config.getTTL(), config.getFormat(), config.getNotificationThresholdMB());
}

Example 15 with StreamProperties

use of co.cask.cdap.proto.StreamProperties in project cdap by caskdata.

the class FileStreamAdmin method getProperties.

@Override
public StreamProperties getProperties(StreamId streamId) throws Exception {
    // User should have any access on the stream to read its properties
    ensureAccess(streamId);
    // get the principal which will be used for impersonation to display as owner
    String ownerPrincipal = ownerAdmin.getOwnerPrincipal(streamId);
    StreamConfig config = getConfig(streamId);
    StreamSpecification spec = streamMetaStore.getStream(streamId);
    return new StreamProperties(config.getTTL(), config.getFormat(), config.getNotificationThresholdMB(), spec.getDescription(), ownerPrincipal);
}