Examples with StreamViewId co.cask.cdap.proto.id.StreamViewId used on opensource projects

Example 6 with StreamViewId

use of co.cask.cdap.proto.id.StreamViewId in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testSearchMetadataDeleteNamespace.

@Test
public void testSearchMetadataDeleteNamespace() throws Exception {
    NamespaceId namespace = new NamespaceId("ns2");
    namespaceClient.create(new NamespaceMeta.Builder().setName(namespace).build());
    // Deploy app
    appClient.deploy(namespace, createAppJarFile(WordCountApp.class, WordCountApp.class.getSimpleName(), "1.0"));
    Set<String> tags = ImmutableSet.of("tag1", "tag2");
    ArtifactId artifact = namespace.artifact("WordCountApp", "1.0");
    ApplicationId app = namespace.app("WordCountApp");
    ProgramId flow = app.flow("WordCountFlow");
    ProgramId service = app.service("WordFrequencyService");
    StreamId stream = namespace.stream("text");
    DatasetId datasetInstance = namespace.dataset("mydataset");
    StreamViewId view = stream.view("view");
    streamViewClient.createOrUpdate(view, new ViewSpecification(new FormatSpecification("csv", null, null)));
    // Add metadata
    addTags(app, tags);
    addTags(flow, tags);
    addTags(stream, tags);
    addTags(datasetInstance, tags);
    addTags(view, tags);
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(stream), new MetadataSearchResultRecord(view)), searchMetadata(namespace, "text"));
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(datasetInstance)), searchMetadata(namespace, "mydataset"));
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(app), new MetadataSearchResultRecord(flow), new MetadataSearchResultRecord(artifact), new MetadataSearchResultRecord(service)), searchMetadata(namespace, "word*"));
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(app), new MetadataSearchResultRecord(flow), new MetadataSearchResultRecord(stream), new MetadataSearchResultRecord(datasetInstance), new MetadataSearchResultRecord(view)), searchMetadata(namespace, "tag1"));
    // Delete namespace
    namespaceClient.delete(namespace);
    // Assert no metadata
    Assert.assertEquals(ImmutableSet.of(), searchMetadata(namespace, "text"));
    Assert.assertEquals(ImmutableSet.of(), searchMetadata(namespace, "mydataset"));
    Assert.assertEquals(ImmutableSet.of(), searchMetadata(namespace, "word*"));
    Assert.assertEquals(ImmutableSet.of(), searchMetadata(namespace, "tag1"));
}

Example 7 with StreamViewId

use of co.cask.cdap.proto.id.StreamViewId in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method assertDataEntitySearch.

private void assertDataEntitySearch() throws Exception {
    DatasetId datasetInstance = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME);
    DatasetId datasetInstance2 = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME2);
    DatasetId datasetInstance3 = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME3);
    DatasetId datasetInstance4 = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME4);
    DatasetId datasetInstance5 = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME5);
    DatasetId datasetInstance6 = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME6);
    DatasetId datasetInstance7 = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME7);
    DatasetId dsWithSchema = NamespaceId.DEFAULT.dataset(AllProgramsApp.DS_WITH_SCHEMA_NAME);
    StreamId streamId = NamespaceId.DEFAULT.stream(AllProgramsApp.STREAM_NAME);
    StreamViewId view = streamId.view("view");
    Set<MetadataSearchResultRecord> expected = ImmutableSet.of(new MetadataSearchResultRecord(streamId), new MetadataSearchResultRecord(mystream));
    Set<MetadataSearchResultRecord> expectedWithView = ImmutableSet.<MetadataSearchResultRecord>builder().addAll(expected).add(new MetadataSearchResultRecord(myview)).build();
    // schema search with fieldname
    Set<MetadataSearchResultRecord> metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, "body");
    Assert.assertEquals(expectedWithView, metadataSearchResultRecords);
    // schema search with fieldname and fieldtype
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, "body:" + Schema.Type.STRING.toString());
    Assert.assertEquals(expected, metadataSearchResultRecords);
    // schema search for partial fieldname
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, "bo*");
    Assert.assertEquals(expectedWithView, metadataSearchResultRecords);
    // schema search with fieldname and all/partial fieldtype
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, "body:STR*");
    Assert.assertEquals(expected, metadataSearchResultRecords);
    // schema search for a field with the given fieldname:fieldtype
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, "body:STRING+field1:STRING");
    Assert.assertEquals(ImmutableSet.<MetadataSearchResultRecord>builder().addAll(expected).add(new MetadataSearchResultRecord(dsWithSchema)).build(), metadataSearchResultRecords);
    // create a view
    Schema viewSchema = Schema.recordOf("record", Schema.Field.of("viewBody", Schema.nullableOf(Schema.of(Schema.Type.BYTES))));
    streamViewClient.createOrUpdate(view, new ViewSpecification(new FormatSpecification("format", viewSchema)));
    // search all entities that have a defined schema
    // add a user property with "schema" as key
    Map<String, String> datasetProperties = ImmutableMap.of("schema", "schemaValue");
    addProperties(datasetInstance, datasetProperties);
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, "schema:*");
    Assert.assertEquals(ImmutableSet.<MetadataSearchResultRecord>builder().addAll(expectedWithView).add(new MetadataSearchResultRecord(datasetInstance)).add(new MetadataSearchResultRecord(dsWithSchema)).add(new MetadataSearchResultRecord(view)).build(), metadataSearchResultRecords);
    // search dataset
    ImmutableSet<MetadataSearchResultRecord> expectedKvTables = ImmutableSet.of(new MetadataSearchResultRecord(datasetInstance), new MetadataSearchResultRecord(datasetInstance2), new MetadataSearchResultRecord(datasetInstance3), new MetadataSearchResultRecord(myds));
    ImmutableSet<MetadataSearchResultRecord> expectedExplorableDatasets = ImmutableSet.<MetadataSearchResultRecord>builder().addAll(expectedKvTables).add(new MetadataSearchResultRecord(datasetInstance4)).add(new MetadataSearchResultRecord(datasetInstance5)).add(new MetadataSearchResultRecord(dsWithSchema)).build();
    ImmutableSet<MetadataSearchResultRecord> expectedAllDatasets = ImmutableSet.<MetadataSearchResultRecord>builder().addAll(expectedExplorableDatasets).add(new MetadataSearchResultRecord(datasetInstance6)).add(new MetadataSearchResultRecord(datasetInstance7)).build();
    ImmutableSet<MetadataSearchResultRecord> expectedExplorables = ImmutableSet.<MetadataSearchResultRecord>builder().addAll(expectedExplorableDatasets).add(new MetadataSearchResultRecord(streamId)).add(new MetadataSearchResultRecord(mystream)).build();
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, "explore");
    Assert.assertEquals(expectedExplorables, metadataSearchResultRecords);
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, KeyValueTable.class.getName());
    Assert.assertEquals(expectedKvTables, metadataSearchResultRecords);
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, "type:*");
    Assert.assertEquals(expectedAllDatasets, metadataSearchResultRecords);
    // search using ttl
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, "ttl:*");
    Assert.assertEquals(expected, metadataSearchResultRecords);
    // search using names
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, AllProgramsApp.STREAM_NAME);
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(streamId), new MetadataSearchResultRecord(view)), metadataSearchResultRecords);
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, AllProgramsApp.STREAM_NAME, EntityTypeSimpleName.STREAM);
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(streamId)), metadataSearchResultRecords);
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, AllProgramsApp.STREAM_NAME, EntityTypeSimpleName.VIEW);
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(view)), metadataSearchResultRecords);
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, "view", EntityTypeSimpleName.VIEW);
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(view)), metadataSearchResultRecords);
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, AllProgramsApp.DATASET_NAME);
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(datasetInstance)), metadataSearchResultRecords);
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, AllProgramsApp.DS_WITH_SCHEMA_NAME);
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(dsWithSchema)), metadataSearchResultRecords);
}

Example 8 with StreamViewId

use of co.cask.cdap.proto.id.StreamViewId in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testSearchMetadataDelete.

@Test
public void testSearchMetadataDelete() throws Exception {
    NamespaceId namespace = new NamespaceId("ns1");
    namespaceClient.create(new NamespaceMeta.Builder().setName(namespace).build());
    // Deploy app
    appClient.deploy(namespace, createAppJarFile(WordCountApp.class, WordCountApp.class.getSimpleName(), "1.0"));
    Set<String> tags = ImmutableSet.of("tag1", "tag2");
    ArtifactId artifact = namespace.artifact("WordCountApp", "1.0");
    ApplicationId app = namespace.app("WordCountApp");
    ProgramId flow = app.flow("WordCountFlow");
    ProgramId service = app.service("WordFrequencyService");
    StreamId stream = namespace.stream("text");
    DatasetId datasetInstance = namespace.dataset("mydataset");
    StreamViewId view = stream.view("view");
    streamViewClient.createOrUpdate(view, new ViewSpecification(new FormatSpecification("csv", null, null)));
    // Add metadata
    addTags(app, tags);
    addTags(flow, tags);
    addTags(stream, tags);
    addTags(datasetInstance, tags);
    addTags(view, tags);
    // Assert metadata
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(stream), new MetadataSearchResultRecord(view)), searchMetadata(namespace, "text"));
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(datasetInstance)), searchMetadata(namespace, "mydataset"));
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(app), new MetadataSearchResultRecord(flow), new MetadataSearchResultRecord(artifact), new MetadataSearchResultRecord(service)), searchMetadata(namespace, "word*"));
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(app), new MetadataSearchResultRecord(flow), new MetadataSearchResultRecord(stream), new MetadataSearchResultRecord(datasetInstance), new MetadataSearchResultRecord(view)), searchMetadata(namespace, "tag1"));
    // Delete entities
    appClient.delete(app);
    streamViewClient.delete(view);
    streamClient.delete(stream);
    datasetClient.delete(datasetInstance);
    artifactClient.delete(artifact);
    // Assert no metadata
    Assert.assertEquals(ImmutableSet.of(), searchMetadata(namespace, "text"));
    Assert.assertEquals(ImmutableSet.of(), searchMetadata(namespace, "mydataset"));
    Assert.assertEquals(ImmutableSet.of(), searchMetadata(namespace, "word*"));
    Assert.assertEquals(ImmutableSet.of(), searchMetadata(namespace, "tag1"));
}

Example 9 with StreamViewId

use of co.cask.cdap.proto.id.StreamViewId in project cdap by caskdata.

the class EntityIdKeyHelper method getTargetIdIdFromKey.

public static NamespacedEntityId getTargetIdIdFromKey(MDSKey.Splitter keySplitter, String type) {
    if (type.equals(TYPE_MAP.get(NamespaceId.class))) {
        String namespaceId = keySplitter.getString();
        return new NamespaceId(namespaceId);
    } else if (type.equals(TYPE_MAP.get(ProgramId.class))) {
        String namespaceId = keySplitter.getString();
        String appId = keySplitter.getString();
        String programType = keySplitter.getString();
        String programId = keySplitter.getString();
        return new ProgramId(namespaceId, appId, programType, programId);
    } else if (type.equals(TYPE_MAP.get(ApplicationId.class))) {
        String namespaceId = keySplitter.getString();
        String appId = keySplitter.getString();
        return new ApplicationId(namespaceId, appId);
    } else if (type.equals(TYPE_MAP.get(ArtifactId.class))) {
        String namespaceId = keySplitter.getString();
        String name = keySplitter.getString();
        String version = keySplitter.getString();
        return new ArtifactId(namespaceId, name, version);
    } else if (type.equals(TYPE_MAP.get(DatasetId.class))) {
        String namespaceId = keySplitter.getString();
        String instanceId = keySplitter.getString();
        return new DatasetId(namespaceId, instanceId);
    } else if (type.equals(TYPE_MAP.get(StreamId.class))) {
        String namespaceId = keySplitter.getString();
        String instanceId = keySplitter.getString();
        return new StreamId(namespaceId, instanceId);
    } else if (type.equals(TYPE_MAP.get(StreamViewId.class))) {
        String namespaceId = keySplitter.getString();
        String streamId = keySplitter.getString();
        String viewId = keySplitter.getString();
        return new StreamViewId(namespaceId, streamId, viewId);
    }
    throw new IllegalArgumentException("Illegal Type " + type + " of metadata source.");
}

Example 10 with StreamViewId

use of co.cask.cdap.proto.id.StreamViewId in project cdap by caskdata.

the class FileStreamAdmin method doDrop.

private void doDrop(final StreamId streamId, final Location streamLocation) throws Exception {
    // Delete the stream config so that calls that try to access the stream will fail after this call returns.
    // The stream coordinator client will notify all clients that stream has been deleted.
    streamCoordinatorClient.deleteStream(streamId, new Runnable() {

        @Override
        public void run() {
            try {
                final Location configLocation = impersonator.doAs(streamId, new Callable<Location>() {

                    @Override
                    public Location call() throws Exception {
                        Location configLocation = getConfigLocation(streamId);
                        return configLocation.exists() ? configLocation : null;
                    }
                });
                if (configLocation == null) {
                    return;
                }
                alterExploreStream(streamId.getParent().stream(StreamUtils.getStreamNameFromLocation(streamLocation)), false, null);
                // Drop the associated views
                List<StreamViewId> views = viewAdmin.list(streamId);
                for (StreamViewId view : views) {
                    viewAdmin.delete(view);
                }
                impersonator.doAs(streamId, new Callable<Void>() {

                    @Override
                    public Void call() throws Exception {
                        if (!configLocation.delete()) {
                            LOG.debug("Could not delete stream config location {}", streamLocation);
                        }
                        // Move the stream directory to the deleted directory
                        // The target directory has a timestamp appended to the stream name
                        // It is for the case when a stream is created and deleted in a short period of time before
                        // the stream janitor kicks in.
                        Location deleted = StreamUtils.getDeletedLocation(getStreamBaseLocation(streamId.getParent()));
                        Locations.mkdirsIfNotExists(deleted);
                        streamLocation.renameTo(deleted.append(streamId.getEntityName() + System.currentTimeMillis()));
                        return null;
                    }
                });
                streamMetaStore.removeStream(streamId);
                ownerAdmin.delete(streamId);
                metadataStore.removeMetadata(streamId);
                // revoke all privileges on the stream
                privilegesManager.revoke(streamId);
                publishAudit(streamId, AuditType.DELETE);
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }
        }
    });
}