Examples with StreamViewId co.cask.cdap.proto.id.StreamViewId used on opensource projects

Example 16 with StreamViewId

use of co.cask.cdap.proto.id.StreamViewId in project cdap by caskdata.

the class FileStreamAdmin method doDrop.

private void doDrop(final StreamId streamId, final Location streamLocation) throws Exception {
    // Delete the stream config so that calls that try to access the stream will fail after this call returns.
    // The stream coordinator client will notify all clients that stream has been deleted.
    streamCoordinatorClient.deleteStream(streamId, new Runnable() {

        @Override
        public void run() {
            try {
                final Location configLocation = impersonator.doAs(streamId, new Callable<Location>() {

                    @Override
                    public Location call() throws Exception {
                        Location configLocation = getConfigLocation(streamId);
                        return configLocation.exists() ? configLocation : null;
                    }
                });
                if (configLocation == null) {
                    return;
                }
                alterExploreStream(streamId.getParent().stream(StreamUtils.getStreamNameFromLocation(streamLocation)), false, null);
                // Drop the associated views
                List<StreamViewId> views = viewAdmin.list(streamId);
                for (StreamViewId view : views) {
                    viewAdmin.delete(view);
                }
                impersonator.doAs(streamId, new Callable<Void>() {

                    @Override
                    public Void call() throws Exception {
                        if (!configLocation.delete()) {
                            LOG.debug("Could not delete stream config location {}", streamLocation);
                        }
                        // Move the stream directory to the deleted directory
                        // The target directory has a timestamp appended to the stream name
                        // It is for the case when a stream is created and deleted in a short period of time before
                        // the stream janitor kicks in.
                        Location deleted = StreamUtils.getDeletedLocation(getStreamBaseLocation(streamId.getParent()));
                        Locations.mkdirsIfNotExists(deleted);
                        streamLocation.renameTo(deleted.append(streamId.getEntityName() + System.currentTimeMillis()));
                        return null;
                    }
                });
                streamMetaStore.removeStream(streamId);
                ownerAdmin.delete(streamId);
                metadataStore.removeMetadata(streamId);
                // revoke all privileges on the stream
                privilegesManager.revoke(streamId);
                publishAudit(streamId, AuditType.DELETE);
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }
        }
    });
}

Example 17 with StreamViewId

use of co.cask.cdap.proto.id.StreamViewId in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testInvalidEntities.

@Test
public void testInvalidEntities() throws IOException {
    ProgramId nonExistingProgram = application.service("NonExistingService");
    DatasetId nonExistingDataset = new DatasetId(NamespaceId.DEFAULT.getNamespace(), "NonExistingDataset");
    StreamId nonExistingStream = NamespaceId.DEFAULT.stream("NonExistingStream");
    StreamViewId nonExistingView = nonExistingStream.view("NonExistingView");
    ApplicationId nonExistingApp = NamespaceId.DEFAULT.app("NonExistingApp");
    Map<String, String> properties = ImmutableMap.of("aKey", "aValue", "aK", "aV");
    addProperties(nonExistingApp, properties, NotFoundException.class);
    addProperties(nonExistingProgram, properties, NotFoundException.class);
    addProperties(nonExistingDataset, properties, NotFoundException.class);
    addProperties(nonExistingView, properties, NotFoundException.class);
    addProperties(nonExistingStream, properties, NotFoundException.class);
}

Example 18 with StreamViewId

use of co.cask.cdap.proto.id.StreamViewId in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testSearchResultPagination.

@Test
public void testSearchResultPagination() throws Exception {
    NamespaceId namespace = new NamespaceId("pagination");
    namespaceClient.create(new NamespaceMeta.Builder().setName(namespace).build());
    StreamId stream = namespace.stream("text");
    DatasetId dataset = namespace.dataset("mydataset");
    StreamViewId view = stream.view("view");
    DatasetId trackerDataset = namespace.dataset("_auditLog");
    // create entities so system metadata is annotated
    streamClient.create(stream);
    streamViewClient.createOrUpdate(view, new ViewSpecification(new FormatSpecification("csv", null, null)));
    datasetClient.create(dataset, new DatasetInstanceConfiguration(Table.class.getName(), Collections.<String, String>emptyMap()));
    datasetClient.create(trackerDataset, new DatasetInstanceConfiguration(Table.class.getName(), Collections.<String, String>emptyMap()));
    // search with showHidden to true
    EnumSet<EntityTypeSimpleName> targets = EnumSet.allOf(EntityTypeSimpleName.class);
    String sort = AbstractSystemMetadataWriter.ENTITY_NAME_KEY + " asc";
    // search to get all the above entities offset 0, limit interger max  and cursors 0
    MetadataSearchResponse searchResponse = searchMetadata(namespace, "*", targets, sort, 0, Integer.MAX_VALUE, 0, null, true);
    List<MetadataSearchResultRecord> expectedResults = ImmutableList.of(new MetadataSearchResultRecord(trackerDataset), new MetadataSearchResultRecord(dataset), new MetadataSearchResultRecord(stream), new MetadataSearchResultRecord(view));
    List<String> expectedCursors = ImmutableList.of();
    Assert.assertEquals(expectedResults, new ArrayList<>(searchResponse.getResults()));
    Assert.assertEquals(expectedCursors, searchResponse.getCursors());
    // no offset, limit 1, no cursors
    searchResponse = searchMetadata(namespace, "*", targets, sort, 0, 1, 0, null);
    expectedResults = ImmutableList.of(new MetadataSearchResultRecord(dataset));
    expectedCursors = ImmutableList.of();
    Assert.assertEquals(expectedResults, new ArrayList<>(searchResponse.getResults()));
    Assert.assertEquals(expectedCursors, searchResponse.getCursors());
    // no offset, limit 1, 2 cursors, should return 1st result, with 2 cursors
    searchResponse = searchMetadata(namespace, "*", targets, sort, 0, 1, 2, null);
    expectedResults = ImmutableList.of(new MetadataSearchResultRecord(dataset));
    expectedCursors = ImmutableList.of(stream.getEntityName(), view.getEntityName());
    Assert.assertEquals(expectedResults, new ArrayList<>(searchResponse.getResults()));
    Assert.assertEquals(expectedCursors, searchResponse.getCursors());
    // offset 1, limit 1, 2 cursors, should return 2nd result, with only 1 cursor since we don't have enough data
    searchResponse = searchMetadata(namespace, "*", targets, sort, 1, 1, 2, null);
    expectedResults = ImmutableList.of(new MetadataSearchResultRecord(stream));
    expectedCursors = ImmutableList.of(view.getEntityName());
    Assert.assertEquals(expectedResults, new ArrayList<>(searchResponse.getResults()));
    Assert.assertEquals(expectedCursors, searchResponse.getCursors());
    // offset 2, limit 1, 2 cursors, should return 3rd result, with 0 cursors since we don't have enough data
    searchResponse = searchMetadata(namespace, "*", targets, sort, 2, 1, 2, null);
    expectedResults = ImmutableList.of(new MetadataSearchResultRecord(view));
    Assert.assertEquals(expectedResults, new ArrayList<>(searchResponse.getResults()));
    Assert.assertTrue(searchResponse.getCursors().isEmpty());
    // offset 3, limit 1, 2 cursors, should 0 results, with 0 cursors since we don't have enough data
    searchResponse = searchMetadata(namespace, "*", targets, sort, 3, 1, 2, null);
    Assert.assertTrue(searchResponse.getResults().isEmpty());
    Assert.assertTrue(searchResponse.getCursors().isEmpty());
    // no offset, no limit, should return everything
    searchResponse = searchMetadata(namespace, "*", targets, sort, 0, Integer.MAX_VALUE, 4, null);
    expectedResults = ImmutableList.of(new MetadataSearchResultRecord(dataset), new MetadataSearchResultRecord(stream), new MetadataSearchResultRecord(view));
    Assert.assertEquals(expectedResults, new ArrayList<>(searchResponse.getResults()));
    Assert.assertTrue(searchResponse.getCursors().isEmpty());
    // cleanup
    namespaceClient.delete(namespace);
}

Example 19 with StreamViewId

use of co.cask.cdap.proto.id.StreamViewId in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testSearchResultSorting.

@Test
public void testSearchResultSorting() throws Exception {
    NamespaceId namespace = new NamespaceId("sorting");
    namespaceClient.create(new NamespaceMeta.Builder().setName(namespace).build());
    StreamId stream = namespace.stream("text");
    DatasetId dataset = namespace.dataset("mydataset");
    StreamViewId view = stream.view("view");
    // create entities so system metadata is annotated
    // also ensure that they are created at least 1 ms apart
    streamClient.create(stream);
    TimeUnit.MILLISECONDS.sleep(1);
    streamViewClient.createOrUpdate(view, new ViewSpecification(new FormatSpecification("csv", null, null)));
    TimeUnit.MILLISECONDS.sleep(1);
    datasetClient.create(dataset, new DatasetInstanceConfiguration(Table.class.getName(), Collections.<String, String>emptyMap()));
    // search with bad sort param
    EnumSet<EntityTypeSimpleName> targets = EnumSet.allOf(EntityTypeSimpleName.class);
    // test ascending order of entity name
    Set<MetadataSearchResultRecord> searchResults = searchMetadata(namespace, "*", targets, AbstractSystemMetadataWriter.ENTITY_NAME_KEY + " asc");
    List<MetadataSearchResultRecord> expected = ImmutableList.of(new MetadataSearchResultRecord(dataset), new MetadataSearchResultRecord(stream), new MetadataSearchResultRecord(view));
    Assert.assertEquals(expected, new ArrayList<>(searchResults));
    // test descending order of entity name
    searchResults = searchMetadata(namespace, "*", targets, AbstractSystemMetadataWriter.ENTITY_NAME_KEY + " desc");
    expected = ImmutableList.of(new MetadataSearchResultRecord(view), new MetadataSearchResultRecord(stream), new MetadataSearchResultRecord(dataset));
    Assert.assertEquals(expected, new ArrayList<>(searchResults));
    // test ascending order of creation time
    searchResults = searchMetadata(namespace, "*", targets, AbstractSystemMetadataWriter.CREATION_TIME_KEY + " asc");
    expected = ImmutableList.of(new MetadataSearchResultRecord(stream), new MetadataSearchResultRecord(view), new MetadataSearchResultRecord(dataset));
    Assert.assertEquals(expected, new ArrayList<>(searchResults));
    // test descending order of creation time
    searchResults = searchMetadata(namespace, "*", targets, AbstractSystemMetadataWriter.CREATION_TIME_KEY + " desc");
    expected = ImmutableList.of(new MetadataSearchResultRecord(dataset), new MetadataSearchResultRecord(view), new MetadataSearchResultRecord(stream));
    Assert.assertEquals(expected, new ArrayList<>(searchResults));
    // cleanup
    namespaceClient.delete(namespace);
}

Example 20 with StreamViewId

use of co.cask.cdap.proto.id.StreamViewId in project cdap by caskdata.

the class ViewSystemMetadataWriterTest method getViewMetadataSchema.

private String getViewMetadataSchema(String format, @Nullable Schema schema) {
    StreamViewId viewId = NamespaceId.DEFAULT.stream("mystream").view("myview");
    FormatSpecification formatSpec = new FormatSpecification(format, schema);
    ViewSpecification viewSpec = new ViewSpecification(formatSpec);
    NoOpMetadataStore metadataStore = new NoOpMetadataStore();
    ViewSystemMetadataWriter writer = new ViewSystemMetadataWriter(metadataStore, viewId, viewSpec, false);
    return writer.getSchemaToAdd();
}