Examples with StreamSpecification co.cask.cdap.api.data.stream.StreamSpecification used on opensource projects

Example 6 with StreamSpecification

use of co.cask.cdap.api.data.stream.StreamSpecification in project cdap by caskdata.

the class DistributedStreamService method createRequirementModifier.

/**
   * Creates a {@link ResourceModifier} that updates stream resource requirement by consulting stream meta store.
   */
private ResourceModifier createRequirementModifier() {
    return new ResourceModifier() {

        @Nullable
        @Override
        public ResourceRequirement apply(@Nullable ResourceRequirement existingRequirement) {
            try {
                // Create one requirement for the resource coordinator for all the streams.
                // One stream is identified by one partition
                ResourceRequirement.Builder builder = ResourceRequirement.builder(Constants.Service.STREAMS);
                for (Map.Entry<NamespaceId, StreamSpecification> streamSpecEntry : streamMetaStore.listStreams().entries()) {
                    StreamId streamId = streamSpecEntry.getKey().stream(streamSpecEntry.getValue().getName());
                    LOG.debug("Adding {} stream as a resource to the coordinator to manager streams leaders.", streamId);
                    builder.addPartition(new ResourceRequirement.Partition(streamId.toString(), 1));
                }
                return builder.build();
            } catch (Throwable e) {
                LOG.warn("Could not create requirement for coordinator in Stream handler leader: " + e.getMessage());
                LOG.debug("Could not create requirement for coordinator in Stream handler leader", e);
                throw Throwables.propagate(e);
            }
        }
    };
}

Example 7 with StreamSpecification

use of co.cask.cdap.api.data.stream.StreamSpecification in project cdap by caskdata.

the class StreamHandler method listStreams.

@GET
@Path("/")
public void listStreams(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws Exception {
    // Check for namespace existence. Throws NotFoundException if namespace doesn't exist
    namespaceQueryAdmin.get(new NamespaceId(namespaceId));
    List<StreamSpecification> specifications = streamAdmin.listStreams(new NamespaceId(namespaceId));
    List<StreamDetail> streamDetails = new ArrayList<>(specifications.size());
    for (StreamSpecification specification : specifications) {
        streamDetails.add(new StreamDetail(specification.getName()));
    }
    responder.sendJson(HttpResponseStatus.OK, streamDetails);
}

Example 8 with StreamSpecification

use of co.cask.cdap.api.data.stream.StreamSpecification in project cdap by caskdata.

the class StreamAdminTest method testListStreams.

@Test
public void testListStreams() throws Exception {
    StreamAdmin streamAdmin = getStreamAdmin();
    NamespaceId nsId = FOO_NAMESPACE;
    grantAndAssertSuccess(nsId, USER, EnumSet.allOf(Action.class));
    StreamId s1 = nsId.stream("s1");
    StreamId s2 = nsId.stream("s2");
    List<StreamSpecification> specifications = streamAdmin.listStreams(nsId);
    Assert.assertTrue(specifications.isEmpty());
    streamAdmin.create(s1);
    streamAdmin.create(s2);
    specifications = streamAdmin.listStreams(nsId);
    Assert.assertEquals(2, specifications.size());
    // Revoke all privileges on s1.
    revokeAndAssertSuccess(s1, USER, EnumSet.allOf(Action.class));
    // User should still be able to list both streams because it has all privilege on the parent
    specifications = streamAdmin.listStreams(nsId);
    Assert.assertEquals(2, specifications.size());
    Set<String> streamNames = ImmutableSet.of(s1.getStream(), s2.getStream());
    Assert.assertTrue(streamNames.contains(specifications.get(0).getName()));
    Assert.assertTrue(streamNames.contains(specifications.get(1).getName()));
    // Revoke all privileges on s2.
    revokeAndAssertSuccess(s2, USER, EnumSet.allOf(Action.class));
    // User should still be able to list both streams because it has all privilege on the parent
    specifications = streamAdmin.listStreams(nsId);
    Assert.assertEquals(2, specifications.size());
    Assert.assertTrue(streamNames.contains(specifications.get(0).getName()));
    Assert.assertTrue(streamNames.contains(specifications.get(1).getName()));
    // Revoke all privileges on the namespace
    revokeAndAssertSuccess(nsId, USER, EnumSet.allOf(Action.class));
    // User shouldn't be able to see any streams
    specifications = streamAdmin.listStreams(nsId);
    Assert.assertTrue(specifications.isEmpty());
    grantAndAssertSuccess(s1, USER, EnumSet.allOf(Action.class));
    grantAndAssertSuccess(s2, USER, EnumSet.allOf(Action.class));
    streamAdmin.drop(s1);
    streamAdmin.drop(s2);
}

Example 9 with StreamSpecification

use of co.cask.cdap.api.data.stream.StreamSpecification in project cdap by caskdata.

the class ApplicationSpecificationCodec method deserialize.

@Override
public ApplicationSpecification deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
    JsonObject jsonObj = json.getAsJsonObject();
    String name = jsonObj.get("name").getAsString();
    String appVersion = ApplicationId.DEFAULT_VERSION;
    if (jsonObj.has("appVersion")) {
        appVersion = jsonObj.get("appVersion").getAsString();
    }
    String description = jsonObj.get("description").getAsString();
    String configuration = null;
    if (jsonObj.has("configuration")) {
        configuration = jsonObj.get("configuration").getAsString();
    }
    ArtifactId artifactId = context.deserialize(jsonObj.get("artifactId"), ArtifactId.class);
    Map<String, StreamSpecification> streams = deserializeMap(jsonObj.get("streams"), context, StreamSpecification.class);
    Map<String, String> datasetModules = deserializeMap(jsonObj.get("datasetModules"), context, String.class);
    Map<String, DatasetCreationSpec> datasetInstances = deserializeMap(jsonObj.get("datasetInstances"), context, DatasetCreationSpec.class);
    Map<String, FlowSpecification> flows = deserializeMap(jsonObj.get("flows"), context, FlowSpecification.class);
    Map<String, MapReduceSpecification> mapReduces = deserializeMap(jsonObj.get("mapReduces"), context, MapReduceSpecification.class);
    Map<String, SparkSpecification> sparks = deserializeMap(jsonObj.get("sparks"), context, SparkSpecification.class);
    Map<String, WorkflowSpecification> workflows = deserializeMap(jsonObj.get("workflows"), context, WorkflowSpecification.class);
    Map<String, ServiceSpecification> services = deserializeMap(jsonObj.get("services"), context, ServiceSpecification.class);
    Map<String, ScheduleSpecification> schedules = deserializeMap(jsonObj.get("schedules"), context, ScheduleSpecification.class);
    Map<String, ScheduleCreationSpec> programSchedules = deserializeMap(jsonObj.get("programSchedules"), context, ScheduleCreationSpec.class);
    Map<String, WorkerSpecification> workers = deserializeMap(jsonObj.get("workers"), context, WorkerSpecification.class);
    Map<String, Plugin> plugins = deserializeMap(jsonObj.get("plugins"), context, Plugin.class);
    return new DefaultApplicationSpecification(name, appVersion, description, configuration, artifactId, streams, datasetModules, datasetInstances, flows, mapReduces, sparks, workflows, services, schedules, programSchedules, workers, plugins);
}

Example 10 with StreamSpecification

use of co.cask.cdap.api.data.stream.StreamSpecification in project cdap by caskdata.

the class ApplicationVerificationStage method verifyData.

protected void verifyData(ApplicationId appId, ApplicationSpecification specification, @Nullable KerberosPrincipalId specifiedOwnerPrincipal) throws DatasetManagementException, UnauthorizedException {
    // NOTE: no special restrictions on dataset module names, etc
    VerifyResult result;
    for (DatasetCreationSpec dataSetCreateSpec : specification.getDatasets().values()) {
        result = getVerifier(DatasetCreationSpec.class).verify(appId, dataSetCreateSpec);
        if (!result.isSuccess()) {
            throw new RuntimeException(result.getMessage());
        }
        String dsName = dataSetCreateSpec.getInstanceName();
        DatasetId datasetInstanceId = appId.getParent().dataset(dsName);
        DatasetSpecification existingSpec = dsFramework.getDatasetSpec(datasetInstanceId);
        if (existingSpec != null && !existingSpec.getType().equals(dataSetCreateSpec.getTypeName())) {
            // New app trying to deploy an dataset with same instanceName but different Type than that of existing.
            throw new DataSetException(String.format("Cannot Deploy Dataset : %s with Type : %s : Dataset with different Type Already Exists", dsName, dataSetCreateSpec.getTypeName()));
        }
        // if the dataset existed verify its owner is same.
        if (existingSpec != null) {
            verifyOwner(datasetInstanceId, specifiedOwnerPrincipal);
        }
    }
    for (StreamSpecification spec : specification.getStreams().values()) {
        result = getVerifier(StreamSpecification.class).verify(appId, spec);
        if (!result.isSuccess()) {
            throw new RuntimeException(result.getMessage());
        }
        // if the stream existed verify the owner to be the same
        if (store.getStream(appId.getNamespaceId(), spec.getName()) != null) {
            verifyOwner(appId.getParent().stream(spec.getName()), specifiedOwnerPrincipal);
        }
    }
}