Examples with StreamProperties co.cask.cdap.proto.StreamProperties used on opensource projects

Example 26 with StreamProperties

use of co.cask.cdap.proto.StreamProperties in project cdap by caskdata.

the class StreamHandlerTest method createStreamWithOwner.

private void createStreamWithOwner(String streamUrl, HttpResponseStatus responseStatus, @Nullable String owner) throws IOException, URISyntaxException {
    HttpURLConnection urlConn = openURL(createURL(streamUrl), HttpMethod.PUT);
    urlConn.setDoOutput(true);
    StreamProperties properties = new StreamProperties(1L, null, 128, null, owner);
    urlConn.getOutputStream().write(GSON.toJson(properties).getBytes(Charsets.UTF_8));
    Assert.assertEquals(responseStatus.getCode(), urlConn.getResponseCode());
    urlConn.disconnect();
}

Example 27 with StreamProperties

use of co.cask.cdap.proto.StreamProperties in project cdap by caskdata.

the class SetStreamFormatCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
    StreamId streamId = cliConfig.getCurrentNamespace().stream(arguments.get(ArgumentName.STREAM.toString()));
    StreamProperties currentProperties = streamClient.getConfig(streamId);
    String formatName = arguments.get(ArgumentName.FORMAT.toString());
    Schema schema = getSchema(arguments);
    Map<String, String> settings = Collections.emptyMap();
    if (arguments.hasArgument(ArgumentName.SETTINGS.toString())) {
        settings = ArgumentParser.parseMap(arguments.get(ArgumentName.SETTINGS.toString()), ArgumentName.SETTINGS.toString());
    }
    FormatSpecification formatSpecification = new FormatSpecification(formatName, schema, settings);
    StreamProperties streamProperties = new StreamProperties(currentProperties.getTTL(), formatSpecification, currentProperties.getNotificationThresholdMB(), currentProperties.getDescription());
    streamClient.setStreamProperties(streamId, streamProperties);
    output.printf("Successfully set format of stream '%s'\n", streamId.getEntityName());
}

Example 28 with StreamProperties

use of co.cask.cdap.proto.StreamProperties in project cdap by caskdata.

the class StreamHandler method getAndValidateConfig.

/**
   * Gets stream properties from the request. If there is request is invalid, a BadRequestException will be thrown.
   */
private StreamProperties getAndValidateConfig(HttpRequest request) throws BadRequestException {
    Reader reader = new InputStreamReader(new ChannelBufferInputStream(request.getContent()));
    StreamProperties properties;
    try {
        properties = GSON.fromJson(reader, StreamProperties.class);
    } catch (Exception e) {
        throw new BadRequestException("Invalid stream configuration. Please check that the " + "configuration is a valid JSON Object with a valid schema. Cause: " + e.getMessage());
    }
    // Validate ttl
    Long ttl = properties.getTTL();
    if (ttl != null && ttl < 0) {
        throw new BadRequestException("Invalid TTL " + ttl + ". TTL value should be positive.");
    }
    // Validate format
    FormatSpecification formatSpec = properties.getFormat();
    if (formatSpec != null) {
        String formatName = formatSpec.getName();
        if (formatName == null) {
            throw new BadRequestException("A format name must be specified.");
        }
        try {
            // if a format is given, make sure it is a valid format,
            // check that we can instantiate the format class
            RecordFormat<?, ?> format = RecordFormats.createInitializedFormat(formatSpec);
            // the request may contain a null schema, in which case the default schema of the format should be used.
            // create a new specification object that is guaranteed to have a non-null schema.
            formatSpec = new FormatSpecification(formatSpec.getName(), format.getSchema(), formatSpec.getSettings());
        } catch (UnsupportedTypeException e) {
            throw new BadRequestException("Format " + formatName + " does not support the requested schema.");
        } catch (Exception e) {
            throw new BadRequestException("Invalid format, unable to instantiate format " + formatName);
        }
    }
    // Validate notification threshold
    Integer threshold = properties.getNotificationThresholdMB();
    if (threshold != null && threshold <= 0) {
        throw new BadRequestException("Invalid threshold " + threshold + ". Threshold value should be greater than zero.");
    }
    // validate owner principal if one is provided
    if (properties.getOwnerPrincipal() != null) {
        SecurityUtil.validateKerberosPrincipal(properties.getOwnerPrincipal());
    }
    return new StreamProperties(ttl, formatSpec, threshold, properties.getDescription(), properties.getOwnerPrincipal());
}

Example 29 with StreamProperties

use of co.cask.cdap.proto.StreamProperties in project cdap by caskdata.

the class StreamHandler method create.

@PUT
@Path("/{stream}")
@AuditPolicy(AuditDetail.REQUEST_BODY)
public void create(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("stream") String stream) throws Exception {
    // Check for namespace existence. Throws NotFoundException if namespace doesn't exist
    namespaceQueryAdmin.get(new NamespaceId(namespaceId));
    StreamId streamId = validateAndGetStreamId(namespaceId, stream);
    Properties props = new Properties();
    StreamProperties streamProperties;
    // If the request to create a stream contains a non-empty body, then construct and set StreamProperties
    if (request.getContent().readable()) {
        streamProperties = getAndValidateConfig(request);
        if (streamProperties.getTTL() != null) {
            props.put(Constants.Stream.TTL, Long.toString(streamProperties.getTTL()));
        }
        if (streamProperties.getNotificationThresholdMB() != null) {
            props.put(Constants.Stream.NOTIFICATION_THRESHOLD, Integer.toString(streamProperties.getNotificationThresholdMB()));
        }
        if (streamProperties.getDescription() != null) {
            props.put(Constants.Stream.DESCRIPTION, streamProperties.getDescription());
        }
        if (streamProperties.getFormat() != null) {
            props.put(Constants.Stream.FORMAT_SPECIFICATION, GSON.toJson(streamProperties.getFormat()));
        }
        if (streamProperties.getOwnerPrincipal() != null) {
            props.put(Constants.Security.PRINCIPAL, streamProperties.getOwnerPrincipal());
        }
    }
    streamAdmin.create(streamId, props);
    responder.sendStatus(HttpResponseStatus.OK);
}

Example 30 with StreamProperties

use of co.cask.cdap.proto.StreamProperties in project cdap by caskdata.

the class CLIMainTest method testStream.

@Test
public void testStream() throws Exception {
    String streamId = PREFIX + "sdf123";
    File file = new File(TMP_FOLDER.newFolder(), "test1.txt");
    StreamProperties streamProperties = new StreamProperties(2L, null, 10, "Golden Stream");
    try (BufferedWriter writer = Files.newWriter(file, Charsets.UTF_8)) {
        writer.write(GSON.toJson(streamProperties));
    }
    testCommandOutputContains(cli, "create stream " + streamId + " " + file.getAbsolutePath(), "Successfully created stream");
    testCommandOutputContains(cli, "describe stream " + streamId, "Golden Stream");
    testCommandOutputContains(cli, "set stream description " + streamId + " 'Silver Stream'", "Successfully set stream description");
    testCommandOutputContains(cli, "describe stream " + streamId, "Silver Stream");
    testCommandOutputContains(cli, "delete stream " + streamId, "Successfully deleted stream");
    testCommandOutputContains(cli, "create stream " + streamId, "Successfully created stream");
    testCommandOutputContains(cli, "list streams", streamId);
    testCommandOutputNotContains(cli, "get stream " + streamId, "helloworld");
    testCommandOutputContains(cli, "send stream " + streamId + " helloworld", "Successfully sent stream event");
    testCommandOutputContains(cli, "get stream " + streamId, "helloworld");
    testCommandOutputContains(cli, "get stream " + streamId + " -10m -0s 1", "helloworld");
    testCommandOutputContains(cli, "get stream " + streamId + " -10m -0s", "helloworld");
    testCommandOutputContains(cli, "get stream " + streamId + " -10m", "helloworld");
    testCommandOutputContains(cli, "truncate stream " + streamId, "Successfully truncated stream");
    testCommandOutputNotContains(cli, "get stream " + streamId, "helloworld");
    testCommandOutputContains(cli, "set stream ttl " + streamId + " 100000", "Successfully set TTL of stream");
    testCommandOutputContains(cli, "set stream notification-threshold " + streamId + " 1", "Successfully set notification threshold of stream");
    testCommandOutputContains(cli, "describe stream " + streamId, "100000");
    file = new File(TMP_FOLDER.newFolder(), "test2.txt");
    // If the file not exist or not a file, upload should fails with an error.
    testCommandOutputContains(cli, "load stream " + streamId + " " + file.getAbsolutePath(), "Not a file");
    testCommandOutputContains(cli, "load stream " + streamId + " " + file.getParentFile().getAbsolutePath(), "Not a file");
    // Generate a file to send
    try (BufferedWriter writer = Files.newWriter(file, Charsets.UTF_8)) {
        for (int i = 0; i < 10; i++) {
            writer.write(String.format("%s, Event %s", i, i));
            writer.newLine();
        }
    }
    testCommandOutputContains(cli, "load stream " + streamId + " " + file.getAbsolutePath(), "Successfully loaded file to stream");
    testCommandOutputContains(cli, "get stream " + streamId, "9, Event 9");
    testCommandOutputContains(cli, "get stream-stats " + streamId, String.format("No schema found for stream '%s'", streamId));
    testCommandOutputContains(cli, "set stream format " + streamId + " csv 'body string'", String.format("Successfully set format of stream '%s'", streamId));
    testCommandOutputContains(cli, "execute 'show tables'", String.format("stream_%s", streamId));
    testCommandOutputContains(cli, "get stream-stats " + streamId, "Analyzed 10 Stream events in the time range [0, 9223372036854775807]");
    testCommandOutputContains(cli, "get stream-stats " + streamId + " limit 5 start 5 end 10", "Analyzed 0 Stream events in the time range [5, 10]");
}