Examples with FormatSpecification co.cask.cdap.api.data.format.FormatSpecification used on opensource projects

Example 56 with FormatSpecification

use of co.cask.cdap.api.data.format.FormatSpecification in project cdap by caskdata.

the class StreamBatchSource method prepareRun.

@Override
public void prepareRun(BatchSourceContext context) {
    long duration = ETLUtils.parseDuration(streamBatchConfig.duration);
    long delay = Strings.isNullOrEmpty(streamBatchConfig.delay) ? 0 : ETLUtils.parseDuration(streamBatchConfig.delay);
    long endTime = context.getLogicalStartTime() - delay;
    long startTime = endTime - duration;
    LOG.info("Setting input to Stream : {}", streamBatchConfig.name);
    FormatSpecification formatSpec = streamBatchConfig.getFormatSpec();
    Input stream;
    if (formatSpec == null) {
        stream = Input.ofStream(streamBatchConfig.name, startTime, endTime);
    } else {
        stream = Input.ofStream(streamBatchConfig.name, startTime, endTime, formatSpec);
    }
    context.setInput(stream);
}

Example 57 with FormatSpecification

use of co.cask.cdap.api.data.format.FormatSpecification in project cdap by caskdata.

the class CreateOrUpdateStreamViewCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
    StreamId streamId = cliConfig.getCurrentNamespace().stream(arguments.get(ArgumentName.STREAM.toString()));
    StreamViewId viewId = streamId.view(arguments.get(ArgumentName.VIEW.toString()));
    String formatName = arguments.get(ArgumentName.FORMAT.toString());
    Schema schema = getSchema(arguments);
    Map<String, String> settings = Collections.emptyMap();
    if (arguments.hasArgument(ArgumentName.SETTINGS.toString())) {
        settings = ArgumentParser.parseMap(arguments.get(ArgumentName.SETTINGS.toString()), ArgumentName.SETTINGS.toString());
    }
    FormatSpecification formatSpecification = new FormatSpecification(formatName, schema, settings);
    ViewSpecification viewSpecification = new ViewSpecification(formatSpecification);
    boolean created = client.createOrUpdate(viewId, viewSpecification);
    if (created) {
        output.printf("Successfully created stream-view '%s'\n", viewId.getEntityName());
    } else {
        output.printf("Successfully updated stream-view '%s'\n", viewId.getEntityName());
    }
}

Example 58 with FormatSpecification

use of co.cask.cdap.api.data.format.FormatSpecification in project cdap by caskdata.

the class StreamViewHttpHandlerTest method testAll.

@Test
public void testAll() throws Exception {
    execute(404, HttpRequest.get(resolve("/v3/namespaces/default/streams/foo/views")).build());
    execute(404, HttpRequest.get(resolve("/v3/namespaces/default/streams/foo/views/view1")).build());
    execute(404, HttpRequest.delete(resolve("/v3/namespaces/default/streams/foo/views/view1")).build());
    execute(200, HttpRequest.put(resolve("/v3/namespaces/default/streams/foo")).build());
    execute(404, HttpRequest.delete(resolve("/v3/namespaces/default/streams/foo/views/nonexistent")).build());
    List<String> views = execute(200, HttpRequest.get(resolve("/v3/namespaces/default/streams/foo/views")).build(), new TypeToken<List<String>>() {
    }.getType());
    Assert.assertEquals(ImmutableList.of(), views);
    Schema schema = Schema.recordOf("foo", Schema.Field.of("name", Schema.of(Schema.Type.STRING)));
    FormatSpecification formatSpec = new FormatSpecification(Formats.AVRO, schema, Collections.<String, String>emptyMap());
    ViewSpecification config = new ViewSpecification(formatSpec);
    // trying to create without request body should give 400
    execute(400, HttpRequest.put(resolve("/v3/namespaces/default/streams/foo/views/view1")).build());
    execute(201, HttpRequest.put(resolve("/v3/namespaces/default/streams/foo/views/view1")).withBody(GSON.toJson(config)).build());
    ViewDetail actualDetail = execute(200, HttpRequest.get(resolve("/v3/namespaces/default/streams/foo/views/view1")).build(), ViewDetail.class);
    Assert.assertEquals(new ViewDetail("view1", new ViewSpecification(config.getFormat(), "stream_foo_view1")), actualDetail);
    views = execute(200, HttpRequest.get(resolve("/v3/namespaces/default/streams/foo/views")).build(), new TypeToken<List<String>>() {
    }.getType());
    Assert.assertEquals(ImmutableList.of("view1"), views);
    execute(201, HttpRequest.put(resolve("/v3/namespaces/default/streams/foo/views/view2")).withBody(GSON.toJson(config)).build());
    actualDetail = execute(200, HttpRequest.get(resolve("/v3/namespaces/default/streams/foo/views/view2")).build(), ViewDetail.class);
    Assert.assertEquals(new ViewDetail("view2", new ViewSpecification(config.getFormat(), "stream_foo_view2")), actualDetail);
    views = execute(200, HttpRequest.get(resolve("/v3/namespaces/default/streams/foo/views")).build(), new TypeToken<List<String>>() {
    }.getType());
    Assert.assertEquals(ImmutableList.of("view1", "view2"), views == null ? null : Ordering.natural().sortedCopy(views));
    execute(200, HttpRequest.delete(resolve("/v3/namespaces/default/streams/foo/views/view1")).build());
    views = execute(200, HttpRequest.get(resolve("/v3/namespaces/default/streams/foo/views")).build(), new TypeToken<List<String>>() {
    }.getType());
    Assert.assertEquals(ImmutableList.of("view2"), views);
    execute(200, HttpRequest.delete(resolve("/v3/namespaces/default/streams/foo/views/view2")).build());
    views = execute(200, HttpRequest.get(resolve("/v3/namespaces/default/streams/foo/views")).build(), new TypeToken<List<String>>() {
    }.getType());
    Assert.assertEquals(ImmutableList.<String>of(), views);
    // Deleting a stream should also delete the associated views
    execute(200, HttpRequest.delete(resolve("/v3/namespaces/default/streams/foo")).build());
    execute(200, HttpRequest.put(resolve("/v3/namespaces/default/streams/foo")).build());
    execute(404, HttpRequest.get(resolve("/v3/namespaces/default/streams/foo/views/view1")).build());
}

Example 59 with FormatSpecification

use of co.cask.cdap.api.data.format.FormatSpecification in project cdap by caskdata.

the class SetStreamFormatCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
    StreamId streamId = cliConfig.getCurrentNamespace().stream(arguments.get(ArgumentName.STREAM.toString()));
    StreamProperties currentProperties = streamClient.getConfig(streamId);
    String formatName = arguments.get(ArgumentName.FORMAT.toString());
    Schema schema = getSchema(arguments);
    Map<String, String> settings = Collections.emptyMap();
    if (arguments.hasArgument(ArgumentName.SETTINGS.toString())) {
        settings = ArgumentParser.parseMap(arguments.get(ArgumentName.SETTINGS.toString()), ArgumentName.SETTINGS.toString());
    }
    FormatSpecification formatSpecification = new FormatSpecification(formatName, schema, settings);
    StreamProperties streamProperties = new StreamProperties(currentProperties.getTTL(), formatSpecification, currentProperties.getNotificationThresholdMB(), currentProperties.getDescription());
    streamClient.setStreamProperties(streamId, streamProperties);
    output.printf("Successfully set format of stream '%s'\n", streamId.getEntityName());
}

Example 60 with FormatSpecification

use of co.cask.cdap.api.data.format.FormatSpecification in project cdap by caskdata.

the class StreamHandler method getAndValidateConfig.

/**
   * Gets stream properties from the request. If there is request is invalid, a BadRequestException will be thrown.
   */
private StreamProperties getAndValidateConfig(HttpRequest request) throws BadRequestException {
    Reader reader = new InputStreamReader(new ChannelBufferInputStream(request.getContent()));
    StreamProperties properties;
    try {
        properties = GSON.fromJson(reader, StreamProperties.class);
    } catch (Exception e) {
        throw new BadRequestException("Invalid stream configuration. Please check that the " + "configuration is a valid JSON Object with a valid schema. Cause: " + e.getMessage());
    }
    // Validate ttl
    Long ttl = properties.getTTL();
    if (ttl != null && ttl < 0) {
        throw new BadRequestException("Invalid TTL " + ttl + ". TTL value should be positive.");
    }
    // Validate format
    FormatSpecification formatSpec = properties.getFormat();
    if (formatSpec != null) {
        String formatName = formatSpec.getName();
        if (formatName == null) {
            throw new BadRequestException("A format name must be specified.");
        }
        try {
            // if a format is given, make sure it is a valid format,
            // check that we can instantiate the format class
            RecordFormat<?, ?> format = RecordFormats.createInitializedFormat(formatSpec);
            // the request may contain a null schema, in which case the default schema of the format should be used.
            // create a new specification object that is guaranteed to have a non-null schema.
            formatSpec = new FormatSpecification(formatSpec.getName(), format.getSchema(), formatSpec.getSettings());
        } catch (UnsupportedTypeException e) {
            throw new BadRequestException("Format " + formatName + " does not support the requested schema.");
        } catch (Exception e) {
            throw new BadRequestException("Invalid format, unable to instantiate format " + formatName);
        }
    }
    // Validate notification threshold
    Integer threshold = properties.getNotificationThresholdMB();
    if (threshold != null && threshold <= 0) {
        throw new BadRequestException("Invalid threshold " + threshold + ". Threshold value should be greater than zero.");
    }
    // validate owner principal if one is provided
    if (properties.getOwnerPrincipal() != null) {
        SecurityUtil.validateKerberosPrincipal(properties.getOwnerPrincipal());
    }
    return new StreamProperties(ttl, formatSpec, threshold, properties.getDescription(), properties.getOwnerPrincipal());
}