Examples with StringCodec com.datatorrent.api.StringCodec used on opensource projects

Example 1 with StringCodec

use of com.datatorrent.api.StringCodec in project apex-core by apache.

the class LogicalPlanConfigurationTest method testLoadFromJson.

@Test
public void testLoadFromJson() throws Exception {
    String resourcePath = "/testTopology.json";
    InputStream is = this.getClass().getResourceAsStream(resourcePath);
    if (is == null) {
        fail("Could not load " + resourcePath);
    }
    StringWriter writer = new StringWriter();
    IOUtils.copy(is, writer);
    JSONObject json = new JSONObject(writer.toString());
    Configuration conf = new Configuration(false);
    conf.set(StreamingApplication.APEX_PREFIX + "operator.operator3.prop.myStringProperty", "o3StringFromConf");
    LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
    LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
    dag.validate();
    assertEquals("DAG attribute CONTAINER_JVM_OPTIONS ", dag.getAttributes().get(DAGContext.CONTAINER_JVM_OPTIONS), "-Xmx16m");
    Map<Class<?>, Class<? extends StringCodec<?>>> stringCodecsMap = Maps.newHashMap();
    stringCodecsMap.put(Integer.class, Integer2String.class);
    assertEquals("DAG attribute STRING_CODECS ", stringCodecsMap, dag.getAttributes().get(DAGContext.STRING_CODECS));
    assertEquals("DAG attribute CONTAINER_OPTS_CONFIGURATOR ", BasicContainerOptConfigurator.class, dag.getAttributes().get(DAGContext.CONTAINER_OPTS_CONFIGURATOR).getClass());
    assertEquals("number of operator confs", 5, dag.getAllOperators().size());
    assertEquals("number of root operators", 1, dag.getRootOperators().size());
    StreamMeta s1 = dag.getStream("n1n2");
    assertNotNull(s1);
    assertTrue("n1n2 inline", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());
    OperatorMeta input = dag.getOperatorMeta("inputOperator");
    TestStatsListener tsl = new TestStatsListener();
    tsl.setIntProp(222);
    List<StatsListener> sll = Lists.<StatsListener>newArrayList(tsl);
    assertEquals("inputOperator STATS_LISTENERS attribute ", sll, input.getAttributes().get(OperatorContext.STATS_LISTENERS));
    for (OutputPortMeta opm : input.getOutputStreams().keySet()) {
        assertTrue("output port of input Operator attribute is JsonStreamCodec ", opm.getAttributes().get(PortContext.STREAM_CODEC) instanceof JsonStreamCodec<?>);
    }
    OperatorMeta operator3 = dag.getOperatorMeta("operator3");
    assertEquals("operator3.classname", GenericTestOperator.class, operator3.getOperator().getClass());
    GenericTestOperator doperator3 = (GenericTestOperator) operator3.getOperator();
    assertEquals("myStringProperty " + doperator3, "o3StringFromConf", doperator3.getMyStringProperty());
    assertFalse("booleanProperty " + doperator3, doperator3.booleanProperty);
    OperatorMeta operator4 = dag.getOperatorMeta("operator4");
    GenericTestOperator doperator4 = (GenericTestOperator) operator4.getOperator();
    assertEquals("myStringProperty " + doperator4, "overrideOperator4", doperator4.getMyStringProperty());
    assertEquals("setterOnlyOperator4 " + doperator4, "setterOnlyOperator4", doperator4.propertySetterOnly);
    assertTrue("booleanProperty " + doperator4, doperator4.booleanProperty);
    StreamMeta input1 = dag.getStream("inputStream");
    assertNotNull(input1);
    OperatorMeta inputOperator = dag.getOperatorMeta("inputOperator");
    Assert.assertEquals("input1 source", inputOperator, input1.getSource().getOperatorMeta());
    Set<OperatorMeta> targetNodes = Sets.newHashSet();
    for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
        targetNodes.add(targetPort.getOperatorMeta());
    }
    Assert.assertEquals("operator attribute " + inputOperator, 64, (int) inputOperator.getValue(OperatorContext.MEMORY_MB));
    Assert.assertEquals("port attribute " + inputOperator, 8, (int) input1.getSource().getValue(PortContext.UNIFIER_LIMIT));
    Assert.assertEquals("input1 target ", Sets.newHashSet(dag.getOperatorMeta("operator1"), operator3, operator4), targetNodes);
}

Example 2 with StringCodec

use of com.datatorrent.api.StringCodec in project apex-core by apache.

the class StramWebServices method init.

@SuppressWarnings({ "rawtypes", "unchecked" })
private void init() {
    // clear content type
    httpResponse.setContentType(null);
    if (!initialized) {
        Map<Class<?>, Class<? extends StringCodec<?>>> codecs = dagManager.getApplicationAttributes().get(DAGContext.STRING_CODECS);
        StringCodecs.loadConverters(codecs);
        if (codecs != null) {
            SimpleModule sm = new SimpleModule("DTSerializationModule", new Version(1, 0, 0, null));
            for (Map.Entry<Class<?>, Class<? extends StringCodec<?>>> entry : codecs.entrySet()) {
                try {
                    final StringCodec<Object> codec = (StringCodec<Object>) entry.getValue().newInstance();
                    sm.addSerializer(new SerializerBase(entry.getKey()) {

                        @Override
                        public void serialize(Object value, JsonGenerator jgen, SerializerProvider provider) throws IOException, JsonProcessingException {
                            jgen.writeString(codec.toString(value));
                        }
                    });
                } catch (Exception ex) {
                    LOG.error("Caught exception when instantiating codec for class {}", entry.getKey().getName(), ex);
                }
            }
            objectMapper.registerModule(sm);
        }
        initialized = true;
    }
}

Example 3 with StringCodec

use of com.datatorrent.api.StringCodec in project apex-core by apache.

the class GenericOperatorPropertyCodecTest method testGenericOperatorPropertyCodec.

@Test
public void testGenericOperatorPropertyCodec() {
    LogicalPlan dag = new LogicalPlan();
    Map<Class<?>, Class<? extends StringCodec<?>>> codecs = new HashMap<>();
    codecs.put(GenericOperatorProperty.class, GenericOperatorProperty.GenericOperatorPropertyStringCodec.class);
    dag.setAttribute(com.datatorrent.api.Context.DAGContext.STRING_CODECS, codecs);
    dag.setAttribute(com.datatorrent.api.Context.OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    OperatorMeta o1Meta = dag.getMeta(o1);
    TestPlanContext ctx = new TestPlanContext();
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    ctx.deploy.clear();
    ctx.undeploy.clear();
    PlanModifier pm = new PlanModifier(plan);
    try {
        pm.setOperatorProperty(o1Meta.getName(), "genericOperatorProperty", "xyz");
        // cannot set properties on an operator that is already deployed.
        Assert.fail("validation error expected");
    } catch (javax.validation.ValidationException e) {
        Assert.assertTrue(e.getMessage().contains(o1Meta.toString()));
    }
    GenericTestOperator newOperator = new GenericTestOperator();
    pm.addOperator("newOperator", newOperator);
    pm.setOperatorProperty("newOperator", "genericOperatorProperty", "xyz");
    Assert.assertEquals("", "xyz", newOperator.getGenericOperatorProperty().obtainString());
}

Example 4 with StringCodec

use of com.datatorrent.api.StringCodec in project apex-core by apache.

the class StringCodecs method loadDefaultConverters.

public static void loadDefaultConverters() {
    LOG.debug("Loading default converters for BeanUtils");
    ConvertUtils.register(new Converter() {

        @Override
        @SuppressWarnings("unchecked")
        public Object convert(Class type, Object value) {
            if (value == null) {
                return null;
            }
            for (Class<?> clazz = value.getClass(); clazz != null; clazz = clazz.getSuperclass()) {
                Class<? extends StringCodec> codec = codecs.get(clazz);
                if (codec == null) {
                    continue;
                }
                StringCodec instance;
                try {
                    instance = codec.newInstance();
                } catch (IllegalAccessException ex) {
                    throw new RuntimeException("Internal Error - it's impossible for this exception to be thrown!", ex);
                } catch (InstantiationException ex) {
                    throw new RuntimeException("Internal Error - it's impossible for this exception to be thrown!", ex);
                }
                return instance.toString(value);
            }
            return value.toString();
        }
    }, String.class);
    ConvertUtils.register(new Converter() {

        @Override
        public Object convert(Class type, Object value) {
            return value == null ? null : URI.create(value.toString());
        }
    }, URI.class);
}

Example 5 with StringCodec

use of com.datatorrent.api.StringCodec in project apex-core by apache.

the class StringCodecs method check.

public static void check() {
    if (classLoaders.putIfAbsent(Thread.currentThread().getContextClassLoader(), Boolean.TRUE) == null) {
        loadDefaultConverters();
        for (Map.Entry<Class<?>, Class<? extends StringCodec<?>>> entry : codecs.entrySet()) {
            try {
                final StringCodec<?> codecInstance = entry.getValue().newInstance();
                ConvertUtils.register(new Converter() {

                    @Override
                    public Object convert(Class type, Object value) {
                        return value == null ? null : codecInstance.fromString(value.toString());
                    }
                }, entry.getKey());
            } catch (Exception ex) {
                throw new RuntimeException(ex);
            }
        }
    }
}