Examples with Attribute com.datatorrent.api.Attribute used on opensource projects

Example 6 with Attribute

use of com.datatorrent.api.Attribute in project apex-core by apache.

the class StreamingContainerManager method saveMetaInfo.

/**
 * This method is for saving meta information about this application in HDFS -- the meta information that generally
 * does not change across multiple attempts
 */
private void saveMetaInfo() throws IOException {
    Path file = new Path(this.vars.appPath, APP_META_FILENAME + "." + System.nanoTime());
    try (FSDataOutputStream os = fileContext.create(file, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), Options.CreateOpts.CreateParent.createParent())) {
        JSONObject top = new JSONObject();
        JSONObject attributes = new JSONObject();
        for (Map.Entry<Attribute<?>, Object> entry : this.plan.getLogicalPlan().getAttributes().entrySet()) {
            attributes.put(entry.getKey().getSimpleName(), entry.getValue());
        }
        JSONObject autoMetrics = new JSONObject();
        for (Map.Entry<String, Map<String, Object>> entry : latestLogicalMetrics.entrySet()) {
            autoMetrics.put(entry.getKey(), new JSONArray(entry.getValue().keySet()));
        }
        top.put(APP_META_KEY_ATTRIBUTES, attributes);
        top.put(APP_META_KEY_METRICS, autoMetrics);
        os.write(top.toString().getBytes());
    } catch (JSONException ex) {
        throw new RuntimeException(ex);
    }
    Path origPath = new Path(this.vars.appPath, APP_META_FILENAME);
    fileContext.rename(file, origPath, Options.Rename.OVERWRITE);
}

Example 7 with Attribute

use of com.datatorrent.api.Attribute in project apex-core by apache.

the class LogicalPlanSerializer method convertToMap.

/**
 * @param dag
 * @return
 */
public static Map<String, Object> convertToMap(LogicalPlan dag, boolean includeModules) {
    HashMap<String, Object> result = new HashMap<>();
    ArrayList<Object> operatorArray = new ArrayList<>();
    ArrayList<Object> streamMap = new ArrayList<>();
    // result.put("applicationName", appConfig.getName());
    result.put("operators", operatorArray);
    result.put("streams", streamMap);
    // LogicalPlan dag = StramAppLauncher.prepareDAG(appConfig, StreamingApplication.LAUNCHMODE_YARN);
    // 
    // should we put the DAGContext info here?
    Map<String, Object> dagAttrs = new HashMap<>();
    for (Map.Entry<Attribute<Object>, Object> e : Attribute.AttributeMap.AttributeInitializer.getAllAttributes(dag, Context.DAGContext.class).entrySet()) {
        dagAttrs.put(e.getKey().getSimpleName(), e.getValue());
    }
    result.put("attributes", dagAttrs);
    Collection<OperatorMeta> allOperators = dag.getAllOperators();
    ObjectMapper propertyObjectMapper = new ObjectMapper();
    propertyObjectMapper.configure(JsonGenerator.Feature.WRITE_NUMBERS_AS_STRINGS, true);
    propertyObjectMapper.configure(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS, false);
    StdTypeResolverBuilder typer = new PropertyTypeResolverBuilder();
    typer.init(JsonTypeInfo.Id.CLASS, null);
    typer = typer.inclusion(JsonTypeInfo.As.PROPERTY);
    propertyObjectMapper.setDefaultTyping(typer);
    for (OperatorMeta operatorMeta : allOperators) {
        HashMap<String, Object> operatorDetailMap = new HashMap<>();
        ArrayList<Map<String, Object>> portList = new ArrayList<>();
        Map<String, Object> attributeMap = new HashMap<>();
        String operatorName = operatorMeta.getName();
        operatorArray.add(operatorDetailMap);
        operatorDetailMap.put("name", operatorName);
        operatorDetailMap.put("ports", portList);
        operatorDetailMap.put("class", operatorMeta.getOperator().getClass().getName());
        operatorDetailMap.put("attributes", attributeMap);
        Map<Attribute<Object>, Object> rawAttributes = Attribute.AttributeMap.AttributeInitializer.getAllAttributes(operatorMeta, Context.OperatorContext.class);
        for (Map.Entry<Attribute<Object>, Object> entry : rawAttributes.entrySet()) {
            attributeMap.put(entry.getKey().getSimpleName(), entry.getValue());
        }
        ObjectMapperString str;
        try {
            str = new ObjectMapperString(propertyObjectMapper.writeValueAsString(operatorMeta.getOperator()));
        } catch (Throwable ex) {
            LOG.error("Got exception when trying to get properties for operator {}", operatorMeta.getName(), ex);
            str = null;
        }
        operatorDetailMap.put("properties", str);
        Operators.PortMappingDescriptor pmd = new Operators.PortMappingDescriptor();
        Operators.describe(operatorMeta.getOperator(), pmd);
        for (Map.Entry<String, PortContextPair<InputPort<?>>> entry : pmd.inputPorts.entrySet()) {
            HashMap<String, Object> portDetailMap = new HashMap<>();
            HashMap<String, Object> portAttributeMap = new HashMap<>();
            InputPortMeta portMeta = operatorMeta.getMeta(entry.getValue().component);
            String portName = portMeta.getPortName();
            portDetailMap.put("name", portName);
            portDetailMap.put("type", "input");
            portDetailMap.put("attributes", portAttributeMap);
            rawAttributes = Attribute.AttributeMap.AttributeInitializer.getAllAttributes(portMeta, Context.PortContext.class);
            for (Map.Entry<Attribute<Object>, Object> attEntry : rawAttributes.entrySet()) {
                portAttributeMap.put(attEntry.getKey().getSimpleName(), attEntry.getValue());
            }
            portList.add(portDetailMap);
        }
        for (Map.Entry<String, PortContextPair<OutputPort<?>>> entry : pmd.outputPorts.entrySet()) {
            HashMap<String, Object> portDetailMap = new HashMap<>();
            HashMap<String, Object> portAttributeMap = new HashMap<>();
            OutputPortMeta portMeta = operatorMeta.getMeta(entry.getValue().component);
            String portName = portMeta.getPortName();
            portDetailMap.put("name", portName);
            portDetailMap.put("type", "output");
            portDetailMap.put("attributes", portAttributeMap);
            rawAttributes = Attribute.AttributeMap.AttributeInitializer.getAllAttributes(portMeta, Context.PortContext.class);
            for (Map.Entry<Attribute<Object>, Object> attEntry : rawAttributes.entrySet()) {
                portAttributeMap.put(attEntry.getKey().getSimpleName(), attEntry.getValue());
            }
            portList.add(portDetailMap);
        }
    }
    Collection<StreamMeta> allStreams = dag.getAllStreams();
    for (StreamMeta streamMeta : allStreams) {
        HashMap<String, Object> streamDetailMap = new HashMap<>();
        String streamName = streamMeta.getName();
        streamMap.add(streamDetailMap);
        String sourcePortName = streamMeta.getSource().getPortName();
        OperatorMeta operatorMeta = streamMeta.getSource().getOperatorMeta();
        HashMap<String, Object> sourcePortDetailMap = new HashMap<>();
        sourcePortDetailMap.put("operatorName", operatorMeta.getName());
        sourcePortDetailMap.put("portName", sourcePortName);
        streamDetailMap.put("name", streamName);
        streamDetailMap.put("source", sourcePortDetailMap);
        Collection<InputPortMeta> sinks = streamMeta.getSinks();
        ArrayList<HashMap<String, Object>> sinkPortList = new ArrayList<>();
        for (InputPortMeta sinkPort : sinks) {
            HashMap<String, Object> sinkPortDetailMap = new HashMap<>();
            sinkPortDetailMap.put("operatorName", sinkPort.getOperatorMeta().getName());
            sinkPortDetailMap.put("portName", sinkPort.getPortName());
            sinkPortList.add(sinkPortDetailMap);
        }
        streamDetailMap.put("sinks", sinkPortList);
        if (streamMeta.getLocality() != null) {
            streamDetailMap.put("locality", streamMeta.getLocality().name());
        }
    }
    if (includeModules) {
        ArrayList<Map<String, Object>> modulesArray = new ArrayList<>();
        result.put("modules", modulesArray);
        for (LogicalPlan.ModuleMeta meta : dag.getAllModules()) {
            modulesArray.add(getLogicalModuleDetails(dag, meta));
        }
    }
    return result;
}

Example 8 with Attribute

use of com.datatorrent.api.Attribute in project apex-core by apache.

the class StramWebServices method getOperatorAttributes.

@GET
@Path(PATH_LOGICAL_PLAN_OPERATORS + "/{operatorName}/attributes")
@Produces(MediaType.APPLICATION_JSON)
public JSONObject getOperatorAttributes(@PathParam("operatorName") String operatorName, @QueryParam("attributeName") String attributeName) {
    init();
    OperatorMeta logicalOperator = dagManager.getLogicalPlan().getOperatorMeta(operatorName);
    if (logicalOperator == null) {
        throw new NotFoundException();
    }
    HashMap<String, String> map = new HashMap<>();
    for (Map.Entry<Attribute<?>, Object> entry : dagManager.getOperatorAttributes(operatorName).entrySet()) {
        if (attributeName == null || entry.getKey().getSimpleName().equals(attributeName)) {
            Map.Entry<Attribute<Object>, Object> entry1 = (Map.Entry<Attribute<Object>, Object>) (Map.Entry) entry;
            map.put(entry1.getKey().getSimpleName(), entry1.getKey().codec.toString(entry1.getValue()));
        }
    }
    return new JSONObject(map);
}

Example 9 with Attribute

use of com.datatorrent.api.Attribute in project apex-core by apache.

the class LogicalPlanTest method testAttributeValuesSerializableCheck.

@Test
public void testAttributeValuesSerializableCheck() throws NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException {
    Attribute<Object> attr = new Attribute<>(new TestAttributeValue(), new Object2String());
    Field nameField = Attribute.class.getDeclaredField("name");
    nameField.setAccessible(true);
    nameField.set(attr, "Test_Attribute");
    nameField.setAccessible(false);
    assertNotNull(attr);
    // Dag attribute not serializable test
    dag.setAttribute(attr, new TestAttributeValue());
    try {
        dag.validate();
        Assert.fail("Setting not serializable attribute should throw exception");
    } catch (ValidationException e) {
        assertEquals("Validation Exception should match ", "Attribute value(s) for Test_Attribute in com.datatorrent.api.DAG are not serializable", e.getMessage());
    }
    // Operator attribute not serializable test
    dag = new LogicalPlan();
    TestGeneratorInputOperator operator = dag.addOperator("TestOperator", TestGeneratorInputOperator.class);
    dag.setOperatorAttribute(operator, attr, new TestAttributeValue());
    try {
        dag.validate();
        Assert.fail("Setting not serializable attribute should throw exception");
    } catch (ValidationException e) {
        assertEquals("Validation Exception should match ", "Attribute value(s) for Test_Attribute in TestOperator are not serializable", e.getMessage());
    }
    // Output Port attribute not serializable test
    dag = new LogicalPlan();
    operator = dag.addOperator("TestOperator", TestGeneratorInputOperator.class);
    dag.setOutputPortAttribute(operator.outport, attr, new TestAttributeValue());
    try {
        dag.validate();
        Assert.fail("Setting not serializable attribute should throw exception");
    } catch (ValidationException e) {
        assertEquals("Validation Exception should match ", "Attribute value(s) for Test_Attribute in TestOperator.outport are not serializable", e.getMessage());
    }
    // Input Port attribute not serializable test
    dag = new LogicalPlan();
    GenericTestOperator operator1 = dag.addOperator("TestOperator", GenericTestOperator.class);
    dag.setInputPortAttribute(operator1.inport1, attr, new TestAttributeValue());
    try {
        dag.validate();
        Assert.fail("Setting non serializable attribute should throw exception");
    } catch (ValidationException e) {
        assertEquals("Validation Exception should match ", "Attribute value(s) for Test_Attribute in TestOperator.inport1 are not serializable", e.getMessage());
    }
}

Example 10 with Attribute

use of com.datatorrent.api.Attribute in project apex-malhar by apache.

the class OperatorContextTestHelper method mockOperatorContext.

public static OperatorContext mockOperatorContext(int id, final AttributeMap map) {
    OperatorContext context = Mockito.mock(OperatorContext.class);
    Mockito.when(context.getId()).thenReturn(id);
    Mockito.when(context.getAttributes()).thenReturn(map);
    Mockito.doThrow(new UnsupportedOperationException("not supported")).when(context).sendMetrics(Mockito.<Collection<String>>any());
    Mockito.doThrow(new UnsupportedOperationException("not supported")).when(context).setCounters(Mockito.any());
    Mockito.when(context.getValue(Mockito.<Attribute>any())).thenAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            final Attribute key = (Attribute) invocation.getArguments()[0];
            Object value = map.get(key);
            if (value != null) {
                return value;
            }
            return key.defaultValue;
        }
    });
    Mockito.doNothing().when(context).setCounters(Mockito.any());
    Mockito.when(context.getWindowsFromCheckpoint()).thenReturn(0);
    return context;
}