Examples with OperatorMeta com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta used on opensource projects

Example 71 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class StramWebServices method getPorts.

@GET
@Path(PATH_LOGICAL_PLAN_OPERATORS + "/{operatorName}/ports")
@Produces(MediaType.APPLICATION_JSON)
public JSONObject getPorts(@PathParam("operatorName") String operatorName) {
    init();
    OperatorMeta logicalOperator = dagManager.getLogicalPlan().getOperatorMeta(operatorName);
    Set<LogicalPlan.InputPortMeta> inputPorts;
    Set<LogicalPlan.OutputPortMeta> outputPorts;
    if (logicalOperator == null) {
        ModuleMeta logicalModule = dagManager.getModuleMeta(operatorName);
        if (logicalModule == null) {
            throw new NotFoundException();
        }
        inputPorts = logicalModule.getInputStreams().keySet();
        outputPorts = logicalModule.getOutputStreams().keySet();
    } else {
        inputPorts = logicalOperator.getInputStreams().keySet();
        outputPorts = logicalOperator.getOutputStreams().keySet();
    }
    JSONObject result = getPortsObjects(inputPorts, outputPorts);
    return result;
}

Example 72 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class StramWebServices method getOperatorAttributes.

@GET
@Path(PATH_LOGICAL_PLAN_OPERATORS + "/{operatorName}/attributes")
@Produces(MediaType.APPLICATION_JSON)
public JSONObject getOperatorAttributes(@PathParam("operatorName") String operatorName, @QueryParam("attributeName") String attributeName) {
    init();
    OperatorMeta logicalOperator = dagManager.getLogicalPlan().getOperatorMeta(operatorName);
    if (logicalOperator == null) {
        throw new NotFoundException();
    }
    HashMap<String, String> map = new HashMap<>();
    for (Map.Entry<Attribute<?>, Object> entry : dagManager.getOperatorAttributes(operatorName).entrySet()) {
        if (attributeName == null || entry.getKey().getSimpleName().equals(attributeName)) {
            Map.Entry<Attribute<Object>, Object> entry1 = (Map.Entry<Attribute<Object>, Object>) (Map.Entry) entry;
            map.put(entry1.getKey().getSimpleName(), entry1.getKey().codec.toString(entry1.getValue()));
        }
    }
    return new JSONObject(map);
}

Example 73 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class StreamingContainerManagerTest method testValidInputOperatorDeployInfoType.

@Test
public void testValidInputOperatorDeployInfoType() {
    TestGeneratorInputOperator.ValidInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.ValidInputOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.addStream("stream1", o1.outport, o2.inport1);
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    PhysicalPlan physicalPlan = scm.getPhysicalPlan();
    List<PTContainer> containers = physicalPlan.getContainers();
    for (int i = 0; i < containers.size(); ++i) {
        assignContainer(scm, "container" + (i + 1));
    }
    OperatorMeta o1Meta = dag.getMeta(o1);
    PTOperator o1Physical = physicalPlan.getOperators(o1Meta).get(0);
    String containerId = o1Physical.getContainer().getExternalId();
    OperatorDeployInfo o1DeployInfo = getDeployInfo(scm.getContainerAgent(containerId)).get(0);
    Assert.assertEquals("type " + o1DeployInfo, OperatorDeployInfo.OperatorType.INPUT, o1DeployInfo.type);
}

Example 74 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class LogicalPlanConfigurationTest method testLoadFromPropertiesFileWithLegacyPrefix.

@Test
public void testLoadFromPropertiesFileWithLegacyPrefix() throws IOException {
    Properties props = new Properties();
    String resourcePath = "/testTopologyLegacyPrefix.properties";
    InputStream is = this.getClass().getResourceAsStream(resourcePath);
    if (is == null) {
        fail("Could not load " + resourcePath);
    }
    props.load(is);
    LogicalPlanConfiguration pb = new LogicalPlanConfiguration(new Configuration(false)).addFromProperties(props, null);
    LogicalPlan dag = new LogicalPlan();
    pb.populateDAG(dag);
    dag.validate();
    assertEquals("number of operators", 2, dag.getAllOperators().size());
    assertEquals("number of root operators", 1, dag.getRootOperators().size());
    StreamMeta s1 = dag.getStream("s1");
    assertNotNull(s1);
    assertTrue("s1 locality", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());
    OperatorMeta o2m = dag.getOperatorMeta("o2");
    assertEquals(GenericTestOperator.class, o2m.getOperator().getClass());
    GenericTestOperator o2 = (GenericTestOperator) o2m.getOperator();
    assertEquals("myStringProperty " + o2, "myStringPropertyValue", o2.getMyStringProperty());
}

Example 75 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class DelayOperatorTest method testCheckpointUpdate.

@Test
public void testCheckpointUpdate() {
    LogicalPlan dag = StramTestSupport.createDAG(testMeta);
    TestGeneratorInputOperator opA = dag.addOperator("A", TestGeneratorInputOperator.class);
    GenericTestOperator opB = dag.addOperator("B", GenericTestOperator.class);
    GenericTestOperator opC = dag.addOperator("C", GenericTestOperator.class);
    GenericTestOperator opD = dag.addOperator("D", GenericTestOperator.class);
    DefaultDelayOperator<Object> opDelay = dag.addOperator("opDelay", new DefaultDelayOperator<>());
    dag.addStream("AtoB", opA.outport, opB.inport1);
    dag.addStream("BtoC", opB.outport1, opC.inport1);
    dag.addStream("CtoD", opC.outport1, opD.inport1);
    dag.addStream("CtoDelay", opC.outport2, opDelay.input);
    dag.addStream("DelayToB", opDelay.output, opB.inport2);
    dag.validate();
    dag.setAttribute(com.datatorrent.api.Context.OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    PhysicalPlan plan = scm.getPhysicalPlan();
    // set all operators as active to enable recovery window id update
    for (PTOperator oper : plan.getAllOperators().values()) {
        oper.setState(PTOperator.State.ACTIVE);
    }
    Clock clock = new SystemClock();
    PTOperator opA1 = plan.getOperators(dag.getMeta(opA)).get(0);
    PTOperator opB1 = plan.getOperators(dag.getMeta(opB)).get(0);
    PTOperator opC1 = plan.getOperators(dag.getMeta(opC)).get(0);
    PTOperator opDelay1 = plan.getOperators(dag.getMeta(opDelay)).get(0);
    PTOperator opD1 = plan.getOperators(dag.getMeta(opD)).get(0);
    Checkpoint cp3 = new Checkpoint(3L, 0, 0);
    Checkpoint cp5 = new Checkpoint(5L, 0, 0);
    Checkpoint cp4 = new Checkpoint(4L, 0, 0);
    opB1.checkpoints.add(cp3);
    opC1.checkpoints.add(cp3);
    opC1.checkpoints.add(cp4);
    opDelay1.checkpoints.add(cp3);
    opDelay1.checkpoints.add(cp5);
    opD1.checkpoints.add(cp5);
    // construct grouping that would be supplied through LogicalPlan
    Set<OperatorMeta> stronglyConnected = Sets.newHashSet(dag.getMeta(opB), dag.getMeta(opC), dag.getMeta(opDelay));
    Map<OperatorMeta, Set<OperatorMeta>> groups = new HashMap<>();
    for (OperatorMeta om : stronglyConnected) {
        groups.put(om, stronglyConnected);
    }
    UpdateCheckpointsContext ctx = new UpdateCheckpointsContext(clock, false, groups);
    scm.updateRecoveryCheckpoints(opB1, ctx, false);
    Assert.assertEquals("checkpoint " + opA1, Checkpoint.INITIAL_CHECKPOINT, opA1.getRecoveryCheckpoint());
    Assert.assertEquals("checkpoint " + opB1, cp3, opC1.getRecoveryCheckpoint());
    Assert.assertEquals("checkpoint " + opC1, cp3, opC1.getRecoveryCheckpoint());
    Assert.assertEquals("checkpoint " + opD1, cp5, opD1.getRecoveryCheckpoint());
}