Examples with StatsListener com.datatorrent.api.StatsListener used on opensource projects

Example 6 with StatsListener

use of com.datatorrent.api.StatsListener in project apex-core by apache.

the class PhysicalPlanTest method testInputOperatorPartitioning.

/**
   * Test partitioning of an input operator (no input port).
   * Cover aspects that are not part of generic operator test.
   * Test scaling from one to multiple partitions with unifier when one partition remains unmodified.
   */
@Test
public void testInputOperatorPartitioning() {
    LogicalPlan dag = new LogicalPlan();
    final TestInputOperator<Object> o1 = dag.addOperator("o1", new TestInputOperator<>());
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.addStream("o1.outport1", o1.output, o2.inport1);
    OperatorMeta o1Meta = dag.getMeta(o1);
    dag.setOperatorAttribute(o1, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { new PartitioningTest.PartitionLoadWatch() }));
    TestPartitioner<TestInputOperator<Object>> partitioner = new TestPartitioner<>();
    dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, partitioner);
    TestPlanContext ctx = new TestPlanContext();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    Assert.assertEquals("number of containers", 2, plan.getContainers().size());
    List<PTOperator> o1Partitions = plan.getOperators(o1Meta);
    Assert.assertEquals("partitions " + o1Partitions, 1, o1Partitions.size());
    PTOperator o1p1 = o1Partitions.get(0);
    // verify load update generates expected events per configuration
    Assert.assertEquals("stats handlers " + o1p1, 1, o1p1.statsListeners.size());
    StatsListener l = o1p1.statsListeners.get(0);
    Assert.assertTrue("stats handlers " + o1p1.statsListeners, l instanceof PartitioningTest.PartitionLoadWatch);
    PartitioningTest.PartitionLoadWatch.put(o1p1, 1);
    plan.onStatusUpdate(o1p1);
    Assert.assertEquals("scale up triggered", 1, ctx.events.size());
    // add another partition, keep existing as is
    partitioner.extraPartitions.add(new DefaultPartition<>(o1));
    Runnable r = ctx.events.remove(0);
    r.run();
    partitioner.extraPartitions.clear();
    o1Partitions = plan.getOperators(o1Meta);
    Assert.assertEquals("operators after scale up", 2, o1Partitions.size());
    Assert.assertEquals("first partition unmodified", o1p1, o1Partitions.get(0));
    Assert.assertEquals("single output", 1, o1p1.getOutputs().size());
    Assert.assertEquals("output to unifier", 1, o1p1.getOutputs().get(0).sinks.size());
    Set<PTOperator> expUndeploy = Sets.newHashSet(plan.getOperators(dag.getMeta(o2)));
    Set<PTOperator> expDeploy = Sets.newHashSet(o1Partitions.get(1));
    expDeploy.addAll(plan.getMergeOperators(dag.getMeta(o1)));
    expDeploy.addAll(expUndeploy);
    expDeploy.add(o1p1.getOutputs().get(0).sinks.get(0).target);
    Assert.assertEquals("undeploy", expUndeploy, ctx.undeploy);
    Assert.assertEquals("deploy", expDeploy, ctx.deploy);
    for (PTOperator p : o1Partitions) {
        Assert.assertEquals("activation window id " + p, Checkpoint.INITIAL_CHECKPOINT, p.recoveryCheckpoint);
        Assert.assertEquals("checkpoints " + p + " " + p.checkpoints, Lists.newArrayList(), p.checkpoints);
        PartitioningTest.PartitionLoadWatch.put(p, -1);
        plan.onStatusUpdate(p);
    }
    ctx.events.remove(0).run();
    Assert.assertEquals("operators after scale down", 1, plan.getOperators(o1Meta).size());
}

Example 7 with StatsListener

use of com.datatorrent.api.StatsListener in project apex-core by apache.

the class PhysicalPlan method onStatusUpdate.

public void onStatusUpdate(PTOperator oper) {
    for (StatsListener l : oper.statsListeners) {
        final StatsListener.Response rsp = l.processStats(oper.stats);
        if (rsp != null) {
            //LOG.debug("Response to processStats = {}", rsp.repartitionRequired);
            oper.loadIndicator = rsp.loadIndicator;
            if (rsp.repartitionRequired) {
                final OperatorMeta om = oper.getOperatorMeta();
                // concurrent heartbeat processing
                if (this.pendingRepartition.putIfAbsent(om, om) != null) {
                    LOG.debug("Skipping repartitioning for {} load {}", oper, oper.loadIndicator);
                } else {
                    LOG.debug("Scheduling repartitioning for {} load {}", oper, oper.loadIndicator);
                    // hand over to monitor thread
                    Runnable r = new Runnable() {

                        @Override
                        public void run() {
                            redoPartitions(logicalToPTOperator.get(om), rsp.repartitionNote);
                            pendingRepartition.remove(om);
                        }
                    };
                    ctx.dispatch(r);
                }
            }
            if (rsp.operatorRequests != null) {
                for (OperatorRequest cmd : rsp.operatorRequests) {
                    StramToNodeRequest request = new StramToNodeRequest();
                    request.operatorId = oper.getId();
                    request.requestType = StramToNodeRequest.RequestType.CUSTOM;
                    request.cmd = cmd;
                    ctx.addOperatorRequest(oper, request);
                }
            }
            // for backward compatibility
            if (rsp.operatorCommands != null) {
                for (@SuppressWarnings("deprecation") com.datatorrent.api.StatsListener.OperatorCommand cmd : rsp.operatorCommands) {
                    StramToNodeRequest request = new StramToNodeRequest();
                    request.operatorId = oper.getId();
                    request.requestType = StramToNodeRequest.RequestType.CUSTOM;
                    OperatorCommandConverter converter = new OperatorCommandConverter();
                    converter.cmd = cmd;
                    request.cmd = converter;
                    ctx.addOperatorRequest(oper, request);
                }
            }
        }
    }
}

Example 8 with StatsListener

use of com.datatorrent.api.StatsListener in project apex-core by apache.

the class PhysicalPlanTest method testDefaultPartitionerWithParallel.

@Test
public void testDefaultPartitionerWithParallel() throws InterruptedException {
    final MutableInt loadInd = new MutableInt();
    StatsListener listener = new StatsListener() {

        @Override
        public Response processStats(BatchedOperatorStats stats) {
            Response response = new Response();
            response.repartitionRequired = true;
            response.loadIndicator = loadInd.intValue();
            return response;
        }
    };
    LogicalPlan dag = new LogicalPlan();
    GenericTestOperator nodeX = dag.addOperator("X", GenericTestOperator.class);
    dag.setOperatorAttribute(nodeX, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
    dag.setOperatorAttribute(nodeX, Context.OperatorContext.STATS_LISTENERS, Lists.newArrayList(listener));
    GenericTestOperator nodeY = dag.addOperator("Y", GenericTestOperator.class);
    dag.setOperatorAttribute(nodeY, Context.OperatorContext.PARTITIONER, new TestPartitioner<GenericTestOperator>());
    GenericTestOperator nodeZ = dag.addOperator("Z", GenericTestOperator.class);
    dag.addStream("Stream1", nodeX.outport1, nodeY.inport1, nodeZ.inport1);
    dag.addStream("Stream2", nodeX.outport2, nodeY.inport2, nodeZ.inport2);
    dag.setInputPortAttribute(nodeY.inport1, Context.PortContext.PARTITION_PARALLEL, true);
    dag.setInputPortAttribute(nodeY.inport2, Context.PortContext.PARTITION_PARALLEL, true);
    dag.setInputPortAttribute(nodeZ.inport1, Context.PortContext.PARTITION_PARALLEL, true);
    dag.setInputPortAttribute(nodeZ.inport2, Context.PortContext.PARTITION_PARALLEL, true);
    StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);
    TestPlanContext ctx = new TestPlanContext();
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    LogicalPlan.OperatorMeta metaOfX = dag.getMeta(nodeX);
    LogicalPlan.OperatorMeta metaOfY = dag.getMeta(nodeY);
    Assert.assertEquals("number operators " + metaOfX.getName(), 2, plan.getOperators(metaOfX).size());
    Assert.assertEquals("number operators " + metaOfY.getName(), 2, plan.getOperators(metaOfY).size());
    List<PTOperator> ptOfX = plan.getOperators(metaOfX);
    for (PTOperator physicalX : ptOfX) {
        Assert.assertEquals("2 streams " + physicalX.getOutputs(), 2, physicalX.getOutputs().size());
        for (PTOutput outputPort : physicalX.getOutputs()) {
            Set<PTOperator> dopers = Sets.newHashSet();
            Assert.assertEquals("sink of " + metaOfX.getName() + " id " + physicalX.id + " port " + outputPort.portName, 2, outputPort.sinks.size());
            for (PTInput inputPort : outputPort.sinks) {
                dopers.add(inputPort.target);
            }
            Assert.assertEquals(2, dopers.size());
        }
    }
    //Invoke redo-partition of PhysicalPlan, no partition change
    loadInd.setValue(0);
    for (PTOperator ptOperator : ptOfX) {
        plan.onStatusUpdate(ptOperator);
    }
    ctx.events.remove(0).run();
    for (PTOperator physicalX : ptOfX) {
        Assert.assertEquals("2 streams " + physicalX.getOutputs(), 2, physicalX.getOutputs().size());
        for (PTOutput outputPort : physicalX.getOutputs()) {
            Set<PTOperator> dopers = Sets.newHashSet();
            Assert.assertEquals("sink of " + metaOfX.getName() + " id " + physicalX.id + " port " + outputPort.portName, 2, outputPort.sinks.size());
            for (PTInput inputPort : outputPort.sinks) {
                dopers.add(inputPort.target);
            }
            Assert.assertEquals(2, dopers.size());
        }
    }
    //scale up by splitting first partition
    loadInd.setValue(1);
    plan.onStatusUpdate(ptOfX.get(0));
    ctx.events.get(0).run();
    List<PTOperator> ptOfXScaleUp = plan.getOperators(metaOfX);
    Assert.assertEquals("3 partitons " + ptOfXScaleUp, 3, ptOfXScaleUp.size());
    for (PTOperator physicalX : ptOfXScaleUp) {
        Assert.assertEquals("2 streams " + physicalX.getOutputs(), 2, physicalX.getOutputs().size());
        for (PTOutput outputPort : physicalX.getOutputs()) {
            Set<PTOperator> dopers = Sets.newHashSet();
            Assert.assertEquals("sink of " + metaOfX.getName() + " id " + physicalX.id + " port " + outputPort.portName, 2, outputPort.sinks.size());
            for (PTInput inputPort : outputPort.sinks) {
                dopers.add(inputPort.target);
            }
            Assert.assertEquals(2, dopers.size());
        }
    }
}

Example 9 with StatsListener

use of com.datatorrent.api.StatsListener in project apex-core by apache.

the class LogicalPlanConfigurationTest method testLoadFromJson.

@Test
public void testLoadFromJson() throws Exception {
    String resourcePath = "/testTopology.json";
    InputStream is = this.getClass().getResourceAsStream(resourcePath);
    if (is == null) {
        fail("Could not load " + resourcePath);
    }
    StringWriter writer = new StringWriter();
    IOUtils.copy(is, writer);
    JSONObject json = new JSONObject(writer.toString());
    Configuration conf = new Configuration(false);
    conf.set(StreamingApplication.APEX_PREFIX + "operator.operator3.prop.myStringProperty", "o3StringFromConf");
    LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
    LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
    dag.validate();
    assertEquals("DAG attribute CONTAINER_JVM_OPTIONS ", dag.getAttributes().get(DAGContext.CONTAINER_JVM_OPTIONS), "-Xmx16m");
    Map<Class<?>, Class<? extends StringCodec<?>>> stringCodecsMap = Maps.newHashMap();
    stringCodecsMap.put(Integer.class, Integer2String.class);
    assertEquals("DAG attribute STRING_CODECS ", stringCodecsMap, dag.getAttributes().get(DAGContext.STRING_CODECS));
    assertEquals("DAG attribute CONTAINER_OPTS_CONFIGURATOR ", BasicContainerOptConfigurator.class, dag.getAttributes().get(DAGContext.CONTAINER_OPTS_CONFIGURATOR).getClass());
    assertEquals("number of operator confs", 5, dag.getAllOperators().size());
    assertEquals("number of root operators", 1, dag.getRootOperators().size());
    StreamMeta s1 = dag.getStream("n1n2");
    assertNotNull(s1);
    assertTrue("n1n2 inline", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());
    OperatorMeta input = dag.getOperatorMeta("inputOperator");
    TestStatsListener tsl = new TestStatsListener();
    tsl.setIntProp(222);
    List<StatsListener> sll = Lists.<StatsListener>newArrayList(tsl);
    assertEquals("inputOperator STATS_LISTENERS attribute ", sll, input.getAttributes().get(OperatorContext.STATS_LISTENERS));
    for (OutputPortMeta opm : input.getOutputStreams().keySet()) {
        assertTrue("output port of input Operator attribute is JsonStreamCodec ", opm.getAttributes().get(PortContext.STREAM_CODEC) instanceof JsonStreamCodec<?>);
    }
    OperatorMeta operator3 = dag.getOperatorMeta("operator3");
    assertEquals("operator3.classname", GenericTestOperator.class, operator3.getOperator().getClass());
    GenericTestOperator doperator3 = (GenericTestOperator) operator3.getOperator();
    assertEquals("myStringProperty " + doperator3, "o3StringFromConf", doperator3.getMyStringProperty());
    assertFalse("booleanProperty " + doperator3, doperator3.booleanProperty);
    OperatorMeta operator4 = dag.getOperatorMeta("operator4");
    GenericTestOperator doperator4 = (GenericTestOperator) operator4.getOperator();
    assertEquals("myStringProperty " + doperator4, "overrideOperator4", doperator4.getMyStringProperty());
    assertEquals("setterOnlyOperator4 " + doperator4, "setterOnlyOperator4", doperator4.propertySetterOnly);
    assertTrue("booleanProperty " + doperator4, doperator4.booleanProperty);
    StreamMeta input1 = dag.getStream("inputStream");
    assertNotNull(input1);
    OperatorMeta inputOperator = dag.getOperatorMeta("inputOperator");
    Assert.assertEquals("input1 source", inputOperator, input1.getSource().getOperatorMeta());
    Set<OperatorMeta> targetNodes = Sets.newHashSet();
    for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
        targetNodes.add(targetPort.getOperatorMeta());
    }
    Assert.assertEquals("operator attribute " + inputOperator, 64, (int) inputOperator.getValue(OperatorContext.MEMORY_MB));
    Assert.assertEquals("port attribute " + inputOperator, 8, (int) input1.getSource().getValue(PortContext.UNIFIER_LIMIT));
    Assert.assertEquals("input1 target ", Sets.newHashSet(dag.getOperatorMeta("operator1"), operator3, operator4), targetNodes);
}

Example 10 with StatsListener

use of com.datatorrent.api.StatsListener in project apex-core by apache.

the class Node method activate.

@SuppressWarnings("unchecked")
public void activate() {
    alive = true;
    APPLICATION_WINDOW_COUNT = context.getValue(OperatorContext.APPLICATION_WINDOW_COUNT);
    if (context.getValue(OperatorContext.SLIDE_BY_WINDOW_COUNT) != null) {
        int slidingWindowCount = context.getValue(OperatorContext.SLIDE_BY_WINDOW_COUNT);
        APPLICATION_WINDOW_COUNT = IntMath.gcd(APPLICATION_WINDOW_COUNT, slidingWindowCount);
    }
    DAG_CHECKPOINT_WINDOW_COUNT = context.getValue(Context.DAGContext.CHECKPOINT_WINDOW_COUNT);
    CHECKPOINT_WINDOW_COUNT = context.getValue(OperatorContext.CHECKPOINT_WINDOW_COUNT);
    Collection<StatsListener> statsListeners = context.getValue(OperatorContext.STATS_LISTENERS);
    if (CHECKPOINT_WINDOW_COUNT % APPLICATION_WINDOW_COUNT != 0) {
        logger.warn("{} is not exact multiple of {} for operator {}. This may cause side effects such as processing to begin without beginWindow preceding it in the first window after activation.", OperatorContext.CHECKPOINT_WINDOW_COUNT, OperatorContext.APPLICATION_WINDOW_COUNT, operator);
    }
    PROCESSING_MODE = context.getValue(OperatorContext.PROCESSING_MODE);
    if (PROCESSING_MODE == ProcessingMode.EXACTLY_ONCE && CHECKPOINT_WINDOW_COUNT != 1) {
        logger.warn("Ignoring {} attribute in favor of {} processing mode", OperatorContext.CHECKPOINT_WINDOW_COUNT.getSimpleName(), ProcessingMode.EXACTLY_ONCE.name());
        CHECKPOINT_WINDOW_COUNT = 1;
    }
    activateSinks();
    if (operator instanceof Operator.ActivationListener) {
        ((Operator.ActivationListener<OperatorContext>) operator).activate(context);
    }
    if (statsListeners != null) {
        Iterator<StatsListener> iterator = statsListeners.iterator();
        while (iterator.hasNext()) {
            DATA_TUPLE_AWARE = iterator.next().getClass().isAnnotationPresent(StatsListener.DataQueueSize.class);
            if (DATA_TUPLE_AWARE) {
                break;
            }
        }
    }
    if (!DATA_TUPLE_AWARE && (operator instanceof StatsListener)) {
        DATA_TUPLE_AWARE = operator.getClass().isAnnotationPresent(StatsListener.DataQueueSize.class);
    }
    /*
     * If there were any requests which needed to be executed before the operator started
     * its normal execution, execute those requests now - e.g. Restarting the operator
     * recording for the operators which failed while recording and being replaced.
     */
    handleRequests(currentWindowId);
}