Search in sources :

Example 1 with Checkpoint

use of com.datatorrent.stram.api.Checkpoint in project apex-core by apache.

the class CheckpointTest method testBackup.

/**
   * Test saving of operator state at window boundary.
   * @throws Exception
   */
@Test
public void testBackup() throws Exception {
    AsyncFSStorageAgent storageAgent = new AsyncFSStorageAgent(testMeta.getPath(), null);
    storageAgent.setSyncCheckpoint(true);
    dag.setAttribute(OperatorContext.STORAGE_AGENT, storageAgent);
    dag.setAttribute(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 1);
    dag.setAttribute(LogicalPlan.HEARTBEAT_INTERVAL_MILLIS, 50);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
    dag.setAttribute(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 50);
    MockInputOperator o1 = dag.addOperator("o1", new MockInputOperator());
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.setOperatorAttribute(o2, OperatorContext.STATELESS, true);
    dag.addStream("o1.outport", o1.outport, o2.inport1).setLocality(Locality.CONTAINER_LOCAL);
    StramLocalCluster sc = new StramLocalCluster(dag);
    sc.setHeartbeatMonitoringEnabled(false);
    sc.run();
    StreamingContainerManager dnm = sc.dnmgr;
    PhysicalPlan plan = dnm.getPhysicalPlan();
    Assert.assertEquals("number required containers", 1, dnm.getPhysicalPlan().getContainers().size());
    PTOperator o1p1 = plan.getOperators(dag.getMeta(o1)).get(0);
    Set<Long> checkpoints = Sets.newHashSet();
    for (long windowId : storageAgent.getWindowIds(o1p1.getId())) {
        checkpoints.add(windowId);
    }
    Assert.assertEquals("number checkpoints " + checkpoints, 3, checkpoints.size());
    Assert.assertTrue("contains " + checkpoints + " " + Stateless.WINDOW_ID, checkpoints.contains(Stateless.WINDOW_ID));
    PTOperator o2p1 = plan.getOperators(dag.getMeta(o2)).get(0);
    checkpoints = Sets.newHashSet();
    for (long windowId : storageAgent.getWindowIds(o2p1.getId())) {
        checkpoints.add(windowId);
    }
    Assert.assertEquals("number checkpoints " + checkpoints, 1, checkpoints.size());
    Assert.assertEquals("checkpoints " + o2p1, Sets.newHashSet(Stateless.WINDOW_ID), checkpoints);
    Assert.assertEquals("checkpoints " + o1p1 + " " + o1p1.checkpoints, 2, o1p1.checkpoints.size());
    Assert.assertNotNull("checkpoint not null for statefull operator " + o1p1, o1p1.stats.checkpointStats);
    for (Checkpoint cp : o1p1.checkpoints) {
        Object load = storageAgent.load(o1p1.getId(), cp.windowId);
        Assert.assertEquals("Stored Operator and Saved State", load.getClass(), o1p1.getOperatorMeta().getOperator().getClass());
    }
}
Also used : PhysicalPlan(com.datatorrent.stram.plan.physical.PhysicalPlan) Checkpoint(com.datatorrent.stram.api.Checkpoint) PTOperator(com.datatorrent.stram.plan.physical.PTOperator) GenericTestOperator(com.datatorrent.stram.engine.GenericTestOperator) AsyncFSStorageAgent(com.datatorrent.common.util.AsyncFSStorageAgent) Test(org.junit.Test)

Example 2 with Checkpoint

use of com.datatorrent.stram.api.Checkpoint in project apex-core by apache.

the class MockContainer method sendHeartbeat.

public void sendHeartbeat() {
    ContainerStats cstats = new ContainerStats(sca.container.getExternalId());
    ContainerHeartbeat hb = new ContainerHeartbeat();
    hb.setContainerStats(cstats);
    for (Map.Entry<Integer, MockOperatorStats> oe : this.stats.entrySet()) {
        OperatorHeartbeat ohb = new OperatorHeartbeat();
        ohb.setNodeId(oe.getKey());
        ohb.setState(oe.getValue().deployState);
        OperatorStats lstats = new OperatorStats();
        lstats.checkpoint = new Checkpoint(oe.getValue().checkpointWindowId, 0, 0);
        lstats.windowId = oe.getValue().currentWindowId;
        //stats.outputPorts = Lists.newArrayList();
        //PortStats ps = new PortStats(TestGeneratorInputOperator.OUTPUT_PORT);
        //ps.bufferServerBytes = 101;
        //ps.tupleCount = 1;
        //stats.outputPorts.add(ps);
        ohb.windowStats = Lists.newArrayList(lstats);
        cstats.operators.add(ohb);
    }
    ContainerHeartbeatResponse chr = sca.dnmgr.processHeartbeat(hb);
    Assert.assertNull(chr.deployRequest);
}
Also used : Checkpoint(com.datatorrent.stram.api.Checkpoint) ContainerStats(com.datatorrent.stram.api.StreamingContainerUmbilicalProtocol.ContainerStats) OperatorHeartbeat(com.datatorrent.stram.api.StreamingContainerUmbilicalProtocol.OperatorHeartbeat) ContainerHeartbeatResponse(com.datatorrent.stram.api.StreamingContainerUmbilicalProtocol.ContainerHeartbeatResponse) OperatorStats(com.datatorrent.api.Stats.OperatorStats) Map(java.util.Map) ContainerHeartbeat(com.datatorrent.stram.api.StreamingContainerUmbilicalProtocol.ContainerHeartbeat)

Example 3 with Checkpoint

use of com.datatorrent.stram.api.Checkpoint in project apex-core by apache.

the class PhysicalPlanTest method setActivationCheckpoint.

public static void setActivationCheckpoint(PTOperator oper, long windowId) {
    try {
        oper.operatorMeta.getValue(OperatorContext.STORAGE_AGENT).save(oper.operatorMeta.getOperator(), oper.id, windowId);
        Checkpoint cp = new Checkpoint(windowId, 0, 0);
        oper.setRecoveryCheckpoint(cp);
        oper.checkpoints.add(cp);
    } catch (Exception e) {
        Assert.fail(e.toString());
    }
}
Also used : Checkpoint(com.datatorrent.stram.api.Checkpoint)

Example 4 with Checkpoint

use of com.datatorrent.stram.api.Checkpoint in project apex-core by apache.

the class PhysicalPlan method assignContainers.

private void assignContainers(Set<PTContainer> newContainers, Set<PTContainer> releaseContainers) {
    Set<PTOperator> mxnUnifiers = Sets.newHashSet();
    for (PTOperator o : this.newOpers.keySet()) {
        mxnUnifiers.addAll(o.upstreamMerge.values());
    }
    Set<PTContainer> updatedContainers = Sets.newHashSet();
    HashMap<PTOperator, PTContainer> operatorContainerMap = Maps.newHashMap();
    for (Map.Entry<PTOperator, Operator> operEntry : this.newOpers.entrySet()) {
        PTOperator oper = operEntry.getKey();
        Checkpoint checkpoint = getActivationCheckpoint(operEntry.getKey());
        initCheckpoint(oper, operEntry.getValue(), checkpoint);
        if (mxnUnifiers.contains(operEntry.getKey())) {
            // MxN unifiers are assigned with the downstream operator
            continue;
        }
        PTContainer newContainer = null;
        int memoryMB = 0;
        // handle container locality
        for (PTOperator inlineOper : oper.getGrouping(Locality.CONTAINER_LOCAL).getOperatorSet()) {
            if (inlineOper.container != null) {
                newContainer = inlineOper.container;
                break;
            }
            memoryMB += inlineOper.operatorMeta.getValue(OperatorContext.MEMORY_MB);
            memoryMB += inlineOper.getBufferServerMemory();
        }
        if (newContainer == null) {
            int vCores = getVCores(oper.getGrouping(Locality.CONTAINER_LOCAL).getOperatorSet());
            // attempt to find empty container with required size
            for (PTContainer c : this.containers) {
                if (c.operators.isEmpty() && c.getState() == PTContainer.State.ACTIVE && c.getAllocatedMemoryMB() == memoryMB && c.getAllocatedVCores() == vCores) {
                    LOG.debug("Reusing existing container {} for {}", c, oper);
                    c.setRequiredMemoryMB(0);
                    c.setRequiredVCores(0);
                    newContainer = c;
                    break;
                }
            }
            if (newContainer == null) {
                // get new container
                LOG.debug("New container for: " + oper);
                newContainer = new PTContainer(this);
                newContainers.add(newContainer);
                containers.add(newContainer);
            }
            updatedContainers.add(newContainer);
        }
        setContainer(oper, newContainer);
    }
    // release containers that are no longer used and update operator to container map for applying anti-affinity
    for (PTContainer c : this.containers) {
        if (c.operators.isEmpty()) {
            LOG.debug("Container {} to be released", c);
            releaseContainers.add(c);
            containers.remove(c);
        } else {
            for (PTOperator oper : c.operators) {
                operatorContainerMap.put(oper, c);
            }
            c.getStrictAntiPrefs().clear();
            c.getPreferredAntiPrefs().clear();
        }
    }
    for (PTContainer c : updatedContainers) {
        updateContainerMemoryWithBufferServer(c);
        c.setRequiredVCores(getVCores(c.getOperators()));
    }
    AffinityRulesSet affinityRuleSet = dag.getAttributes().get(DAGContext.AFFINITY_RULES_SET);
    // Add anti-affinity restrictions in Containers
    if (affinityRuleSet != null && affinityRuleSet.getAffinityRules() != null) {
        setAntiAffinityForContainers(dag, affinityRuleSet.getAffinityRules(), operatorContainerMap);
    }
}
Also used : Operator(com.datatorrent.api.Operator) Checkpoint(com.datatorrent.stram.api.Checkpoint) Map(java.util.Map) HashMap(java.util.HashMap) ConcurrentMap(java.util.concurrent.ConcurrentMap) LinkedHashMap(java.util.LinkedHashMap) Checkpoint(com.datatorrent.stram.api.Checkpoint) AffinityRulesSet(com.datatorrent.api.AffinityRulesSet)

Example 5 with Checkpoint

use of com.datatorrent.stram.api.Checkpoint in project apex-core by apache.

the class PTOperator method addCheckpoint.

public Checkpoint addCheckpoint(long windowId, long startTime) {
    int widthMillis = operatorMeta.getDAG().getValue(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS);
    long millis = WindowGenerator.getNextWindowMillis(windowId, startTime, widthMillis);
    long count = WindowGenerator.getWindowCount(millis, startTime, widthMillis);
    Checkpoint c = new Checkpoint(windowId, (int) (count % operatorMeta.getValue(OperatorContext.APPLICATION_WINDOW_COUNT)), (int) (count % operatorMeta.getValue(OperatorContext.CHECKPOINT_WINDOW_COUNT)));
    this.checkpoints.add(c);
    return c;
}
Also used : Checkpoint(com.datatorrent.stram.api.Checkpoint) Checkpoint(com.datatorrent.stram.api.Checkpoint)

Aggregations

Checkpoint (com.datatorrent.stram.api.Checkpoint)28 PTOperator (com.datatorrent.stram.plan.physical.PTOperator)15 GenericTestOperator (com.datatorrent.stram.engine.GenericTestOperator)9 PhysicalPlan (com.datatorrent.stram.plan.physical.PhysicalPlan)9 Test (org.junit.Test)9 ContainerStats (com.datatorrent.stram.api.StreamingContainerUmbilicalProtocol.ContainerStats)6 OperatorMeta (com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta)6 MemoryStorageAgent (com.datatorrent.stram.support.StramTestSupport.MemoryStorageAgent)6 OperatorStats (com.datatorrent.api.Stats.OperatorStats)5 UpdateCheckpointsContext (com.datatorrent.stram.StreamingContainerManager.UpdateCheckpointsContext)5 HashMap (java.util.HashMap)5 Map (java.util.Map)5 OperatorHeartbeat (com.datatorrent.stram.api.StreamingContainerUmbilicalProtocol.OperatorHeartbeat)4 TestGeneratorInputOperator (com.datatorrent.stram.engine.TestGeneratorInputOperator)4 IOException (java.io.IOException)4 LinkedHashMap (java.util.LinkedHashMap)4 Operator (com.datatorrent.api.Operator)3 StatsListener (com.datatorrent.api.StatsListener)3 AsyncFSStorageAgent (com.datatorrent.common.util.AsyncFSStorageAgent)3 ContainerHeartbeatResponse (com.datatorrent.stram.api.StreamingContainerUmbilicalProtocol.ContainerHeartbeatResponse)3