use of com.datatorrent.stram.api.Checkpoint in project apex-core by apache.
the class CheckpointTest method testBackup.
/**
* Test saving of operator state at window boundary.
* @throws Exception
*/
@Test
public void testBackup() throws Exception {
AsyncFSStorageAgent storageAgent = new AsyncFSStorageAgent(testMeta.getPath(), null);
storageAgent.setSyncCheckpoint(true);
dag.setAttribute(OperatorContext.STORAGE_AGENT, storageAgent);
dag.setAttribute(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 1);
dag.setAttribute(LogicalPlan.HEARTBEAT_INTERVAL_MILLIS, 50);
dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
dag.setAttribute(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 50);
MockInputOperator o1 = dag.addOperator("o1", new MockInputOperator());
GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
dag.setOperatorAttribute(o2, OperatorContext.STATELESS, true);
dag.addStream("o1.outport", o1.outport, o2.inport1).setLocality(Locality.CONTAINER_LOCAL);
StramLocalCluster sc = new StramLocalCluster(dag);
sc.setHeartbeatMonitoringEnabled(false);
sc.run();
StreamingContainerManager dnm = sc.dnmgr;
PhysicalPlan plan = dnm.getPhysicalPlan();
Assert.assertEquals("number required containers", 1, dnm.getPhysicalPlan().getContainers().size());
PTOperator o1p1 = plan.getOperators(dag.getMeta(o1)).get(0);
Set<Long> checkpoints = Sets.newHashSet();
for (long windowId : storageAgent.getWindowIds(o1p1.getId())) {
checkpoints.add(windowId);
}
Assert.assertEquals("number checkpoints " + checkpoints, 3, checkpoints.size());
Assert.assertTrue("contains " + checkpoints + " " + Stateless.WINDOW_ID, checkpoints.contains(Stateless.WINDOW_ID));
PTOperator o2p1 = plan.getOperators(dag.getMeta(o2)).get(0);
checkpoints = Sets.newHashSet();
for (long windowId : storageAgent.getWindowIds(o2p1.getId())) {
checkpoints.add(windowId);
}
Assert.assertEquals("number checkpoints " + checkpoints, 1, checkpoints.size());
Assert.assertEquals("checkpoints " + o2p1, Sets.newHashSet(Stateless.WINDOW_ID), checkpoints);
Assert.assertEquals("checkpoints " + o1p1 + " " + o1p1.checkpoints, 2, o1p1.checkpoints.size());
Assert.assertNotNull("checkpoint not null for statefull operator " + o1p1, o1p1.stats.checkpointStats);
for (Checkpoint cp : o1p1.checkpoints) {
Object load = storageAgent.load(o1p1.getId(), cp.windowId);
Assert.assertEquals("Stored Operator and Saved State", load.getClass(), o1p1.getOperatorMeta().getOperator().getClass());
}
}
use of com.datatorrent.stram.api.Checkpoint in project apex-core by apache.
the class MockContainer method sendHeartbeat.
public void sendHeartbeat() {
ContainerStats cstats = new ContainerStats(sca.container.getExternalId());
ContainerHeartbeat hb = new ContainerHeartbeat();
hb.setContainerStats(cstats);
for (Map.Entry<Integer, MockOperatorStats> oe : this.stats.entrySet()) {
OperatorHeartbeat ohb = new OperatorHeartbeat();
ohb.setNodeId(oe.getKey());
ohb.setState(oe.getValue().deployState);
OperatorStats lstats = new OperatorStats();
lstats.checkpoint = new Checkpoint(oe.getValue().checkpointWindowId, 0, 0);
lstats.windowId = oe.getValue().currentWindowId;
//stats.outputPorts = Lists.newArrayList();
//PortStats ps = new PortStats(TestGeneratorInputOperator.OUTPUT_PORT);
//ps.bufferServerBytes = 101;
//ps.tupleCount = 1;
//stats.outputPorts.add(ps);
ohb.windowStats = Lists.newArrayList(lstats);
cstats.operators.add(ohb);
}
ContainerHeartbeatResponse chr = sca.dnmgr.processHeartbeat(hb);
Assert.assertNull(chr.deployRequest);
}
use of com.datatorrent.stram.api.Checkpoint in project apex-core by apache.
the class PhysicalPlanTest method setActivationCheckpoint.
public static void setActivationCheckpoint(PTOperator oper, long windowId) {
try {
oper.operatorMeta.getValue(OperatorContext.STORAGE_AGENT).save(oper.operatorMeta.getOperator(), oper.id, windowId);
Checkpoint cp = new Checkpoint(windowId, 0, 0);
oper.setRecoveryCheckpoint(cp);
oper.checkpoints.add(cp);
} catch (Exception e) {
Assert.fail(e.toString());
}
}
use of com.datatorrent.stram.api.Checkpoint in project apex-core by apache.
the class PhysicalPlan method assignContainers.
private void assignContainers(Set<PTContainer> newContainers, Set<PTContainer> releaseContainers) {
Set<PTOperator> mxnUnifiers = Sets.newHashSet();
for (PTOperator o : this.newOpers.keySet()) {
mxnUnifiers.addAll(o.upstreamMerge.values());
}
Set<PTContainer> updatedContainers = Sets.newHashSet();
HashMap<PTOperator, PTContainer> operatorContainerMap = Maps.newHashMap();
for (Map.Entry<PTOperator, Operator> operEntry : this.newOpers.entrySet()) {
PTOperator oper = operEntry.getKey();
Checkpoint checkpoint = getActivationCheckpoint(operEntry.getKey());
initCheckpoint(oper, operEntry.getValue(), checkpoint);
if (mxnUnifiers.contains(operEntry.getKey())) {
// MxN unifiers are assigned with the downstream operator
continue;
}
PTContainer newContainer = null;
int memoryMB = 0;
// handle container locality
for (PTOperator inlineOper : oper.getGrouping(Locality.CONTAINER_LOCAL).getOperatorSet()) {
if (inlineOper.container != null) {
newContainer = inlineOper.container;
break;
}
memoryMB += inlineOper.operatorMeta.getValue(OperatorContext.MEMORY_MB);
memoryMB += inlineOper.getBufferServerMemory();
}
if (newContainer == null) {
int vCores = getVCores(oper.getGrouping(Locality.CONTAINER_LOCAL).getOperatorSet());
// attempt to find empty container with required size
for (PTContainer c : this.containers) {
if (c.operators.isEmpty() && c.getState() == PTContainer.State.ACTIVE && c.getAllocatedMemoryMB() == memoryMB && c.getAllocatedVCores() == vCores) {
LOG.debug("Reusing existing container {} for {}", c, oper);
c.setRequiredMemoryMB(0);
c.setRequiredVCores(0);
newContainer = c;
break;
}
}
if (newContainer == null) {
// get new container
LOG.debug("New container for: " + oper);
newContainer = new PTContainer(this);
newContainers.add(newContainer);
containers.add(newContainer);
}
updatedContainers.add(newContainer);
}
setContainer(oper, newContainer);
}
// release containers that are no longer used and update operator to container map for applying anti-affinity
for (PTContainer c : this.containers) {
if (c.operators.isEmpty()) {
LOG.debug("Container {} to be released", c);
releaseContainers.add(c);
containers.remove(c);
} else {
for (PTOperator oper : c.operators) {
operatorContainerMap.put(oper, c);
}
c.getStrictAntiPrefs().clear();
c.getPreferredAntiPrefs().clear();
}
}
for (PTContainer c : updatedContainers) {
updateContainerMemoryWithBufferServer(c);
c.setRequiredVCores(getVCores(c.getOperators()));
}
AffinityRulesSet affinityRuleSet = dag.getAttributes().get(DAGContext.AFFINITY_RULES_SET);
// Add anti-affinity restrictions in Containers
if (affinityRuleSet != null && affinityRuleSet.getAffinityRules() != null) {
setAntiAffinityForContainers(dag, affinityRuleSet.getAffinityRules(), operatorContainerMap);
}
}
use of com.datatorrent.stram.api.Checkpoint in project apex-core by apache.
the class PTOperator method addCheckpoint.
public Checkpoint addCheckpoint(long windowId, long startTime) {
int widthMillis = operatorMeta.getDAG().getValue(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS);
long millis = WindowGenerator.getNextWindowMillis(windowId, startTime, widthMillis);
long count = WindowGenerator.getWindowCount(millis, startTime, widthMillis);
Checkpoint c = new Checkpoint(windowId, (int) (count % operatorMeta.getValue(OperatorContext.APPLICATION_WINDOW_COUNT)), (int) (count % operatorMeta.getValue(OperatorContext.CHECKPOINT_WINDOW_COUNT)));
this.checkpoints.add(c);
return c;
}
Aggregations