Examples with FSStorageAgent com.datatorrent.common.util.FSStorageAgent used on opensource projects

Example 6 with FSStorageAgent

use of com.datatorrent.common.util.FSStorageAgent in project apex-core by apache.

the class CascadeStorageAgentTest method testSingleIndirection.

@Test
public void testSingleIndirection() throws IOException {
    String oldAppPath = testMeta.applicationPath;
    FSStorageAgent storageAgent = new FSStorageAgent(oldAppPath, null);
    storageAgent.save("1", 1, 1);
    storageAgent.save("2", 1, 2);
    storageAgent.save("3", 2, 1);
    String newAppPath = oldAppPath + ".new";
    CascadeStorageAgent cascade = new CascadeStorageAgent(storageAgent, new FSStorageAgent(newAppPath, null));
    long[] operatorIds = cascade.getWindowIds(1);
    Assert.assertArrayEquals("Returned window ids ", operatorIds, new long[] { 1L, 2L });
    operatorIds = cascade.getWindowIds(2);
    Assert.assertArrayEquals("Returned window ids ", operatorIds, new long[] { 1L });
    /* save should happen to new location */
    cascade.save("4", 1, 4);
    FileContext fileContext = FileContext.getFileContext();
    Assert.assertFalse("operator 1 window 4 file does not exists in old directory", fileContext.util().exists(new Path(oldAppPath + "/" + 1 + "/" + 4)));
    Assert.assertTrue("operator 1 window 4 file exists in new directory", fileContext.util().exists(new Path(newAppPath + "/" + 1 + "/" + 4)));
    // check for delete,
    // delete for old checkpoint should be ignored
    cascade.save("5", 1, 5);
    cascade.delete(1, 2L);
    Assert.assertTrue("operator 1 window 2 file exists in old directory", fileContext.util().exists(new Path(oldAppPath + "/" + 1 + "/" + 2)));
    cascade.delete(1, 4L);
    Assert.assertFalse("operator 1 window 4 file does not exists in old directory", fileContext.util().exists(new Path(newAppPath + "/" + 1 + "/" + 4)));
    /* chaining of storage agent */
    String latestAppPath = oldAppPath + ".latest";
    cascade = new CascadeStorageAgent(storageAgent, new FSStorageAgent(newAppPath, null));
    CascadeStorageAgent latest = new CascadeStorageAgent(cascade, new FSStorageAgent(latestAppPath, null));
    operatorIds = latest.getWindowIds(1);
    Assert.assertArrayEquals("Window ids ", operatorIds, new long[] { 1, 2, 5 });
    latest.save("6", 1, 6);
    Assert.assertFalse("operator 1 window 6 file does not exists in old directory", fileContext.util().exists(new Path(oldAppPath + "/" + 1 + "/" + 6)));
    Assert.assertFalse("operator 1 window 6 file does not exists in old directory", fileContext.util().exists(new Path(newAppPath + "/" + 1 + "/" + 6)));
    Assert.assertTrue("operator 1 window 6 file exists in new directory", fileContext.util().exists(new Path(latestAppPath + "/" + 1 + "/" + 6)));
}

Example 7 with FSStorageAgent

use of com.datatorrent.common.util.FSStorageAgent in project apex-core by apache.

the class CheckpointTest method testBeforeCheckpointNotification.

@Test
public void testBeforeCheckpointNotification() throws IOException, ClassNotFoundException {
    FSStorageAgent storageAgent = new FSStorageAgent(testMeta.getPath(), null);
    dag.setAttribute(OperatorContext.STORAGE_AGENT, storageAgent);
    dag.setAttribute(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 1);
    dag.setAttribute(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 50);
    MockInputOperator o1 = dag.addOperator("o1", new MockInputOperator());
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.setOperatorAttribute(o2, OperatorContext.STATELESS, true);
    dag.addStream("o1.outport", o1.outport, o2.inport1);
    StramLocalCluster sc = new StramLocalCluster(dag);
    sc.setHeartbeatMonitoringEnabled(false);
    sc.run();
    StreamingContainerManager dnm = sc.dnmgr;
    PhysicalPlan plan = dnm.getPhysicalPlan();
    List<PTOperator> o1ps = plan.getOperators(dag.getMeta(o1));
    Assert.assertEquals("Number partitions", 1, o1ps.size());
    PTOperator o1p1 = o1ps.get(0);
    long[] ckWIds = storageAgent.getWindowIds(o1p1.getId());
    Arrays.sort(ckWIds);
    int expectedState = 0;
    for (long windowId : ckWIds) {
        Object ckState = storageAgent.load(o1p1.getId(), windowId);
        Assert.assertEquals("Checkpointed state class", MockInputOperator.class, ckState.getClass());
        Assert.assertEquals("Checkpoint state", expectedState++, ((MockInputOperator) ckState).checkpointState);
    }
}

Example 8 with FSStorageAgent

use of com.datatorrent.common.util.FSStorageAgent in project apex-core by apache.

the class NodeTest method testDoubleCheckpointHandling.

@SuppressWarnings("SleepWhileInLoop")
public static void testDoubleCheckpointHandling(ProcessingMode processingMode, boolean trueGenericFalseInput, String path) throws Exception {
    WindowGenerator windowGenerator = new WindowGenerator(new ScheduledThreadPoolExecutor(1, "WindowGenerator"), 1024);
    windowGenerator.setResetWindow(0L);
    windowGenerator.setFirstWindow(0L);
    windowGenerator.setWindowWidth(100);
    windowGenerator.setCheckpointCount(1, 0);
    CheckpointTestOperator checkpointTestOperator;
    if (trueGenericFalseInput) {
        checkpointTestOperator = new CheckpointTestOperator();
    } else {
        checkpointTestOperator = new InputCheckpointTestOperator();
    }
    DefaultAttributeMap dam = new DefaultAttributeMap();
    dam.put(com.datatorrent.stram.engine.OperatorContext.APPLICATION_WINDOW_COUNT, 2);
    dam.put(com.datatorrent.stram.engine.OperatorContext.CHECKPOINT_WINDOW_COUNT, 2);
    dam.put(com.datatorrent.stram.engine.OperatorContext.PROCESSING_MODE, processingMode);
    dam.put(com.datatorrent.stram.engine.OperatorContext.STORAGE_AGENT, new FSStorageAgent(path, new Configuration()));
    final Node in;
    if (trueGenericFalseInput) {
        in = new GenericNode(checkpointTestOperator, new com.datatorrent.stram.engine.OperatorContext(0, "operator", dam, null));
    } else {
        in = new InputNode((InputCheckpointTestOperator) checkpointTestOperator, new com.datatorrent.stram.engine.OperatorContext(0, "operator", dam, null));
    }
    in.setId(1);
    TestSink testSink = new TestSink();
    String inputPort;
    if (trueGenericFalseInput) {
        inputPort = "ip1";
    } else {
        inputPort = Node.INPUT;
    }
    in.connectInputPort(inputPort, windowGenerator.acquireReservoir(String.valueOf(in.id), 1024));
    in.connectOutputPort("output", testSink);
    in.firstWindowMillis = 0;
    in.windowWidthMillis = 100;
    windowGenerator.activate(null);
    Thread t = new Thread() {

        @Override
        public void run() {
            in.activate();
            in.run();
            in.deactivate();
        }
    };
    t.start();
    long startTime = System.currentTimeMillis();
    long endTime = 0;
    while (checkpointTestOperator.numWindows < 3 && ((endTime = System.currentTimeMillis()) - startTime) < 6000) {
        Thread.sleep(50);
    }
    in.shutdown();
    t.join();
    windowGenerator.deactivate();
    Assert.assertFalse(checkpointTestOperator.checkpointTwice);
    Assert.assertTrue("Timed out", (endTime - startTime) < 5000);
}