Examples with StateMap org.apache.nifi.components.state.StateMap used on opensource projects

Example 56 with StateMap

use of org.apache.nifi.components.state.StateMap in project nifi by apache.

the class ITRedisStateProvider method testClear.

@Test
public void testClear() throws IOException {
    final StateProvider provider = getProvider();
    StateMap stateMap = provider.getState(componentId);
    assertNotNull(stateMap);
    assertEquals(-1L, stateMap.getVersion());
    assertTrue(stateMap.toMap().isEmpty());
    provider.setState(Collections.singletonMap("testClear", "value"), componentId);
    stateMap = provider.getState(componentId);
    assertNotNull(stateMap);
    assertEquals(0, stateMap.getVersion());
    assertEquals("value", stateMap.get("testClear"));
    provider.clear(componentId);
    stateMap = provider.getState(componentId);
    assertNotNull(stateMap);
    assertEquals(1L, stateMap.getVersion());
    assertTrue(stateMap.toMap().isEmpty());
}

Example 57 with StateMap

use of org.apache.nifi.components.state.StateMap in project nifi by apache.

the class TailFile method recoverState.

@OnScheduled
public void recoverState(final ProcessContext context) throws IOException {
    // set isMultiChanging
    isMultiChanging.set(context.getProperty(MODE).getValue().equals(MODE_MULTIFILE.getValue()));
    // set last lookup to now
    lastLookup.set(new Date().getTime());
    // maxAge
    long maxAge = context.getProperty(MAXIMUM_AGE).getValue() == null ? Long.MAX_VALUE : context.getProperty(MAXIMUM_AGE).asTimePeriod(TimeUnit.MILLISECONDS);
    // get list of files to tail
    List<String> filesToTail = new ArrayList<String>();
    if (context.getProperty(MODE).getValue().equals(MODE_MULTIFILE.getValue())) {
        filesToTail.addAll(getFilesToTail(context.getProperty(BASE_DIRECTORY).evaluateAttributeExpressions().getValue(), context.getProperty(FILENAME).evaluateAttributeExpressions().getValue(), context.getProperty(RECURSIVE).asBoolean(), maxAge));
    } else {
        filesToTail.add(context.getProperty(FILENAME).evaluateAttributeExpressions().getValue());
    }
    final Scope scope = getStateScope(context);
    final StateMap stateMap = context.getStateManager().getState(scope);
    if (stateMap.getVersion() == -1L) {
        // state has been cleared or never stored so recover as 'empty state'
        initStates(filesToTail, Collections.emptyMap(), true);
        recoverState(context, filesToTail, Collections.emptyMap());
        return;
    }
    Map<String, String> statesMap = stateMap.toMap();
    if (statesMap.containsKey(TailFileState.StateKeys.FILENAME) && !statesMap.keySet().stream().anyMatch(key -> key.startsWith(MAP_PREFIX))) {
        // If statesMap contains "filename" key without "file.0." prefix,
        // and there's no key with "file." prefix, then
        // it indicates that the statesMap is created with earlier version of NiFi.
        // In this case, we need to migrate the state by adding prefix indexed with 0.
        final Map<String, String> migratedStatesMap = new HashMap<>(statesMap.size());
        for (String key : statesMap.keySet()) {
            migratedStatesMap.put(MAP_PREFIX + "0." + key, statesMap.get(key));
        }
        // LENGTH is added from NiFi 1.1.0. Set the value with using the last position so that we can use existing state
        // to avoid sending duplicated log data after updating NiFi.
        migratedStatesMap.put(MAP_PREFIX + "0." + TailFileState.StateKeys.LENGTH, statesMap.get(TailFileState.StateKeys.POSITION));
        statesMap = Collections.unmodifiableMap(migratedStatesMap);
        getLogger().info("statesMap has been migrated. {}", new Object[] { migratedStatesMap });
    }
    initStates(filesToTail, statesMap, false);
    recoverState(context, filesToTail, statesMap);
}

Example 58 with StateMap

use of org.apache.nifi.components.state.StateMap in project nifi by apache.

the class TestMonitorActivity method testClusterMonitorActiveCopyAttribute.

@Test
public void testClusterMonitorActiveCopyAttribute() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(new TestableProcessor(100));
    runner.setClustered(true);
    runner.setPrimaryNode(false);
    runner.setProperty(MonitorActivity.MONITORING_SCOPE, MonitorActivity.SCOPE_CLUSTER);
    // This has to be very small threshold, otherwise, MonitorActivity skip persisting state.
    runner.setProperty(MonitorActivity.THRESHOLD, "1 ms");
    runner.setProperty(MonitorActivity.COPY_ATTRIBUTES, "true");
    final HashMap<String, String> attributes = new HashMap<>();
    attributes.put("key1", "value1");
    attributes.put("key2", "value2");
    runner.enqueue("Incoming data", attributes);
    runner.run();
    runner.assertAllFlowFilesTransferred(MonitorActivity.REL_SUCCESS);
    final StateMap updatedState = runner.getStateManager().getState(Scope.CLUSTER);
    assertNotNull("Latest timestamp should be persisted", updatedState.get(MonitorActivity.STATE_KEY_LATEST_SUCCESS_TRANSFER));
    assertEquals("value1", updatedState.get("key1"));
    assertEquals("value2", updatedState.get("key2"));
}

Example 59 with StateMap

use of org.apache.nifi.components.state.StateMap in project nifi by apache.

the class TestMonitorActivity method testClusterMonitorActivityRestoredBySelfOnNode.

@Test
public void testClusterMonitorActivityRestoredBySelfOnNode() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(new TestableProcessor(10000));
    runner.setClustered(true);
    runner.setPrimaryNode(false);
    runner.setProperty(MonitorActivity.MONITORING_SCOPE, MonitorActivity.SCOPE_CLUSTER);
    runner.setProperty(MonitorActivity.REPORTING_NODE, MonitorActivity.REPORT_NODE_PRIMARY);
    runner.setProperty(MonitorActivity.THRESHOLD, "100 ms");
    runner.setProperty(MonitorActivity.COPY_ATTRIBUTES, "true");
    // Becomes inactive
    runner.run();
    // This node won't send notification files
    runner.assertTransferCount(MonitorActivity.REL_INACTIVE, 0);
    runner.clearTransferState();
    // Activity restored
    final HashMap<String, String> attributes = new HashMap<>();
    attributes.put("key1", "value1");
    attributes.put("key2", "value2");
    runner.enqueue("Incoming data", attributes);
    runNext(runner);
    // This node should not send restored flow file
    runner.assertAllFlowFilesTransferred(MonitorActivity.REL_SUCCESS, 1);
    // Latest activity should be persisted
    final StateMap updatedState = runner.getStateManager().getState(Scope.CLUSTER);
    assertNotNull("Latest timestamp should be persisted", updatedState.get(MonitorActivity.STATE_KEY_LATEST_SUCCESS_TRANSFER));
    assertEquals("value1", updatedState.get("key1"));
    assertEquals("value2", updatedState.get("key2"));
    runner.clearTransferState();
}

Example 60 with StateMap

use of org.apache.nifi.components.state.StateMap in project nifi by apache.

the class TestMonitorActivity method testClusterMonitorActiveFallbackToNodeScope.

@Test
public void testClusterMonitorActiveFallbackToNodeScope() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(new TestableProcessor(100));
    runner.setClustered(false);
    runner.setPrimaryNode(false);
    runner.setProperty(MonitorActivity.MONITORING_SCOPE, MonitorActivity.SCOPE_CLUSTER);
    // This has to be very small threshold, otherwise, MonitorActivity skip persisting state.
    runner.setProperty(MonitorActivity.THRESHOLD, "1 ms");
    runner.enqueue("Incoming data");
    runner.run();
    runner.assertAllFlowFilesTransferred(MonitorActivity.REL_SUCCESS);
    final StateMap updatedState = runner.getStateManager().getState(Scope.CLUSTER);
    assertNull("Latest timestamp should NOT be persisted, because it's running as 'node' scope", updatedState.get(MonitorActivity.STATE_KEY_LATEST_SUCCESS_TRANSFER));
}