Examples with StateManager org.apache.nifi.components.state.StateManager used on opensource projects

Example 11 with StateManager

use of org.apache.nifi.components.state.StateManager in project nifi by apache.

the class StandardStateManagerProvider method getStateManager.

/**
 * Returns the State Manager that has been created for the given component ID, or <code>null</code> if none exists
 *
 * @return the StateManager that can be used by the component with the given ID, or <code>null</code> if none exists
 */
@Override
public synchronized StateManager getStateManager(final String componentId) {
    StateManager stateManager = stateManagers.get(componentId);
    if (stateManager != null) {
        return stateManager;
    }
    stateManager = new StandardStateManager(localStateProvider, clusterStateProvider, componentId);
    stateManagers.put(componentId, stateManager);
    return stateManager;
}

Example 12 with StateManager

use of org.apache.nifi.components.state.StateManager in project nifi by apache.

the class StandardControllerServiceProviderTest method setup.

@Before
public void setup() throws Exception {
    String id = "id";
    String clazz = "org.apache.nifi.controller.service.util.TestControllerService";
    ControllerServiceProvider provider = new StandardControllerServiceProvider(null, null, null, new StateManagerProvider() {

        @Override
        public StateManager getStateManager(final String componentId) {
            return Mockito.mock(StateManager.class);
        }

        @Override
        public void shutdown() {
        }

        @Override
        public void enableClusterProvider() {
        }

        @Override
        public void disableClusterProvider() {
        }

        @Override
        public void onComponentRemoved(String componentId) {
        }
    }, variableRegistry, nifiProperties);
    ControllerServiceNode node = provider.createControllerService(clazz, id, systemBundle.getBundleDetails().getCoordinate(), null, true);
    proxied = node.getProxiedControllerService();
    implementation = node.getControllerServiceImplementation();
}

Example 13 with StateManager

use of org.apache.nifi.components.state.StateManager in project nifi by apache.

the class CaptureChangeMySQL method setup.

public void setup(ProcessContext context) {
    final ComponentLog logger = getLogger();
    final StateManager stateManager = context.getStateManager();
    final StateMap stateMap;
    try {
        stateMap = stateManager.getState(Scope.CLUSTER);
    } catch (final IOException ioe) {
        logger.error("Failed to retrieve observed maximum values from the State Manager. Will not attempt " + "connection until this is accomplished.", ioe);
        context.yield();
        return;
    }
    PropertyValue dbNameValue = context.getProperty(DATABASE_NAME_PATTERN);
    databaseNamePattern = dbNameValue.isSet() ? Pattern.compile(dbNameValue.getValue()) : null;
    PropertyValue tableNameValue = context.getProperty(TABLE_NAME_PATTERN);
    tableNamePattern = tableNameValue.isSet() ? Pattern.compile(tableNameValue.getValue()) : null;
    stateUpdateInterval = context.getProperty(STATE_UPDATE_INTERVAL).evaluateAttributeExpressions().asTimePeriod(TimeUnit.MILLISECONDS);
    boolean getAllRecords = context.getProperty(RETRIEVE_ALL_RECORDS).asBoolean();
    includeBeginCommit = context.getProperty(INCLUDE_BEGIN_COMMIT).asBoolean();
    includeDDLEvents = context.getProperty(INCLUDE_DDL_EVENTS).asBoolean();
    // Set current binlog filename to whatever is in State, falling back to the Retrieve All Records then Initial Binlog Filename if no State variable is present
    currentBinlogFile = stateMap.get(BinlogEventInfo.BINLOG_FILENAME_KEY);
    if (currentBinlogFile == null) {
        if (!getAllRecords) {
            if (context.getProperty(INIT_BINLOG_FILENAME).isSet()) {
                currentBinlogFile = context.getProperty(INIT_BINLOG_FILENAME).evaluateAttributeExpressions().getValue();
            }
        } else {
            // If we're starting from the beginning of all binlogs, the binlog filename must be the empty string (not null)
            currentBinlogFile = "";
        }
    }
    // Set current binlog position to whatever is in State, falling back to the Retrieve All Records then Initial Binlog Filename if no State variable is present
    String binlogPosition = stateMap.get(BinlogEventInfo.BINLOG_POSITION_KEY);
    if (binlogPosition != null) {
        currentBinlogPosition = Long.valueOf(binlogPosition);
    } else if (!getAllRecords) {
        if (context.getProperty(INIT_BINLOG_POSITION).isSet()) {
            currentBinlogPosition = context.getProperty(INIT_BINLOG_POSITION).evaluateAttributeExpressions().asLong();
        } else {
            currentBinlogPosition = DO_NOT_SET;
        }
    } else {
        currentBinlogPosition = -1;
    }
    // Get current sequence ID from state
    String seqIdString = stateMap.get(EventWriter.SEQUENCE_ID_KEY);
    if (StringUtils.isEmpty(seqIdString)) {
        // Use Initial Sequence ID property if none is found in state
        PropertyValue seqIdProp = context.getProperty(INIT_SEQUENCE_ID);
        if (seqIdProp.isSet()) {
            currentSequenceId.set(seqIdProp.evaluateAttributeExpressions().asInteger());
        }
    } else {
        currentSequenceId.set(Integer.parseInt(seqIdString));
    }
    // Get reference to Distributed Cache if one exists. If it does not, no enrichment (resolution of column names, e.g.) will be performed
    boolean createEnrichmentConnection = false;
    if (context.getProperty(DIST_CACHE_CLIENT).isSet()) {
        cacheClient = context.getProperty(DIST_CACHE_CLIENT).asControllerService(DistributedMapCacheClient.class);
        createEnrichmentConnection = true;
    } else {
        logger.warn("No Distributed Map Cache Client is specified, so no event enrichment (resolution of column names, e.g.) will be performed.");
        cacheClient = null;
    }
    // Save off MySQL cluster and JDBC driver information, will be used to connect for event enrichment as well as for the binlog connector
    try {
        List<InetSocketAddress> hosts = getHosts(context.getProperty(HOSTS).evaluateAttributeExpressions().getValue());
        String username = context.getProperty(USERNAME).evaluateAttributeExpressions().getValue();
        String password = context.getProperty(PASSWORD).evaluateAttributeExpressions().getValue();
        // BinaryLogClient expects a non-null password, so set it to the empty string if it is not provided
        if (password == null) {
            password = "";
        }
        long connectTimeout = context.getProperty(CONNECT_TIMEOUT).evaluateAttributeExpressions().asTimePeriod(TimeUnit.MILLISECONDS);
        String driverLocation = context.getProperty(DRIVER_LOCATION).evaluateAttributeExpressions().getValue();
        String driverName = context.getProperty(DRIVER_NAME).evaluateAttributeExpressions().getValue();
        Long serverId = context.getProperty(SERVER_ID).evaluateAttributeExpressions().asLong();
        connect(hosts, username, password, serverId, createEnrichmentConnection, driverLocation, driverName, connectTimeout);
    } catch (IOException | IllegalStateException e) {
        context.yield();
        binlogClient = null;
        throw new ProcessException(e.getMessage(), e);
    }
}

Example 14 with StateManager

use of org.apache.nifi.components.state.StateManager in project nifi by apache.

the class CaptureChangeMySQL method onTrigger.

@Override
public void onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory) throws ProcessException {
    // Indicate that this processor has executed at least once, so we know whether or not the state values are valid and should be updated
    hasRun.set(true);
    ComponentLog log = getLogger();
    StateManager stateManager = context.getStateManager();
    // Create a client if we don't have one
    if (binlogClient == null) {
        setup(context);
    }
    // If the client has been disconnected, try to reconnect
    if (!binlogClient.isConnected()) {
        Exception e = lifecycleListener.getException();
        // If there's no exception, the listener callback might not have been executed yet, so try again later. Otherwise clean up and start over next time
        if (e != null) {
            // Communications failure, disconnect and try next time
            log.error("Binlog connector communications failure: " + e.getMessage(), e);
            try {
                stop(stateManager);
            } catch (CDCException ioe) {
                throw new ProcessException(ioe);
            }
        }
        // Try again later
        context.yield();
        return;
    }
    if (currentSession == null) {
        currentSession = sessionFactory.createSession();
    }
    try {
        outputEvents(currentSession, stateManager, log);
        long now = System.currentTimeMillis();
        long timeSinceLastUpdate = now - lastStateUpdate;
        if (stateUpdateInterval != 0 && timeSinceLastUpdate >= stateUpdateInterval) {
            updateState(stateManager, currentBinlogFile, currentBinlogPosition, currentSequenceId.get());
            lastStateUpdate = now;
        }
    } catch (IOException ioe) {
        try {
            // Perform some processor-level "rollback", then rollback the session
            currentBinlogFile = xactBinlogFile == null ? "" : xactBinlogFile;
            currentBinlogPosition = xactBinlogPosition;
            currentSequenceId.set(xactSequenceId);
            inTransaction = false;
            stop(stateManager);
            queue.clear();
            currentSession.rollback();
        } catch (Exception e) {
            // Not much we can recover from here
            log.warn("Error occurred during rollback", e);
        }
        throw new ProcessException(ioe);
    }
}

Example 15 with StateManager

use of org.apache.nifi.components.state.StateManager in project nifi by apache.

the class FlowController method reload.

@Override
public void reload(final ProcessorNode existingNode, final String newType, final BundleCoordinate bundleCoordinate, final Set<URL> additionalUrls) throws ProcessorInstantiationException {
    if (existingNode == null) {
        throw new IllegalStateException("Existing ProcessorNode cannot be null");
    }
    final String id = existingNode.getProcessor().getIdentifier();
    // ghost components will have a null logger
    if (existingNode.getLogger() != null) {
        existingNode.getLogger().debug("Reloading component {} to type {} from bundle {}", new Object[] { id, newType, bundleCoordinate });
    }
    // createProcessor will create a new instance class loader for the same id so
    // save the instance class loader to use it for calling OnRemoved on the existing processor
    final ClassLoader existingInstanceClassLoader = ExtensionManager.getInstanceClassLoader(id);
    // create a new node with firstTimeAdded as true so lifecycle methods get fired
    // attempt the creation to make sure it works before firing the OnRemoved methods below
    final ProcessorNode newNode = createProcessor(newType, id, bundleCoordinate, additionalUrls, true, false);
    // call OnRemoved for the existing processor using the previous instance class loader
    try (final NarCloseable x = NarCloseable.withComponentNarLoader(existingInstanceClassLoader)) {
        final StateManager stateManager = getStateManagerProvider().getStateManager(id);
        final StandardProcessContext processContext = new StandardProcessContext(existingNode, controllerServiceProvider, encryptor, stateManager, () -> false);
        ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnRemoved.class, existingNode.getProcessor(), processContext);
    } finally {
        ExtensionManager.closeURLClassLoader(id, existingInstanceClassLoader);
    }
    // set the new processor in the existing node
    final ComponentLog componentLogger = new SimpleProcessLogger(id, newNode.getProcessor());
    final TerminationAwareLogger terminationAwareLogger = new TerminationAwareLogger(componentLogger);
    LogRepositoryFactory.getRepository(id).setLogger(terminationAwareLogger);
    final LoggableComponent<Processor> newProcessor = new LoggableComponent<>(newNode.getProcessor(), newNode.getBundleCoordinate(), terminationAwareLogger);
    existingNode.setProcessor(newProcessor);
    existingNode.setExtensionMissing(newNode.isExtensionMissing());
    // need to refresh the properties in case we are changing from ghost component to real component
    existingNode.refreshProperties();
}