Search in sources :

Example 76 with EMPTY

use of org.apache.commons.lang3.StringUtils.EMPTY in project alf.io by alfio-event.

the class MvcConfiguration method getDefaultTemplateObjectsFiller.

@Bean
public HandlerInterceptorAdapter getDefaultTemplateObjectsFiller() {
    return new HandlerInterceptorAdapter() {

        @Override
        public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler, ModelAndView modelAndView) throws Exception {
            Optional.ofNullable(modelAndView).filter(mv -> !StringUtils.startsWith(mv.getViewName(), "redirect:")).ifPresent(mv -> {
                mv.addObject("request", request);
                final ModelMap modelMap = mv.getModelMap();
                boolean demoModeEnabled = environment.acceptsProfiles(Initializer.PROFILE_DEMO);
                modelMap.put("demoModeEnabled", demoModeEnabled);
                Optional.ofNullable(request.getAttribute("ALFIO_EVENT_NAME")).map(Object::toString).ifPresent(eventName -> {
                    List<?> availableLanguages = i18nManager.getEventLanguages(eventName);
                    modelMap.put("showAvailableLanguagesInPageTop", availableLanguages.size() > 1);
                    modelMap.put("availableLanguages", availableLanguages);
                });
                modelMap.putIfAbsent("event", null);
                modelMap.putIfAbsent("pageTitle", "empty");
                Event event = modelMap.get("event") == null ? null : modelMap.get("event") instanceof Event ? (Event) modelMap.get("event") : ((EventDescriptor) modelMap.get("event")).getEvent();
                ConfigurationPathKey googleAnalyticsKey = Optional.ofNullable(event).map(e -> alfio.model.system.Configuration.from(e.getOrganizationId(), e.getId(), GOOGLE_ANALYTICS_KEY)).orElseGet(() -> alfio.model.system.Configuration.getSystemConfiguration(GOOGLE_ANALYTICS_KEY));
                modelMap.putIfAbsent("analyticsEnabled", StringUtils.isNotBlank(configurationManager.getStringConfigValue(googleAnalyticsKey, "")));
                if (demoModeEnabled) {
                    modelMap.putIfAbsent("paypalTestUsername", configurationManager.getStringConfigValue(alfio.model.system.Configuration.getSystemConfiguration(PAYPAL_DEMO_MODE_USERNAME), "<missing>"));
                    modelMap.putIfAbsent("paypalTestPassword", configurationManager.getStringConfigValue(alfio.model.system.Configuration.getSystemConfiguration(PAYPAL_DEMO_MODE_PASSWORD), "<missing>"));
                }
            });
        }
    };
}
Also used : HttpServletRequest(javax.servlet.http.HttpServletRequest) PathVariable(org.springframework.web.bind.annotation.PathVariable) ZonedDateTime(java.time.ZonedDateTime) Autowired(org.springframework.beans.factory.annotation.Autowired) StringUtils(org.apache.commons.lang3.StringUtils) org.springframework.web.servlet.config.annotation(org.springframework.web.servlet.config.annotation) DeserializationFeature(com.fasterxml.jackson.databind.DeserializationFeature) ModelMap(org.springframework.ui.ModelMap) HandlerInterceptorAdapter(org.springframework.web.servlet.handler.HandlerInterceptorAdapter) RequestContextUtils(org.springframework.web.servlet.support.RequestContextUtils) LocalizationMessageInterceptor(org.springframework.web.servlet.view.mustache.jmustache.LocalizationMessageInterceptor) HandlerMethod(org.springframework.web.method.HandlerMethod) Matcher(java.util.regex.Matcher) DefaultMessageSourceResolvable(org.springframework.context.support.DefaultMessageSourceResolvable) JavaTimeModule(com.fasterxml.jackson.datatype.jsr310.JavaTimeModule) EventDescriptor(alfio.controller.decorator.EventDescriptor) JMustacheTemplateFactory(org.springframework.web.servlet.view.mustache.jmustache.JMustacheTemplateFactory) ConfigurationPathKey(alfio.model.system.Configuration.ConfigurationPathKey) MediaType(org.springframework.http.MediaType) ContentLanguage(alfio.model.ContentLanguage) Collectors(java.util.stream.Collectors) JMustacheTemplateLoader(org.springframework.web.servlet.view.mustache.jmustache.JMustacheTemplateLoader) StringHttpMessageConverter(org.springframework.http.converter.StringHttpMessageConverter) Configuration(org.springframework.context.annotation.Configuration) LocaleChangeInterceptor(org.springframework.web.servlet.i18n.LocaleChangeInterceptor) Environment(org.springframework.core.env.Environment) HttpMessageConverter(org.springframework.http.converter.HttpMessageConverter) Pattern(java.util.regex.Pattern) java.util(java.util) RequestMapping(org.springframework.web.bind.annotation.RequestMapping) ConfigurationManager(alfio.manager.system.ConfigurationManager) CommonsMultipartResolver(org.springframework.web.multipart.commons.CommonsMultipartResolver) MustacheViewResolver(org.springframework.web.servlet.view.mustache.MustacheViewResolver) HttpServletRequest(javax.servlet.http.HttpServletRequest) org.springframework.web.servlet(org.springframework.web.servlet) MessageSource(org.springframework.context.MessageSource) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) HttpServletResponse(javax.servlet.http.HttpServletResponse) SessionLocaleResolver(org.springframework.web.servlet.i18n.SessionLocaleResolver) Mustache(com.samskivert.mustache.Mustache) ComponentScan(org.springframework.context.annotation.ComponentScan) MappingJackson2HttpMessageConverter(org.springframework.http.converter.json.MappingJackson2HttpMessageConverter) DateTimeFormatter(java.time.format.DateTimeFormatter) SerializationFeature(com.fasterxml.jackson.databind.SerializationFeature) I18nManager(alfio.manager.i18n.I18nManager) Event(alfio.model.Event) Bean(org.springframework.context.annotation.Bean) CsrfToken(org.springframework.security.web.csrf.CsrfToken) ConfigurationKeys(alfio.model.system.ConfigurationKeys) MustacheCustomTagInterceptor(alfio.util.MustacheCustomTagInterceptor) ConfigurationPathKey(alfio.model.system.Configuration.ConfigurationPathKey) EventDescriptor(alfio.controller.decorator.EventDescriptor) ModelMap(org.springframework.ui.ModelMap) HandlerInterceptorAdapter(org.springframework.web.servlet.handler.HandlerInterceptorAdapter) HttpServletResponse(javax.servlet.http.HttpServletResponse) Event(alfio.model.Event) Bean(org.springframework.context.annotation.Bean)

Example 77 with EMPTY

use of org.apache.commons.lang3.StringUtils.EMPTY in project alf.io by alfio-event.

the class WaitingQueueManager method distributeAvailableSeats.

private Stream<Triple<WaitingQueueSubscription, TicketReservationWithOptionalCodeModification, ZonedDateTime>> distributeAvailableSeats(Event event, Ticket.TicketStatus status, Supplier<Integer> availableSeatSupplier) {
    int availableSeats = availableSeatSupplier.get();
    int eventId = event.getId();
    log.debug("processing {} subscribers from waiting queue", availableSeats);
    List<TicketCategory> unboundedCategories = ticketCategoryRepository.findUnboundedOrderByExpirationDesc(eventId);
    Iterator<Ticket> tickets = ticketRepository.selectWaitingTicketsForUpdate(eventId, status.name(), availableSeats).stream().filter(t -> t.getCategoryId() != null || unboundedCategories.size() > 0).iterator();
    int expirationTimeout = configurationManager.getIntConfigValue(Configuration.from(event.getOrganizationId(), event.getId(), WAITING_QUEUE_RESERVATION_TIMEOUT), 4);
    ZonedDateTime expiration = ZonedDateTime.now(event.getZoneId()).plusHours(expirationTimeout).with(WorkingDaysAdjusters.defaultWorkingDays());
    if (!tickets.hasNext()) {
        log.warn("Unable to assign tickets, returning an empty stream");
        return Stream.empty();
    }
    return waitingQueueRepository.loadWaiting(eventId, availableSeats).stream().map(wq -> Pair.of(wq, tickets.next())).map(pair -> {
        TicketReservationModification ticketReservation = new TicketReservationModification();
        ticketReservation.setAmount(1);
        Integer categoryId = Optional.ofNullable(pair.getValue().getCategoryId()).orElseGet(() -> findBestCategory(unboundedCategories, pair.getKey()).orElseThrow(RuntimeException::new).getId());
        ticketReservation.setTicketCategoryId(categoryId);
        return Pair.of(pair.getLeft(), new TicketReservationWithOptionalCodeModification(ticketReservation, Optional.<SpecialPrice>empty()));
    }).map(pair -> Triple.of(pair.getKey(), pair.getValue(), expiration));
}
Also used : java.util(java.util) TicketReservationModification(alfio.model.modification.TicketReservationModification) TicketCategoryRepository(alfio.repository.TicketCategoryRepository) AffectedRowCountAndKey(ch.digitalfondue.npjt.AffectedRowCountAndKey) WaitingQueueRepository(alfio.repository.WaitingQueueRepository) ZonedDateTime(java.time.ZonedDateTime) NamedParameterJdbcTemplate(org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate) MapSqlParameterSource(org.springframework.jdbc.core.namedparam.MapSqlParameterSource) ConfigurationManager(alfio.manager.system.ConfigurationManager) Supplier(java.util.function.Supplier) Pair(org.apache.commons.lang3.tuple.Pair) WorkingDaysAdjusters(alfio.util.WorkingDaysAdjusters) Triple(org.apache.commons.lang3.tuple.Triple) TemplateResource(alfio.util.TemplateResource) MessageSource(org.springframework.context.MessageSource) EventUtil.determineAvailableSeats(alfio.util.EventUtil.determineAvailableSeats) OrganizationRepository(alfio.repository.user.OrganizationRepository) TicketRepository(alfio.repository.TicketRepository) TicketReservationWithOptionalCodeModification(alfio.model.modification.TicketReservationWithOptionalCodeModification) Organization(alfio.model.user.Organization) TemplateManager(alfio.util.TemplateManager) EventRepository(alfio.repository.EventRepository) DuplicateKeyException(org.springframework.dao.DuplicateKeyException) Component(org.springframework.stereotype.Component) Validate(org.apache.commons.lang3.Validate) Stream(java.util.stream.Stream) alfio.model(alfio.model) Configuration(alfio.model.system.Configuration) Log4j2(lombok.extern.log4j.Log4j2) AllArgsConstructor(lombok.AllArgsConstructor) ConfigurationKeys(alfio.model.system.ConfigurationKeys) PreReservedTicketDistributor(alfio.util.PreReservedTicketDistributor) TicketReservationWithOptionalCodeModification(alfio.model.modification.TicketReservationWithOptionalCodeModification) ZonedDateTime(java.time.ZonedDateTime) TicketReservationModification(alfio.model.modification.TicketReservationModification)

Example 78 with EMPTY

use of org.apache.commons.lang3.StringUtils.EMPTY in project kylo by Teradata.

the class StripHeader method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    final StripHeaderSupport headerSupport = new StripHeaderSupport();
    final FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final boolean isEnabled = context.getProperty(ENABLED).evaluateAttributeExpressions(flowFile).asBoolean();
    final int headerCount = context.getProperty(HEADER_LINE_COUNT).evaluateAttributeExpressions(flowFile).asInteger();
    // Empty files and no work to do will simply pass along content
    if (!isEnabled || headerCount == 0 || flowFile.getSize() == 0L) {
        final FlowFile contentFlowFile = session.clone(flowFile);
        session.transfer(contentFlowFile, REL_CONTENT);
        session.transfer(flowFile, REL_ORIGINAL);
        return;
    }
    final MutableLong headerBoundaryInBytes = new MutableLong(-1);
    session.read(flowFile, false, rawIn -> {
        try {
            // Identify the byte boundary of the header
            long bytes = headerSupport.findHeaderBoundary(headerCount, rawIn);
            headerBoundaryInBytes.setValue(bytes);
            if (bytes < 0) {
                getLog().error("Unable to strip header {} expecting at least {} lines in file", new Object[] { flowFile, headerCount });
            }
        } catch (IOException e) {
            getLog().error("Unable to strip header {} due to {}; routing to failure", new Object[] { flowFile, e.getLocalizedMessage() }, e);
        }
    });
    long headerBytes = headerBoundaryInBytes.getValue();
    if (headerBytes < 0) {
        session.transfer(flowFile, REL_FAILURE);
    } else {
        // Transfer header
        final FlowFile headerFlowFile = session.clone(flowFile, 0, headerBytes);
        session.transfer(headerFlowFile, REL_HEADER);
        // Transfer content
        long contentBytes = flowFile.getSize() - headerBytes;
        final FlowFile contentFlowFile = session.clone(flowFile, headerBytes, contentBytes);
        session.transfer(contentFlowFile, REL_CONTENT);
        session.transfer(flowFile, REL_ORIGINAL);
    }
}
Also used : FlowFile(org.apache.nifi.flowfile.FlowFile) MutableLong(org.apache.commons.lang3.mutable.MutableLong) StripHeaderSupport(com.thinkbiganalytics.ingest.StripHeaderSupport) IOException(java.io.IOException)

Example 79 with EMPTY

use of org.apache.commons.lang3.StringUtils.EMPTY in project kylo by Teradata.

the class ImportReusableTemplate method validateRemoteInputPorts.

/**
 * Validates the user has supplied some input ports to be created as remote ports
 * @param remoteProcessGroupOption the user supplied option and details for remote process group input port processing
 * @return true if valid, false if not
 */
public boolean validateRemoteInputPorts(ImportComponentOption remoteProcessGroupOption) {
    // find list of input ports that have been created already (connected to this same reusable template)
    // 1) find input ports on parent nifi canvas that connect to the reusable template with this same name
    // 2) add these as 'selected' to the list
    // 3) if some of those dont appear in the new list add as warning (these will be removed)
    boolean valid = true;
    if (!isClustered()) {
        return true;
    }
    // This templates input ports as a map by name
    this.templateInputPorts = importTemplate.getTemplateResults().getProcessGroupEntity().getContents().getInputPorts().stream().collect(Collectors.toMap(p -> p.getName(), v -> v));
    // set the map of input ports in this template as potential Remote Input port candidates.
    this.remoteProcessGroupInputPortMap = this.templateInputPorts.values().stream().map(p -> new RemoteProcessGroupInputPort(importTemplate.getTemplateName(), p.getName())).collect(Collectors.toMap(p -> p.getInputPortName(), p -> p));
    // If the incoming list is empty send it back to the user to validate what input ports they would like (if any) to be created as remote input ports
    if (!remoteProcessGroupOption.isUserAcknowledged()) {
        // present back to the user the list of input ports to select
        importTemplate.setRemoteProcessGroupInputPortsNeeded(true);
        valid = false;
        // WARN if the remoteProcessGroupInputPortMap has names that are not in the 'thisTemplatePorts'
        List<String> invalidPorts = remoteProcessGroupInputPortMap.keySet().stream().filter(name -> !this.templateInputPorts.keySet().contains(name)).collect(Collectors.toList());
        if (!invalidPorts.isEmpty()) {
            // the following ports (invalidPorts) will be deleted from the as they no longer exist for this template.
            // Any remote Process group ports created for them will also be deleted
            importTemplate.getTemplateResults().addError(NifiError.SEVERITY.WARN, " Missing 'remote process group ' input ports  ", "");
            remoteProcessGroupOption.getErrorMessages().add(" The following 'remote process group ' input ports are no longer part of this template. " + invalidPorts.stream().collect(Collectors.joining(",")) + ". Are you sure you want to continue?  They will be deleted. ");
        }
        valid &= markExistingRemoteInputPorts(remoteProcessGroupOption, remoteProcessGroupInputPortMap, this.templateInputPorts, true);
    } else {
        // user has already supplied some ports... validate the ports exist for this template
        // warn if the user supplied input port selections that dont exist for this template
        Set<String> nonExistentPortNames = remoteProcessGroupOption.getRemoteProcessGroupInputPortsForTemplate(importTemplate.getTemplateName()).stream().filter(r -> !this.templateInputPorts.keySet().contains(r.getInputPortName())).map(r -> r.getInputPortName()).collect(Collectors.toSet());
        if (!nonExistentPortNames.isEmpty()) {
            importTemplate.getTemplateResults().addError(NifiError.SEVERITY.FATAL, " Invalid input port names supplied", "");
            remoteProcessGroupOption.getErrorMessages().add("The following input ports you supplied as remote ports dont existing in this template: " + nonExistentPortNames.stream().collect(Collectors.joining(",")) + ".");
            importTemplate.setRemoteProcessGroupInputPortsNeeded(true);
            valid = false;
        }
        valid &= markExistingRemoteInputPorts(remoteProcessGroupOption, remoteProcessGroupInputPortMap, this.templateInputPorts, false);
    }
    importTemplate.setRemoteProcessGroupInputPortNames(new ArrayList<>(remoteProcessGroupInputPortMap.values()));
    importTemplate.setSuccess(valid);
    importTemplate.setValid(valid);
    return valid;
}
Also used : UploadProgressService(com.thinkbiganalytics.feedmgr.service.UploadProgressService) VersionedProcessGroup(com.thinkbiganalytics.nifi.rest.model.VersionedProcessGroup) RegisteredTemplateService(com.thinkbiganalytics.feedmgr.service.template.RegisteredTemplateService) NifiError(com.thinkbiganalytics.nifi.rest.model.NifiError) LoggerFactory(org.slf4j.LoggerFactory) ReusableTemplateConnectionInfo(com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo) ConnectionDTO(org.apache.nifi.web.api.dto.ConnectionDTO) NifiProcessUtil(com.thinkbiganalytics.nifi.rest.support.NifiProcessUtil) StringUtils(org.apache.commons.lang3.StringUtils) NifiClientRuntimeException(com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException) ProcessGroupDTO(org.apache.nifi.web.api.dto.ProcessGroupDTO) NifiConnectionUtil(com.thinkbiganalytics.nifi.rest.support.NifiConnectionUtil) TemplateDTO(org.apache.nifi.web.api.dto.TemplateDTO) ConnectionStatusEntity(org.apache.nifi.web.api.entity.ConnectionStatusEntity) Map(java.util.Map) AccessController(com.thinkbiganalytics.security.AccessController) NifiFlowUtil(com.thinkbiganalytics.nifi.rest.support.NifiFlowUtil) RemoteProcessGroupInputPort(com.thinkbiganalytics.feedmgr.rest.model.RemoteProcessGroupInputPort) ImportTemplate(com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate) ReusableTemplateCreationCallback(com.thinkbiganalytics.nifi.feedmgr.ReusableTemplateCreationCallback) NifiProperty(com.thinkbiganalytics.nifi.rest.model.NifiProperty) Collection(java.util.Collection) Set(java.util.Set) Collectors(java.util.stream.Collectors) UploadProgressMessage(com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage) RegisteredTemplateCache(com.thinkbiganalytics.feedmgr.service.template.RegisteredTemplateCache) PortDTO(org.apache.nifi.web.api.dto.PortDTO) List(java.util.List) ProcessorDTO(org.apache.nifi.web.api.dto.ProcessorDTO) ImportTemplateOptions(com.thinkbiganalytics.feedmgr.rest.model.ImportTemplateOptions) Optional(java.util.Optional) NiFiTemplateImport(com.thinkbiganalytics.feedmgr.service.template.importing.model.NiFiTemplateImport) ImportComponent(com.thinkbiganalytics.feedmgr.rest.ImportComponent) NifiProcessGroup(com.thinkbiganalytics.nifi.rest.model.NifiProcessGroup) ProcessGroupStatusDTO(org.apache.nifi.web.api.dto.status.ProcessGroupStatusDTO) HashMap(java.util.HashMap) ImportSection(com.thinkbiganalytics.feedmgr.rest.ImportSection) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Inject(javax.inject.Inject) PropertyExpressionResolver(com.thinkbiganalytics.feedmgr.nifi.PropertyExpressionResolver) NifiComponentNotFoundException(com.thinkbiganalytics.nifi.rest.client.NifiComponentNotFoundException) ProcessGroupFlowDTO(org.apache.nifi.web.api.dto.flow.ProcessGroupFlowDTO) FlowDTO(org.apache.nifi.web.api.dto.flow.FlowDTO) NifiConstants(com.thinkbiganalytics.nifi.rest.support.NifiConstants) Nullable(javax.annotation.Nullable) NiFiClusterSummary(com.thinkbiganalytics.nifi.rest.model.NiFiClusterSummary) Logger(org.slf4j.Logger) NifiFlowCache(com.thinkbiganalytics.feedmgr.nifi.cache.NifiFlowCache) TemplateConnectionUtil(com.thinkbiganalytics.feedmgr.nifi.TemplateConnectionUtil) TemplateRemoteInputPortConnections(com.thinkbiganalytics.feedmgr.rest.model.TemplateRemoteInputPortConnections) Collections(java.util.Collections) LegacyNifiRestClient(com.thinkbiganalytics.nifi.rest.client.LegacyNifiRestClient) ConnectableDTO(org.apache.nifi.web.api.dto.ConnectableDTO) ImportComponentOption(com.thinkbiganalytics.feedmgr.rest.model.ImportComponentOption) RemoteProcessGroupInputPort(com.thinkbiganalytics.feedmgr.rest.model.RemoteProcessGroupInputPort)

Example 80 with EMPTY

use of org.apache.commons.lang3.StringUtils.EMPTY in project nifi by apache.

the class StandardFlowSynchronizer method sync.

@Override
public void sync(final FlowController controller, final DataFlow proposedFlow, final StringEncryptor encryptor) throws FlowSerializationException, UninheritableFlowException, FlowSynchronizationException, MissingBundleException {
    // handle corner cases involving no proposed flow
    if (proposedFlow == null) {
        if (controller.getGroup(controller.getRootGroupId()).isEmpty()) {
            // no sync to perform
            return;
        } else {
            throw new UninheritableFlowException("Proposed configuration is empty, but the controller contains a data flow.");
        }
    }
    // determine if the controller already had flow sync'd to it
    final boolean flowAlreadySynchronized = controller.isFlowSynchronized();
    logger.debug("Synching FlowController with proposed flow: Controller is Already Synchronized = {}", flowAlreadySynchronized);
    // serialize controller state to bytes
    final byte[] existingFlow;
    final boolean existingFlowEmpty;
    try {
        if (flowAlreadySynchronized) {
            existingFlow = toBytes(controller);
            existingFlowEmpty = controller.getGroup(controller.getRootGroupId()).isEmpty() && controller.getAllReportingTasks().isEmpty() && controller.getAllControllerServices().isEmpty() && controller.getFlowRegistryClient().getRegistryIdentifiers().isEmpty();
        } else {
            existingFlow = readFlowFromDisk();
            if (existingFlow == null || existingFlow.length == 0) {
                existingFlowEmpty = true;
            } else {
                final Document document = parseFlowBytes(existingFlow);
                final Element rootElement = document.getDocumentElement();
                final FlowEncodingVersion encodingVersion = FlowEncodingVersion.parse(rootElement);
                logger.trace("Setting controller thread counts");
                final Integer maxThreadCount = getInteger(rootElement, "maxThreadCount");
                if (maxThreadCount == null) {
                    controller.setMaxTimerDrivenThreadCount(getInt(rootElement, "maxTimerDrivenThreadCount"));
                    controller.setMaxEventDrivenThreadCount(getInt(rootElement, "maxEventDrivenThreadCount"));
                } else {
                    controller.setMaxTimerDrivenThreadCount(maxThreadCount * 2 / 3);
                    controller.setMaxEventDrivenThreadCount(maxThreadCount / 3);
                }
                final Element reportingTasksElement = DomUtils.getChild(rootElement, "reportingTasks");
                final List<Element> taskElements;
                if (reportingTasksElement == null) {
                    taskElements = Collections.emptyList();
                } else {
                    taskElements = DomUtils.getChildElementsByTagName(reportingTasksElement, "reportingTask");
                }
                final Element controllerServicesElement = DomUtils.getChild(rootElement, "controllerServices");
                final List<Element> unrootedControllerServiceElements;
                if (controllerServicesElement == null) {
                    unrootedControllerServiceElements = Collections.emptyList();
                } else {
                    unrootedControllerServiceElements = DomUtils.getChildElementsByTagName(controllerServicesElement, "controllerService");
                }
                final boolean registriesPresent;
                final Element registriesElement = DomUtils.getChild(rootElement, "registries");
                if (registriesElement == null) {
                    registriesPresent = false;
                } else {
                    final List<Element> flowRegistryElems = DomUtils.getChildElementsByTagName(registriesElement, "flowRegistry");
                    registriesPresent = !flowRegistryElems.isEmpty();
                }
                logger.trace("Parsing process group from DOM");
                final Element rootGroupElement = (Element) rootElement.getElementsByTagName("rootGroup").item(0);
                final ProcessGroupDTO rootGroupDto = FlowFromDOMFactory.getProcessGroup(null, rootGroupElement, encryptor, encodingVersion);
                existingFlowEmpty = taskElements.isEmpty() && unrootedControllerServiceElements.isEmpty() && isEmpty(rootGroupDto) && !registriesPresent;
                logger.debug("Existing Flow Empty = {}", existingFlowEmpty);
            }
        }
    } catch (final IOException e) {
        throw new FlowSerializationException(e);
    }
    logger.trace("Exporting snippets from controller");
    final byte[] existingSnippets = controller.getSnippetManager().export();
    logger.trace("Getting Authorizer fingerprint from controller");
    final byte[] existingAuthFingerprint;
    final ManagedAuthorizer managedAuthorizer;
    final Authorizer authorizer = controller.getAuthorizer();
    if (AuthorizerCapabilityDetection.isManagedAuthorizer(authorizer)) {
        managedAuthorizer = (ManagedAuthorizer) authorizer;
        existingAuthFingerprint = managedAuthorizer.getFingerprint().getBytes(StandardCharsets.UTF_8);
    } else {
        existingAuthFingerprint = null;
        managedAuthorizer = null;
    }
    final Set<String> missingComponents = new HashSet<>();
    controller.getAllControllerServices().stream().filter(cs -> cs.isExtensionMissing()).forEach(cs -> missingComponents.add(cs.getIdentifier()));
    controller.getAllReportingTasks().stream().filter(r -> r.isExtensionMissing()).forEach(r -> missingComponents.add(r.getIdentifier()));
    controller.getRootGroup().findAllProcessors().stream().filter(p -> p.isExtensionMissing()).forEach(p -> missingComponents.add(p.getIdentifier()));
    final DataFlow existingDataFlow = new StandardDataFlow(existingFlow, existingSnippets, existingAuthFingerprint, missingComponents);
    Document configuration = null;
    // check that the proposed flow is inheritable by the controller
    try {
        if (existingFlowEmpty) {
            configuration = parseFlowBytes(proposedFlow.getFlow());
            if (configuration != null) {
                logger.trace("Checking bundle compatibility");
                checkBundleCompatibility(configuration);
            }
        } else {
            logger.trace("Checking flow inheritability");
            final String problemInheritingFlow = checkFlowInheritability(existingDataFlow, proposedFlow, controller);
            if (problemInheritingFlow != null) {
                throw new UninheritableFlowException("Proposed configuration is not inheritable by the flow controller because of flow differences: " + problemInheritingFlow);
            }
        }
    } catch (final FingerprintException fe) {
        throw new FlowSerializationException("Failed to generate flow fingerprints", fe);
    }
    logger.trace("Checking missing component inheritability");
    final String problemInheritingMissingComponents = checkMissingComponentsInheritability(existingDataFlow, proposedFlow);
    if (problemInheritingMissingComponents != null) {
        throw new UninheritableFlowException("Proposed Flow is not inheritable by the flow controller because of differences in missing components: " + problemInheritingMissingComponents);
    }
    logger.trace("Checking authorizer inheritability");
    final AuthorizerInheritability authInheritability = checkAuthorizerInheritability(authorizer, existingDataFlow, proposedFlow);
    if (!authInheritability.isInheritable() && authInheritability.getReason() != null) {
        throw new UninheritableFlowException("Proposed Authorizer is not inheritable by the flow controller because of Authorizer differences: " + authInheritability.getReason());
    }
    // create document by parsing proposed flow bytes
    logger.trace("Parsing proposed flow bytes as DOM document");
    if (configuration == null) {
        configuration = parseFlowBytes(proposedFlow.getFlow());
    }
    // attempt to sync controller with proposed flow
    try {
        if (configuration != null) {
            synchronized (configuration) {
                // get the root element
                final Element rootElement = (Element) configuration.getElementsByTagName("flowController").item(0);
                final FlowEncodingVersion encodingVersion = FlowEncodingVersion.parse(rootElement);
                // set controller config
                logger.trace("Updating flow config");
                final Integer maxThreadCount = getInteger(rootElement, "maxThreadCount");
                if (maxThreadCount == null) {
                    controller.setMaxTimerDrivenThreadCount(getInt(rootElement, "maxTimerDrivenThreadCount"));
                    controller.setMaxEventDrivenThreadCount(getInt(rootElement, "maxEventDrivenThreadCount"));
                } else {
                    controller.setMaxTimerDrivenThreadCount(maxThreadCount * 2 / 3);
                    controller.setMaxEventDrivenThreadCount(maxThreadCount / 3);
                }
                // get the root group XML element
                final Element rootGroupElement = (Element) rootElement.getElementsByTagName("rootGroup").item(0);
                if (!flowAlreadySynchronized || existingFlowEmpty) {
                    final Element registriesElement = DomUtils.getChild(rootElement, "registries");
                    if (registriesElement != null) {
                        final List<Element> flowRegistryElems = DomUtils.getChildElementsByTagName(registriesElement, "flowRegistry");
                        for (final Element flowRegistryElement : flowRegistryElems) {
                            final String registryId = getString(flowRegistryElement, "id");
                            final String registryName = getString(flowRegistryElement, "name");
                            final String registryUrl = getString(flowRegistryElement, "url");
                            final String description = getString(flowRegistryElement, "description");
                            final FlowRegistryClient client = controller.getFlowRegistryClient();
                            client.addFlowRegistry(registryId, registryName, registryUrl, description);
                        }
                    }
                }
                // if this controller isn't initialized or its empty, add the root group, otherwise update
                final ProcessGroup rootGroup;
                if (!flowAlreadySynchronized || existingFlowEmpty) {
                    logger.trace("Adding root process group");
                    rootGroup = addProcessGroup(controller, /* parent group */
                    null, rootGroupElement, encryptor, encodingVersion);
                } else {
                    logger.trace("Updating root process group");
                    rootGroup = updateProcessGroup(controller, /* parent group */
                    null, rootGroupElement, encryptor, encodingVersion);
                }
                rootGroup.findAllRemoteProcessGroups().forEach(RemoteProcessGroup::initialize);
                // If there are any Templates that do not exist in the Proposed Flow that do exist in the 'existing flow', we need
                // to ensure that we also add those to the appropriate Process Groups, so that we don't lose them.
                final Document existingFlowConfiguration = parseFlowBytes(existingFlow);
                if (existingFlowConfiguration != null) {
                    final Element existingRootElement = (Element) existingFlowConfiguration.getElementsByTagName("flowController").item(0);
                    if (existingRootElement != null) {
                        final Element existingRootGroupElement = (Element) existingRootElement.getElementsByTagName("rootGroup").item(0);
                        if (existingRootElement != null) {
                            final FlowEncodingVersion existingEncodingVersion = FlowEncodingVersion.parse(existingFlowConfiguration.getDocumentElement());
                            addLocalTemplates(existingRootGroupElement, rootGroup, existingEncodingVersion);
                        }
                    }
                }
                // get all the reporting task elements
                final Element reportingTasksElement = DomUtils.getChild(rootElement, "reportingTasks");
                final List<Element> reportingTaskElements = new ArrayList<>();
                if (reportingTasksElement != null) {
                    reportingTaskElements.addAll(DomUtils.getChildElementsByTagName(reportingTasksElement, "reportingTask"));
                }
                // get/create all the reporting task nodes and DTOs, but don't apply their scheduled state yet
                final Map<ReportingTaskNode, ReportingTaskDTO> reportingTaskNodesToDTOs = new HashMap<>();
                for (final Element taskElement : reportingTaskElements) {
                    final ReportingTaskDTO dto = FlowFromDOMFactory.getReportingTask(taskElement, encryptor);
                    final ReportingTaskNode reportingTask = getOrCreateReportingTask(controller, dto, flowAlreadySynchronized, existingFlowEmpty);
                    reportingTaskNodesToDTOs.put(reportingTask, dto);
                }
                final Element controllerServicesElement = DomUtils.getChild(rootElement, "controllerServices");
                if (controllerServicesElement != null) {
                    final List<Element> serviceElements = DomUtils.getChildElementsByTagName(controllerServicesElement, "controllerService");
                    if (!flowAlreadySynchronized || existingFlowEmpty) {
                        // If the encoding version is null, we are loading a flow from NiFi 0.x, where Controller
                        // Services could not be scoped by Process Group. As a result, we want to move the Process Groups
                        // to the root Group. Otherwise, we want to use a null group, which indicates a Controller-level
                        // Controller Service.
                        final ProcessGroup group = (encodingVersion == null) ? rootGroup : null;
                        final Map<ControllerServiceNode, Element> controllerServices = ControllerServiceLoader.loadControllerServices(serviceElements, controller, group, encryptor);
                        // reference them, and if so we need to clone the CS and update the reporting task reference
                        if (group != null) {
                            // find all the controller service ids referenced by reporting tasks
                            final Set<String> controllerServicesInReportingTasks = reportingTaskNodesToDTOs.keySet().stream().flatMap(r -> r.getProperties().entrySet().stream()).filter(e -> e.getKey().getControllerServiceDefinition() != null).map(e -> e.getValue()).collect(Collectors.toSet());
                            // find the controller service nodes for each id referenced by a reporting task
                            final Set<ControllerServiceNode> controllerServicesToClone = controllerServices.keySet().stream().filter(cs -> controllerServicesInReportingTasks.contains(cs.getIdentifier())).collect(Collectors.toSet());
                            // clone the controller services and map the original id to the clone
                            final Map<String, ControllerServiceNode> controllerServiceMapping = new HashMap<>();
                            for (ControllerServiceNode controllerService : controllerServicesToClone) {
                                final ControllerServiceNode clone = ControllerServiceLoader.cloneControllerService(controller, controllerService);
                                controller.addRootControllerService(clone);
                                controllerServiceMapping.put(controllerService.getIdentifier(), clone);
                            }
                            // update the reporting tasks to reference the cloned controller services
                            updateReportingTaskControllerServices(reportingTaskNodesToDTOs.keySet(), controllerServiceMapping);
                            // enable all the cloned controller services
                            ControllerServiceLoader.enableControllerServices(controllerServiceMapping.values(), controller, autoResumeState);
                        }
                        // enable all the original controller services
                        ControllerServiceLoader.enableControllerServices(controllerServices, controller, encryptor, autoResumeState);
                    }
                }
                scaleRootGroup(rootGroup, encodingVersion);
                // now that controller services are loaded and enabled we can apply the scheduled state to each reporting task
                for (Map.Entry<ReportingTaskNode, ReportingTaskDTO> entry : reportingTaskNodesToDTOs.entrySet()) {
                    applyReportingTaskScheduleState(controller, entry.getValue(), entry.getKey(), flowAlreadySynchronized, existingFlowEmpty);
                }
            }
        }
        // clear the snippets that are currently in memory
        logger.trace("Clearing existing snippets");
        final SnippetManager snippetManager = controller.getSnippetManager();
        snippetManager.clear();
        // if proposed flow has any snippets, load them
        logger.trace("Loading proposed snippets");
        final byte[] proposedSnippets = proposedFlow.getSnippets();
        if (proposedSnippets != null && proposedSnippets.length > 0) {
            for (final StandardSnippet snippet : SnippetManager.parseBytes(proposedSnippets)) {
                snippetManager.addSnippet(snippet);
            }
        }
        // if auths are inheritable and we have a policy based authorizer, then inherit
        if (authInheritability.isInheritable() && managedAuthorizer != null) {
            logger.trace("Inheriting authorizations");
            final String proposedAuthFingerprint = new String(proposedFlow.getAuthorizerFingerprint(), StandardCharsets.UTF_8);
            managedAuthorizer.inheritFingerprint(proposedAuthFingerprint);
        }
        logger.debug("Finished syncing flows");
    } catch (final Exception ex) {
        throw new FlowSynchronizationException(ex);
    }
}
Also used : Arrays(java.util.Arrays) GZIPInputStream(java.util.zip.GZIPInputStream) Size(org.apache.nifi.connectable.Size) ConnectionDTO(org.apache.nifi.web.api.dto.ConnectionDTO) StringUtils(org.apache.commons.lang3.StringUtils) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) PositionDTO(org.apache.nifi.web.api.dto.PositionDTO) ProcessGroupDTO(org.apache.nifi.web.api.dto.ProcessGroupDTO) Document(org.w3c.dom.Document) Map(java.util.Map) FlowSerializationException(org.apache.nifi.controller.serialization.FlowSerializationException) RootGroupPort(org.apache.nifi.remote.RootGroupPort) Connectable(org.apache.nifi.connectable.Connectable) Connection(org.apache.nifi.connectable.Connection) Path(java.nio.file.Path) FunnelDTO(org.apache.nifi.web.api.dto.FunnelDTO) LoggingXmlParserErrorHandler(org.apache.nifi.util.LoggingXmlParserErrorHandler) FlowFilePrioritizer(org.apache.nifi.flowfile.FlowFilePrioritizer) FileUtils(org.apache.nifi.util.file.FileUtils) Set(java.util.Set) StandardFlowSerializer(org.apache.nifi.controller.serialization.StandardFlowSerializer) StandardCharsets(java.nio.charset.StandardCharsets) StandardDataFlow(org.apache.nifi.cluster.protocol.StandardDataFlow) PortDTO(org.apache.nifi.web.api.dto.PortDTO) AuthorizerCapabilityDetection(org.apache.nifi.authorization.AuthorizerCapabilityDetection) Position(org.apache.nifi.connectable.Position) RemoteProcessGroup(org.apache.nifi.groups.RemoteProcessGroup) DocumentBuilderFactory(javax.xml.parsers.DocumentBuilderFactory) StandardVersionControlInformation(org.apache.nifi.registry.flow.StandardVersionControlInformation) ByteArrayOutputStream(java.io.ByteArrayOutputStream) SiteToSiteTransportProtocol(org.apache.nifi.remote.protocol.SiteToSiteTransportProtocol) ReportingInitializationContext(org.apache.nifi.reporting.ReportingInitializationContext) Schema(javax.xml.validation.Schema) CollectionUtils(org.apache.commons.collections4.CollectionUtils) ArrayList(java.util.ArrayList) Relationship(org.apache.nifi.processor.Relationship) VersionedFlowState(org.apache.nifi.registry.flow.VersionedFlowState) SchemaFactory(javax.xml.validation.SchemaFactory) Files(java.nio.file.Files) FlowEncodingVersion(org.apache.nifi.controller.serialization.FlowEncodingVersion) IOException(java.io.IOException) ExecutionNode(org.apache.nifi.scheduling.ExecutionNode) BulletinFactory(org.apache.nifi.events.BulletinFactory) Authorizer(org.apache.nifi.authorization.Authorizer) NiFiProperties(org.apache.nifi.util.NiFiProperties) FlowFromDOMFactory(org.apache.nifi.controller.serialization.FlowFromDOMFactory) DocumentBuilder(javax.xml.parsers.DocumentBuilder) Severity(org.apache.nifi.reporting.Severity) ProcessorInstantiationException(org.apache.nifi.controller.exception.ProcessorInstantiationException) ControllerServiceLoader(org.apache.nifi.controller.service.ControllerServiceLoader) ProcessGroup(org.apache.nifi.groups.ProcessGroup) BundleCoordinate(org.apache.nifi.bundle.BundleCoordinate) ProcessorConfigDTO(org.apache.nifi.web.api.dto.ProcessorConfigDTO) URL(java.net.URL) InitializationException(org.apache.nifi.reporting.InitializationException) ConnectableType(org.apache.nifi.connectable.ConnectableType) LoggerFactory(org.slf4j.LoggerFactory) Port(org.apache.nifi.connectable.Port) FingerprintException(org.apache.nifi.fingerprint.FingerprintException) BundleDTO(org.apache.nifi.web.api.dto.BundleDTO) ReportingTaskInstantiationException(org.apache.nifi.controller.reporting.ReportingTaskInstantiationException) FlowSynchronizer(org.apache.nifi.controller.serialization.FlowSynchronizer) LabelDTO(org.apache.nifi.web.api.dto.LabelDTO) ByteArrayInputStream(java.io.ByteArrayInputStream) TemplateDTO(org.apache.nifi.web.api.dto.TemplateDTO) SchedulingStrategy(org.apache.nifi.scheduling.SchedulingStrategy) Label(org.apache.nifi.controller.label.Label) FlowRegistryClient(org.apache.nifi.registry.flow.FlowRegistryClient) ControllerServiceDTO(org.apache.nifi.web.api.dto.ControllerServiceDTO) StandardOpenOption(java.nio.file.StandardOpenOption) RemoteProcessGroupPortDescriptor(org.apache.nifi.groups.RemoteProcessGroupPortDescriptor) BundleUtils(org.apache.nifi.util.BundleUtils) Collectors(java.util.stream.Collectors) List(java.util.List) UninheritableAuthorizationsException(org.apache.nifi.authorization.exception.UninheritableAuthorizationsException) FingerprintFactory(org.apache.nifi.fingerprint.FingerprintFactory) SAXException(org.xml.sax.SAXException) ProcessorDTO(org.apache.nifi.web.api.dto.ProcessorDTO) ControllerServiceState(org.apache.nifi.controller.service.ControllerServiceState) ReportingTaskDTO(org.apache.nifi.web.api.dto.ReportingTaskDTO) FlowSnippetDTO(org.apache.nifi.web.api.dto.FlowSnippetDTO) DataFlow(org.apache.nifi.cluster.protocol.DataFlow) RemoteProcessGroupDTO(org.apache.nifi.web.api.dto.RemoteProcessGroupDTO) Funnel(org.apache.nifi.connectable.Funnel) ControllerServiceNode(org.apache.nifi.controller.service.ControllerServiceNode) SimpleProcessLogger(org.apache.nifi.processor.SimpleProcessLogger) HashMap(java.util.HashMap) ComponentLog(org.apache.nifi.logging.ComponentLog) DomUtils(org.apache.nifi.util.DomUtils) HashSet(java.util.HashSet) FlowRegistry(org.apache.nifi.registry.flow.FlowRegistry) StringEncryptor(org.apache.nifi.encrypt.StringEncryptor) VersionControlInformationDTO(org.apache.nifi.web.api.dto.VersionControlInformationDTO) Node(org.w3c.dom.Node) XMLConstants(javax.xml.XMLConstants) LogLevel(org.apache.nifi.logging.LogLevel) FlowSynchronizationException(org.apache.nifi.controller.serialization.FlowSynchronizationException) ManagedAuthorizer(org.apache.nifi.authorization.ManagedAuthorizer) Logger(org.slf4j.Logger) NodeList(org.w3c.dom.NodeList) RemoteGroupPort(org.apache.nifi.remote.RemoteGroupPort) TimeUnit(java.util.concurrent.TimeUnit) Element(org.w3c.dom.Element) ParserConfigurationException(javax.xml.parsers.ParserConfigurationException) Collections(java.util.Collections) StandardReportingInitializationContext(org.apache.nifi.controller.reporting.StandardReportingInitializationContext) ConnectableDTO(org.apache.nifi.web.api.dto.ConnectableDTO) InputStream(java.io.InputStream) HashMap(java.util.HashMap) Element(org.w3c.dom.Element) FlowRegistryClient(org.apache.nifi.registry.flow.FlowRegistryClient) ArrayList(java.util.ArrayList) FlowEncodingVersion(org.apache.nifi.controller.serialization.FlowEncodingVersion) Document(org.w3c.dom.Document) Authorizer(org.apache.nifi.authorization.Authorizer) ManagedAuthorizer(org.apache.nifi.authorization.ManagedAuthorizer) ProcessGroupDTO(org.apache.nifi.web.api.dto.ProcessGroupDTO) RemoteProcessGroupDTO(org.apache.nifi.web.api.dto.RemoteProcessGroupDTO) FingerprintException(org.apache.nifi.fingerprint.FingerprintException) HashSet(java.util.HashSet) RemoteProcessGroup(org.apache.nifi.groups.RemoteProcessGroup) FlowSynchronizationException(org.apache.nifi.controller.serialization.FlowSynchronizationException) FlowSerializationException(org.apache.nifi.controller.serialization.FlowSerializationException) IOException(java.io.IOException) StandardDataFlow(org.apache.nifi.cluster.protocol.StandardDataFlow) DataFlow(org.apache.nifi.cluster.protocol.DataFlow) FlowSerializationException(org.apache.nifi.controller.serialization.FlowSerializationException) IOException(java.io.IOException) ProcessorInstantiationException(org.apache.nifi.controller.exception.ProcessorInstantiationException) InitializationException(org.apache.nifi.reporting.InitializationException) FingerprintException(org.apache.nifi.fingerprint.FingerprintException) ReportingTaskInstantiationException(org.apache.nifi.controller.reporting.ReportingTaskInstantiationException) UninheritableAuthorizationsException(org.apache.nifi.authorization.exception.UninheritableAuthorizationsException) SAXException(org.xml.sax.SAXException) FlowSynchronizationException(org.apache.nifi.controller.serialization.FlowSynchronizationException) ParserConfigurationException(javax.xml.parsers.ParserConfigurationException) StandardDataFlow(org.apache.nifi.cluster.protocol.StandardDataFlow) ControllerServiceNode(org.apache.nifi.controller.service.ControllerServiceNode) ManagedAuthorizer(org.apache.nifi.authorization.ManagedAuthorizer) RemoteProcessGroup(org.apache.nifi.groups.RemoteProcessGroup) ProcessGroup(org.apache.nifi.groups.ProcessGroup) ReportingTaskDTO(org.apache.nifi.web.api.dto.ReportingTaskDTO) Map(java.util.Map) HashMap(java.util.HashMap)

Aggregations

List (java.util.List)44 Map (java.util.Map)42 ArrayList (java.util.ArrayList)41 StringUtils (org.apache.commons.lang3.StringUtils)38 Collectors (java.util.stream.Collectors)37 HashMap (java.util.HashMap)33 IOException (java.io.IOException)27 Set (java.util.Set)25 HashSet (java.util.HashSet)22 LoggerFactory (org.slf4j.LoggerFactory)22 Pair (org.apache.commons.lang3.tuple.Pair)20 Logger (org.slf4j.Logger)20 Optional (java.util.Optional)19 Collections (java.util.Collections)17 ImmutablePair (org.apache.commons.lang3.tuple.ImmutablePair)17 java.util (java.util)15 Arrays.asList (java.util.Arrays.asList)14 Collection (java.util.Collection)14 Stream (java.util.stream.Stream)14 Arrays (java.util.Arrays)12