Examples with LoadFilterType org.jumpmind.symmetric.model.LoadFilter.LoadFilterType used on opensource projects

Example 1 with LoadFilterType

use of org.jumpmind.symmetric.model.LoadFilter.LoadFilterType in project symmetric-ds by JumpMind.

the class LoadFilterService method refreshCache.

protected void refreshCache() {
    // get the cache timeout
    long cacheTimeoutInMs = parameterService.getLong(ParameterConstants.CACHE_TIMEOUT_LOAD_FILTER_IN_MS);
    synchronized (this) {
        if (System.currentTimeMillis() - lastCacheTimeInMs >= cacheTimeoutInMs || loadFilterCacheByNodeGroupLink == null) {
            loadFilterCacheByNodeGroupLink = new HashMap<NodeGroupLink, Map<LoadFilterType, Map<String, List<LoadFilter>>>>();
            List<LoadFilterNodeGroupLink> loadFilters = getLoadFiltersFromDB();
            boolean ignoreCase = this.parameterService.is(ParameterConstants.DB_METADATA_IGNORE_CASE);
            for (LoadFilterNodeGroupLink loadFilter : loadFilters) {
                NodeGroupLink nodeGroupLink = loadFilter.getNodeGroupLink();
                if (nodeGroupLink != null) {
                    Map<LoadFilterType, Map<String, List<LoadFilter>>> loadFiltersByType = loadFilterCacheByNodeGroupLink.get(nodeGroupLink);
                    if (loadFiltersByType == null) {
                        loadFiltersByType = new HashMap<LoadFilterType, Map<String, List<LoadFilter>>>();
                        loadFilterCacheByNodeGroupLink.put(nodeGroupLink, loadFiltersByType);
                    }
                    Map<String, List<LoadFilter>> loadFiltersByTable = loadFiltersByType.get(loadFilter.getLoadFilterType());
                    if (loadFiltersByTable == null) {
                        loadFiltersByTable = new HashMap<String, List<LoadFilter>>();
                        loadFiltersByType.put(loadFilter.getLoadFilterType(), loadFiltersByTable);
                    }
                    String tableName = loadFilter.getTargetTableName();
                    if (StringUtils.isBlank(tableName)) {
                        tableName = FormatUtils.WILDCARD;
                    } else if (ignoreCase) {
                        tableName = tableName.toUpperCase();
                    }
                    String schemaName = loadFilter.getTargetSchemaName();
                    if (StringUtils.isBlank(schemaName)) {
                        schemaName = FormatUtils.WILDCARD;
                    } else if (ignoreCase) {
                        schemaName = schemaName.toUpperCase();
                    }
                    String catalogName = loadFilter.getTargetCatalogName();
                    if (StringUtils.isBlank(catalogName)) {
                        catalogName = FormatUtils.WILDCARD;
                    } else if (ignoreCase) {
                        catalogName = catalogName.toUpperCase();
                    }
                    String qualifiedName = Table.getFullyQualifiedTableName(catalogName, schemaName, tableName);
                    List<LoadFilter> loadFiltersForTable = loadFiltersByTable.get(qualifiedName);
                    if (loadFiltersForTable == null) {
                        loadFiltersForTable = new ArrayList<LoadFilter>();
                        loadFiltersByTable.put(qualifiedName, loadFiltersForTable);
                    }
                    loadFiltersForTable.add(loadFilter);
                }
            }
            lastCacheTimeInMs = System.currentTimeMillis();
        }
    }
}

Example 2 with LoadFilterType

use of org.jumpmind.symmetric.model.LoadFilter.LoadFilterType in project symmetric-ds by JumpMind.

the class DataLoaderService method buildDataWriter.

protected IDataWriter buildDataWriter(ProcessInfo processInfo, String sourceNodeId, String channelId, long batchId, boolean isRetry) {
    TransformTable[] transforms = null;
    NodeGroupLink link = null;
    List<ResolvedData> resolvedDatas = new ArrayList<ResolvedData>();
    List<IDatabaseWriterFilter> filters = extensionService.getExtensionPointList(IDatabaseWriterFilter.class);
    List<IDatabaseWriterFilter> dynamicFilters = filters;
    List<IDatabaseWriterErrorHandler> errorHandlers = extensionService.getExtensionPointList(IDatabaseWriterErrorHandler.class);
    List<IDatabaseWriterErrorHandler> dynamicErrorHandlers = errorHandlers;
    if (sourceNodeId != null) {
        Node sourceNode = nodeService.findNode(sourceNodeId, true);
        if (sourceNode != null) {
            link = new NodeGroupLink(sourceNode.getNodeGroupId(), parameterService.getNodeGroupId());
        }
        Map<LoadFilterType, Map<String, List<LoadFilter>>> loadFilters = loadFilterService.findLoadFiltersFor(link, true);
        List<DynamicDatabaseWriterFilter> databaseWriterFilters = DynamicDatabaseWriterFilter.getDatabaseWriterFilters(engine, loadFilters);
        if (loadFilters != null && loadFilters.size() > 0) {
            dynamicFilters = new ArrayList<IDatabaseWriterFilter>(filters.size() + 1);
            dynamicFilters.addAll(filters);
            dynamicFilters.addAll(databaseWriterFilters);
            dynamicErrorHandlers = new ArrayList<IDatabaseWriterErrorHandler>(errorHandlers.size() + 1);
            dynamicErrorHandlers.addAll(errorHandlers);
            dynamicErrorHandlers.addAll(databaseWriterFilters);
        }
        List<TransformTableNodeGroupLink> transformsList = transformService.findTransformsFor(link, TransformPoint.LOAD);
        transforms = transformsList != null ? transformsList.toArray(new TransformTable[transformsList.size()]) : null;
        if (isRetry) {
            List<IncomingError> incomingErrors = getIncomingErrors(batchId, sourceNodeId);
            for (IncomingError incomingError : incomingErrors) {
                if (incomingError.isResolveIgnore() || StringUtils.isNotBlank(incomingError.getResolveData())) {
                    resolvedDatas.add(new ResolvedData(incomingError.getFailedRowNumber(), incomingError.getResolveData(), incomingError.isResolveIgnore()));
                }
            }
        }
    }
    TransformWriter transformWriter = new TransformWriter(platform, TransformPoint.LOAD, null, transformService.getColumnTransforms(), transforms);
    IDataWriter targetWriter = getFactory(channelId).getDataWriter(sourceNodeId, symmetricDialect, transformWriter, dynamicFilters, dynamicErrorHandlers, getConflictSettingsNodeGroupLinks(link, false), resolvedDatas);
    transformWriter.setNestedWriter(new ProcessInfoDataWriter(targetWriter, processInfo));
    return transformWriter;
}

Example 3 with LoadFilterType

use of org.jumpmind.symmetric.model.LoadFilter.LoadFilterType in project symmetric-ds by JumpMind.

the class DataLoaderService method buildDataWriter.

protected IDataWriter buildDataWriter(ProcessInfo processInfo, String sourceNodeId, String channelId, long batchId) {
    TransformTable[] transforms = null;
    NodeGroupLink link = null;
    List<ResolvedData> resolvedDatas = new ArrayList<ResolvedData>();
    List<IDatabaseWriterFilter> filters = extensionService.getExtensionPointList(IDatabaseWriterFilter.class);
    List<IDatabaseWriterFilter> dynamicFilters = filters;
    List<IDatabaseWriterErrorHandler> errorHandlers = extensionService.getExtensionPointList(IDatabaseWriterErrorHandler.class);
    List<IDatabaseWriterErrorHandler> dynamicErrorHandlers = errorHandlers;
    if (sourceNodeId != null) {
        Node sourceNode = nodeService.findNode(sourceNodeId);
        if (sourceNode != null) {
            link = new NodeGroupLink(sourceNode.getNodeGroupId(), parameterService.getNodeGroupId());
        }
        Map<LoadFilterType, Map<String, List<LoadFilter>>> loadFilters = loadFilterService.findLoadFiltersFor(link, true);
        List<DynamicDatabaseWriterFilter> databaseWriterFilters = DynamicDatabaseWriterFilter.getDatabaseWriterFilters(engine, loadFilters);
        if (loadFilters != null && loadFilters.size() > 0) {
            dynamicFilters = new ArrayList<IDatabaseWriterFilter>(filters.size() + 1);
            dynamicFilters.addAll(filters);
            dynamicFilters.addAll(databaseWriterFilters);
            dynamicErrorHandlers = new ArrayList<IDatabaseWriterErrorHandler>(errorHandlers.size() + 1);
            dynamicErrorHandlers.addAll(errorHandlers);
            dynamicErrorHandlers.addAll(databaseWriterFilters);
        }
        List<TransformTableNodeGroupLink> transformsList = transformService.findTransformsFor(link, TransformPoint.LOAD);
        transforms = transformsList != null ? transformsList.toArray(new TransformTable[transformsList.size()]) : null;
        List<IncomingError> incomingErrors = getIncomingErrors(batchId, sourceNodeId);
        for (IncomingError incomingError : incomingErrors) {
            if (incomingError.isResolveIgnore() || StringUtils.isNotBlank(incomingError.getResolveData())) {
                resolvedDatas.add(new ResolvedData(incomingError.getFailedRowNumber(), incomingError.getResolveData(), incomingError.isResolveIgnore()));
            }
        }
    }
    TransformWriter transformWriter = new TransformWriter(platform, TransformPoint.LOAD, null, transformService.getColumnTransforms(), transforms);
    IDataWriter targetWriter = getFactory(channelId).getDataWriter(sourceNodeId, symmetricDialect, transformWriter, dynamicFilters, dynamicErrorHandlers, getConflictSettingsNodeGroupLinks(link, false), resolvedDatas);
    transformWriter.setNestedWriter(new ProcessInfoDataWriter(targetWriter, processInfo));
    return transformWriter;
}