Examples with IDataWriter org.jumpmind.symmetric.io.data.IDataWriter used on opensource projects

Example 1 with IDataWriter

use of org.jumpmind.symmetric.io.data.IDataWriter in project symmetric-ds by JumpMind.

the class MongoDataLoaderFactory method getDataWriter.

@Override
public IDataWriter getDataWriter(String sourceNodeId, ISymmetricDialect symmetricDialect, TransformWriter transformWriter, List<IDatabaseWriterFilter> filters, List<IDatabaseWriterErrorHandler> errorHandlers, List<? extends Conflict> conflictSettings, List<ResolvedData> resolvedData) {
    try {
        if (objectMapper == null) {
            objectMapper = (IDBObjectMapper) Class.forName("org.jumpmind.symmetric.io.SimpleDBObjectMapper").newInstance();
        }
        Method method = objectMapper.getClass().getMethod("setDefaultDatabaseName", String.class);
        if (method != null) {
            method.invoke(objectMapper, parameterService.getString("mongodb.default.databasename", "default"));
        }
    } catch (Exception e) {
        log.debug("Failed to call setDefaultDatabaseName on mapper", e);
    }
    try {
        Class<?> clientManagerClass = Class.forName("org.jumpmind.symmetric.io.SimpleMongoClientManager");
        Constructor<?> clientManagerConstrutor = clientManagerClass.getConstructor(new Class<?>[] { IParameterService.class, String.class });
        Class<?> dbWriterClass = Class.forName("org.jumpmind.symmetric.io.MongoDatabaseWriter");
        Constructor<?> dbWriterConstructor = dbWriterClass.getConstructor(new Class<?>[] { IDBObjectMapper.class, IMongoClientManager.class, IDatabaseWriterConflictResolver.class, DatabaseWriterSettings.class });
        Object clientManager = clientManagerConstrutor.newInstance(parameterService, typeName);
        return (IDataWriter) dbWriterConstructor.newInstance(objectMapper, clientManager, new DefaultTransformWriterConflictResolver(transformWriter), buildDatabaseWriterSettings(filters, errorHandlers, conflictSettings, resolvedData));
    } catch (Exception e) {
        log.warn("Failed to create the mongo database writer.  Check to see if all of the required jars have been added");
        if (e instanceof RuntimeException) {
            throw (RuntimeException) e;
        } else {
            throw new RuntimeException(e);
        }
    }
}

Example 2 with IDataWriter

use of org.jumpmind.symmetric.io.data.IDataWriter in project symmetric-ds by JumpMind.

the class DataExtractorService method extractOutgoingBatch.

protected OutgoingBatch extractOutgoingBatch(ProcessInfo processInfo, Node targetNode, IDataWriter dataWriter, OutgoingBatch currentBatch, boolean useStagingDataWriter, boolean updateBatchStatistics, ExtractMode mode) {
    if (currentBatch.getStatus() != Status.OK || ExtractMode.EXTRACT_ONLY == mode || ExtractMode.FOR_SYM_CLIENT == mode) {
        Node sourceNode = nodeService.findIdentity();
        IDataWriter writer = wrapWithTransformWriter(sourceNode, targetNode, processInfo, dataWriter, useStagingDataWriter);
        long ts = System.currentTimeMillis();
        long extractTimeInMs = 0l;
        long byteCount = 0l;
        long transformTimeInMs = 0l;
        if (currentBatch.getStatus() == Status.IG) {
            cleanupIgnoredBatch(sourceNode, targetNode, currentBatch, writer);
        } else if (!isPreviouslyExtracted(currentBatch, true)) {
            String semaphoreKey = useStagingDataWriter ? Long.toString(currentBatch.getBatchId()) : currentBatch.getNodeBatchId();
            Semaphore lock = null;
            try {
                synchronized (locks) {
                    lock = locks.get(semaphoreKey);
                    if (lock == null) {
                        lock = new Semaphore(1);
                        locks.put(semaphoreKey, lock);
                    }
                    try {
                        lock.acquire();
                    } catch (InterruptedException e) {
                        throw new org.jumpmind.exception.InterruptedException(e);
                    }
                }
                if (!isPreviouslyExtracted(currentBatch, true)) {
                    currentBatch.setExtractCount(currentBatch.getExtractCount() + 1);
                    if (updateBatchStatistics) {
                        changeBatchStatus(Status.QY, currentBatch, mode);
                    }
                    currentBatch.resetStats();
                    DataContext ctx = new DataContext();
                    ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_ID, targetNode.getNodeId());
                    ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_EXTERNAL_ID, targetNode.getExternalId());
                    ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_GROUP_ID, targetNode.getNodeGroupId());
                    ctx.put(Constants.DATA_CONTEXT_TARGET_NODE, targetNode);
                    ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE, sourceNode);
                    ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_ID, sourceNode.getNodeId());
                    ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_EXTERNAL_ID, sourceNode.getExternalId());
                    ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_GROUP_ID, sourceNode.getNodeGroupId());
                    IDataReader dataReader = buildExtractDataReader(sourceNode, targetNode, currentBatch, processInfo);
                    new DataProcessor(dataReader, writer, "extract").process(ctx);
                    extractTimeInMs = System.currentTimeMillis() - ts;
                    Statistics stats = getExtractStats(writer);
                    if (stats != null) {
                        transformTimeInMs = stats.get(DataWriterStatisticConstants.TRANSFORMMILLIS);
                        extractTimeInMs = extractTimeInMs - transformTimeInMs;
                        byteCount = stats.get(DataWriterStatisticConstants.BYTECOUNT);
                        statisticManager.incrementDataBytesExtracted(currentBatch.getChannelId(), byteCount);
                        statisticManager.incrementDataExtracted(currentBatch.getChannelId(), stats.get(DataWriterStatisticConstants.STATEMENTCOUNT));
                    }
                }
            } catch (RuntimeException ex) {
                IStagedResource resource = getStagedResource(currentBatch);
                if (resource != null) {
                    resource.close();
                    resource.delete();
                }
                throw ex;
            } finally {
                IStagedResource resource = getStagedResource(currentBatch);
                if (resource != null) {
                    resource.setState(State.DONE);
                }
                lock.release();
                synchronized (locks) {
                    locks.remove(semaphoreKey);
                }
            }
        }
        if (updateBatchStatistics) {
            long dataEventCount = currentBatch.getDataEventCount();
            long insertEventCount = currentBatch.getInsertEventCount();
            currentBatch = requeryIfEnoughTimeHasPassed(ts, currentBatch);
            // preserve in the case of a reload event
            if (dataEventCount > currentBatch.getDataEventCount()) {
                currentBatch.setDataEventCount(dataEventCount);
            }
            // preserve in the case of a reload event
            if (insertEventCount > currentBatch.getInsertEventCount()) {
                currentBatch.setInsertEventCount(insertEventCount);
            }
            // "re-queried"
            if (extractTimeInMs > 0) {
                currentBatch.setExtractMillis(extractTimeInMs);
            }
            if (byteCount > 0) {
                currentBatch.setByteCount(byteCount);
            }
        }
    }
    processInfo.incrementCurrentBatchCount();
    return currentBatch;
}

Example 3 with IDataWriter

use of org.jumpmind.symmetric.io.data.IDataWriter in project symmetric-ds by JumpMind.

the class DataExtractorService method extract.

public List<OutgoingBatch> extract(ProcessInfo processInfo, Node targetNode, String queue, IOutgoingTransport transport) {
    /*
         * make sure that data is routed before extracting if the route job is
         * not configured to start automatically
         */
    if (!parameterService.is(ParameterConstants.START_ROUTE_JOB) && parameterService.is(ParameterConstants.ROUTE_ON_EXTRACT)) {
        routerService.routeData(true);
    }
    OutgoingBatches batches = null;
    if (queue != null) {
        NodeGroupLinkAction defaultAction = configurationService.getNodeGroupLinkFor(nodeService.findIdentity().getNodeGroupId(), targetNode.getNodeGroupId(), false).getDataEventAction();
        ProcessInfoKey.ProcessType processType = processInfo.getKey().getProcessType();
        NodeGroupLinkAction action = null;
        if (processType.equals(ProcessInfoKey.ProcessType.PUSH_JOB)) {
            action = NodeGroupLinkAction.P;
        } else if (processType.equals(ProcessInfoKey.ProcessType.PULL_HANDLER)) {
            action = NodeGroupLinkAction.W;
        }
        batches = outgoingBatchService.getOutgoingBatches(targetNode.getNodeId(), queue, action, defaultAction, false);
    } else {
        batches = outgoingBatchService.getOutgoingBatches(targetNode.getNodeId(), false);
    }
    if (batches.containsBatches()) {
        ChannelMap channelMap = transport.getSuspendIgnoreChannelLists(configurationService, queue, targetNode);
        List<OutgoingBatch> activeBatches = filterBatchesForExtraction(batches, channelMap);
        if (activeBatches.size() > 0) {
            BufferedWriter writer = transport.openWriter();
            IDataWriter dataWriter = new ProtocolDataWriter(nodeService.findIdentityNodeId(), writer, targetNode.requires13Compatiblity());
            return extract(processInfo, targetNode, activeBatches, dataWriter, writer, ExtractMode.FOR_SYM_CLIENT);
        }
    }
    return Collections.emptyList();
}

Example 4 with IDataWriter

use of org.jumpmind.symmetric.io.data.IDataWriter in project symmetric-ds by JumpMind.

the class DataLoaderService method buildDataWriter.

protected IDataWriter buildDataWriter(ProcessInfo processInfo, String sourceNodeId, String channelId, long batchId, boolean isRetry) {
    TransformTable[] transforms = null;
    NodeGroupLink link = null;
    List<ResolvedData> resolvedDatas = new ArrayList<ResolvedData>();
    List<IDatabaseWriterFilter> filters = extensionService.getExtensionPointList(IDatabaseWriterFilter.class);
    List<IDatabaseWriterFilter> dynamicFilters = filters;
    List<IDatabaseWriterErrorHandler> errorHandlers = extensionService.getExtensionPointList(IDatabaseWriterErrorHandler.class);
    List<IDatabaseWriterErrorHandler> dynamicErrorHandlers = errorHandlers;
    if (sourceNodeId != null) {
        Node sourceNode = nodeService.findNode(sourceNodeId, true);
        if (sourceNode != null) {
            link = new NodeGroupLink(sourceNode.getNodeGroupId(), parameterService.getNodeGroupId());
        }
        Map<LoadFilterType, Map<String, List<LoadFilter>>> loadFilters = loadFilterService.findLoadFiltersFor(link, true);
        List<DynamicDatabaseWriterFilter> databaseWriterFilters = DynamicDatabaseWriterFilter.getDatabaseWriterFilters(engine, loadFilters);
        if (loadFilters != null && loadFilters.size() > 0) {
            dynamicFilters = new ArrayList<IDatabaseWriterFilter>(filters.size() + 1);
            dynamicFilters.addAll(filters);
            dynamicFilters.addAll(databaseWriterFilters);
            dynamicErrorHandlers = new ArrayList<IDatabaseWriterErrorHandler>(errorHandlers.size() + 1);
            dynamicErrorHandlers.addAll(errorHandlers);
            dynamicErrorHandlers.addAll(databaseWriterFilters);
        }
        List<TransformTableNodeGroupLink> transformsList = transformService.findTransformsFor(link, TransformPoint.LOAD);
        transforms = transformsList != null ? transformsList.toArray(new TransformTable[transformsList.size()]) : null;
        if (isRetry) {
            List<IncomingError> incomingErrors = getIncomingErrors(batchId, sourceNodeId);
            for (IncomingError incomingError : incomingErrors) {
                if (incomingError.isResolveIgnore() || StringUtils.isNotBlank(incomingError.getResolveData())) {
                    resolvedDatas.add(new ResolvedData(incomingError.getFailedRowNumber(), incomingError.getResolveData(), incomingError.isResolveIgnore()));
                }
            }
        }
    }
    TransformWriter transformWriter = new TransformWriter(platform, TransformPoint.LOAD, null, transformService.getColumnTransforms(), transforms);
    IDataWriter targetWriter = getFactory(channelId).getDataWriter(sourceNodeId, symmetricDialect, transformWriter, dynamicFilters, dynamicErrorHandlers, getConflictSettingsNodeGroupLinks(link, false), resolvedDatas);
    transformWriter.setNestedWriter(new ProcessInfoDataWriter(targetWriter, processInfo));
    return transformWriter;
}

Example 5 with IDataWriter

use of org.jumpmind.symmetric.io.data.IDataWriter in project symmetric-ds by JumpMind.

the class DataExtractorService method extractOnlyOutgoingBatch.

/**
     * This method will extract an outgoing batch, but will not update the outgoing batch status
     */
public boolean extractOnlyOutgoingBatch(String nodeId, long batchId, Writer writer) {
    boolean extracted = false;
    Node targetNode = null;
    if (Constants.UNROUTED_NODE_ID.equals(nodeId)) {
        targetNode = new Node(nodeId, parameterService.getNodeGroupId());
    } else {
        targetNode = nodeService.findNode(nodeId);
    }
    if (targetNode != null) {
        OutgoingBatch batch = outgoingBatchService.findOutgoingBatch(batchId, nodeId);
        if (batch != null) {
            IDataWriter dataWriter = new ProtocolDataWriter(nodeService.findIdentityNodeId(), writer, targetNode.requires13Compatiblity());
            List<OutgoingBatch> batches = new ArrayList<OutgoingBatch>(1);
            batches.add(batch);
            batches = extract(new ProcessInfo(), targetNode, batches, dataWriter, null, ExtractMode.EXTRACT_ONLY);
            extracted = batches.size() > 0;
        }
    }
    return extracted;
}