use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.
the class DataExtractorService method extract.
protected List<OutgoingBatch> extract(final ProcessInfo processInfo, final Node targetNode, final List<OutgoingBatch> activeBatches, final IDataWriter dataWriter, final BufferedWriter writer, final ExtractMode mode) {
if (activeBatches.size() > 0) {
final List<OutgoingBatch> processedBatches = new ArrayList<OutgoingBatch>(activeBatches.size());
Set<String> channelsProcessed = new HashSet<String>();
long batchesSelectedAtMs = System.currentTimeMillis();
OutgoingBatch currentBatch = null;
ExecutorService executor = null;
try {
final boolean streamToFileEnabled = parameterService.is(ParameterConstants.STREAM_TO_FILE_ENABLED);
long keepAliveMillis = parameterService.getLong(ParameterConstants.DATA_LOADER_SEND_ACK_KEEPALIVE);
Node sourceNode = nodeService.findIdentity();
final FutureExtractStatus status = new FutureExtractStatus();
executor = Executors.newFixedThreadPool(1, new CustomizableThreadFactory(String.format("dataextractor-%s-%s", targetNode.getNodeGroupId(), targetNode.getNodeGroupId())));
List<Future<FutureOutgoingBatch>> futures = new ArrayList<Future<FutureOutgoingBatch>>();
processInfo.setBatchCount(activeBatches.size());
for (int i = 0; i < activeBatches.size(); i++) {
currentBatch = activeBatches.get(i);
processInfo.setCurrentLoadId(currentBatch.getLoadId());
processInfo.setDataCount(currentBatch.getDataEventCount());
processInfo.setCurrentBatchId(currentBatch.getBatchId());
channelsProcessed.add(currentBatch.getChannelId());
currentBatch = requeryIfEnoughTimeHasPassed(batchesSelectedAtMs, currentBatch);
processInfo.setStatus(ProcessInfo.Status.EXTRACTING);
final OutgoingBatch extractBatch = currentBatch;
Callable<FutureOutgoingBatch> callable = new Callable<FutureOutgoingBatch>() {
public FutureOutgoingBatch call() throws Exception {
return extractBatch(extractBatch, status, processInfo, targetNode, dataWriter, mode, activeBatches);
}
};
if (status.shouldExtractSkip) {
break;
}
futures.add(executor.submit(callable));
}
if (parameterService.is(ParameterConstants.SYNCHRONIZE_ALL_JOBS)) {
executor.shutdown();
boolean isProcessed = false;
while (!isProcessed) {
try {
isProcessed = executor.awaitTermination(keepAliveMillis, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
if (!isProcessed) {
writeKeepAliveAck(writer, sourceNode, streamToFileEnabled);
}
}
}
Iterator<OutgoingBatch> activeBatchIter = activeBatches.iterator();
for (Future<FutureOutgoingBatch> future : futures) {
currentBatch = activeBatchIter.next();
boolean isProcessed = false;
while (!isProcessed) {
try {
FutureOutgoingBatch extractBatch = future.get(keepAliveMillis, TimeUnit.MILLISECONDS);
currentBatch = extractBatch.getOutgoingBatch();
if (extractBatch.isExtractSkipped) {
break;
}
if (streamToFileEnabled || mode == ExtractMode.FOR_PAYLOAD_CLIENT) {
processInfo.setStatus(ProcessInfo.Status.TRANSFERRING);
processInfo.setCurrentLoadId(currentBatch.getLoadId());
boolean isRetry = extractBatch.isRetry() && extractBatch.getOutgoingBatch().getStatus() != OutgoingBatch.Status.IG;
currentBatch = sendOutgoingBatch(processInfo, targetNode, currentBatch, isRetry, dataWriter, writer, mode);
}
processedBatches.add(currentBatch);
isProcessed = true;
if (currentBatch.getStatus() != Status.OK) {
currentBatch.setLoadCount(currentBatch.getLoadCount() + 1);
changeBatchStatus(Status.LD, currentBatch, mode);
}
} catch (ExecutionException e) {
if (isNotBlank(e.getMessage()) && e.getMessage().contains("string truncation")) {
throw new RuntimeException("There is a good chance that the truncation error you are receiving is because contains_big_lobs on the '" + currentBatch.getChannelId() + "' channel needs to be turned on.", e.getCause() != null ? e.getCause() : e);
}
throw new RuntimeException(e.getCause() != null ? e.getCause() : e);
} catch (InterruptedException e) {
throw new RuntimeException(e);
} catch (TimeoutException e) {
writeKeepAliveAck(writer, sourceNode, streamToFileEnabled);
}
}
}
} catch (RuntimeException e) {
SQLException se = unwrapSqlException(e);
if (currentBatch != null) {
try {
/* Reread batch in case the ignore flag has been set */
currentBatch = outgoingBatchService.findOutgoingBatch(currentBatch.getBatchId(), currentBatch.getNodeId());
statisticManager.incrementDataExtractedErrors(currentBatch.getChannelId(), 1);
if (se != null) {
currentBatch.setSqlState(se.getSQLState());
currentBatch.setSqlCode(se.getErrorCode());
currentBatch.setSqlMessage(se.getMessage());
} else {
currentBatch.setSqlMessage(getRootMessage(e));
}
currentBatch.revertStatsOnError();
if (currentBatch.getStatus() != Status.IG && currentBatch.getStatus() != Status.OK) {
currentBatch.setStatus(Status.ER);
currentBatch.setErrorFlag(true);
}
outgoingBatchService.updateOutgoingBatch(currentBatch);
} catch (Exception ex) {
log.error("Failed to update the outgoing batch status for failed batch {}", currentBatch, ex);
} finally {
if (!isStreamClosedByClient(e)) {
if (e instanceof ProtocolException) {
IStagedResource resource = getStagedResource(currentBatch);
if (resource != null) {
resource.delete();
}
}
if (e.getCause() instanceof InterruptedException) {
log.info("Extract of batch {} was interrupted", currentBatch);
} else {
log.error("Failed to extract batch {}", currentBatch, e);
}
}
processInfo.setStatus(ProcessInfo.Status.ERROR);
}
} else {
log.error("Could not log the outgoing batch status because the batch was null", e);
}
} finally {
if (executor != null) {
executor.shutdown();
}
}
// Next, we update the node channel controls to the
// current timestamp
Calendar now = Calendar.getInstance();
for (String channelProcessed : channelsProcessed) {
NodeChannel nodeChannel = configurationService.getNodeChannel(channelProcessed, targetNode.getNodeId(), false);
if (nodeChannel != null && nodeChannel.getExtractPeriodMillis() > 0) {
nodeChannel.setLastExtractTime(now.getTime());
configurationService.updateLastExtractTime(nodeChannel);
}
}
return processedBatches;
} else {
return Collections.emptyList();
}
}
use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.
the class DataExtractorService method extractBatchRange.
public boolean extractBatchRange(Writer writer, String nodeId, Date startBatchTime, Date endBatchTime, String... channelIds) {
boolean foundBatch = false;
Node sourceNode = nodeService.findIdentity();
OutgoingBatches batches = outgoingBatchService.getOutgoingBatchRange(nodeId, startBatchTime, endBatchTime, channelIds);
List<OutgoingBatch> list = batches.getBatches();
for (OutgoingBatch outgoingBatch : list) {
Node targetNode = nodeService.findNode(nodeId);
if (targetNode == null && Constants.UNROUTED_NODE_ID.equals(nodeId)) {
targetNode = new Node();
targetNode.setNodeId("-1");
}
if (targetNode != null) {
IDataReader dataReader = new ExtractDataReader(symmetricDialect.getPlatform(), new SelectFromSymDataSource(outgoingBatch, sourceNode, targetNode, new ProcessInfo()));
DataContext ctx = new DataContext();
ctx.put(Constants.DATA_CONTEXT_TARGET_NODE, targetNode);
ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE, nodeService.findIdentity());
new DataProcessor(dataReader, createTransformDataWriter(nodeService.findIdentity(), targetNode, new ProtocolDataWriter(nodeService.findIdentityNodeId(), writer, targetNode.requires13Compatiblity())), "extract range").process(ctx);
foundBatch = true;
}
}
return foundBatch;
}
use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.
the class DataExtractorService method filterBatchesForExtraction.
private List<OutgoingBatch> filterBatchesForExtraction(OutgoingBatches batches, ChannelMap suspendIgnoreChannelsList) {
if (parameterService.is(ParameterConstants.FILE_SYNC_ENABLE)) {
List<Channel> fileSyncChannels = configurationService.getFileSyncChannels();
for (Channel channel : fileSyncChannels) {
batches.filterBatchesForChannel(channel);
}
}
// We now have either our local suspend/ignore list, or the combined
// remote send/ignore list and our local list (along with a
// reservation, if we go this far...)
// Now, we need to skip the suspended channels and ignore the
// ignored ones by ultimately setting the status to ignored and
// updating them.
List<OutgoingBatch> ignoredBatches = batches.filterBatchesForChannels(suspendIgnoreChannelsList.getIgnoreChannels());
// will be skipped in the future.
for (OutgoingBatch batch : ignoredBatches) {
batch.setStatus(OutgoingBatch.Status.OK);
batch.incrementIgnoreCount();
if (log.isDebugEnabled()) {
log.debug("Batch {} is being ignored", batch.getBatchId());
}
}
outgoingBatchService.updateOutgoingBatches(ignoredBatches);
batches.filterBatchesForChannels(suspendIgnoreChannelsList.getSuspendChannels());
// any other batches are loaded.
if (parameterService.is(ParameterConstants.INITIAL_LOAD_BLOCK_CHANNELS, true)) {
if (!batches.containsBatchesInError() && batches.containsLoadBatches()) {
batches.removeNonLoadBatches();
}
}
return batches.getBatches();
}
use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.
the class DataExtractorService method extract.
public List<OutgoingBatch> extract(ProcessInfo processInfo, Node targetNode, String queue, IOutgoingTransport transport) {
/*
* make sure that data is routed before extracting if the route job is
* not configured to start automatically
*/
if (!parameterService.is(ParameterConstants.START_ROUTE_JOB) && parameterService.is(ParameterConstants.ROUTE_ON_EXTRACT)) {
routerService.routeData(true);
}
OutgoingBatches batches = null;
if (queue != null) {
NodeGroupLinkAction defaultAction = configurationService.getNodeGroupLinkFor(nodeService.findIdentity().getNodeGroupId(), targetNode.getNodeGroupId(), false).getDataEventAction();
ProcessInfoKey.ProcessType processType = processInfo.getKey().getProcessType();
NodeGroupLinkAction action = null;
if (processType.equals(ProcessInfoKey.ProcessType.PUSH_JOB)) {
action = NodeGroupLinkAction.P;
} else if (processType.equals(ProcessInfoKey.ProcessType.PULL_HANDLER)) {
action = NodeGroupLinkAction.W;
}
batches = outgoingBatchService.getOutgoingBatches(targetNode.getNodeId(), queue, action, defaultAction, false);
} else {
batches = outgoingBatchService.getOutgoingBatches(targetNode.getNodeId(), false);
}
if (batches.containsBatches()) {
ChannelMap channelMap = transport.getSuspendIgnoreChannelLists(configurationService, queue, targetNode);
List<OutgoingBatch> activeBatches = filterBatchesForExtraction(batches, channelMap);
if (activeBatches.size() > 0) {
BufferedWriter writer = transport.openWriter();
IDataWriter dataWriter = new ProtocolDataWriter(nodeService.findIdentityNodeId(), writer, targetNode.requires13Compatiblity());
return extract(processInfo, targetNode, activeBatches, dataWriter, writer, ExtractMode.FOR_SYM_CLIENT);
}
}
return Collections.emptyList();
}
use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.
the class DataExtractorService method extractToPayload.
public List<OutgoingBatchWithPayload> extractToPayload(ProcessInfo processInfo, Node targetNode, PayloadType payloadType, boolean useJdbcTimestampFormat, boolean useUpsertStatements, boolean useDelimiterIdentifiers) {
OutgoingBatches batches = outgoingBatchService.getOutgoingBatches(targetNode.getNodeId(), false);
if (batches.containsBatches()) {
ChannelMap channelMap = configurationService.getSuspendIgnoreChannelLists(targetNode.getNodeId());
List<OutgoingBatch> activeBatches = filterBatchesForExtraction(batches, channelMap);
if (activeBatches.size() > 0) {
IDdlBuilder builder = DdlBuilderFactory.createDdlBuilder(targetNode.getDatabaseType());
if (builder == null) {
throw new IllegalStateException("Could not find a ddl builder registered for the database type of " + targetNode.getDatabaseType() + ". Please check the database type setting for node '" + targetNode.getNodeId() + "'");
}
StructureDataWriter writer = new StructureDataWriter(symmetricDialect.getPlatform(), targetNode.getDatabaseType(), payloadType, useDelimiterIdentifiers, symmetricDialect.getBinaryEncoding(), useJdbcTimestampFormat, useUpsertStatements);
List<OutgoingBatch> extractedBatches = extract(processInfo, targetNode, activeBatches, writer, null, ExtractMode.FOR_PAYLOAD_CLIENT);
List<OutgoingBatchWithPayload> batchesWithPayload = new ArrayList<OutgoingBatchWithPayload>();
for (OutgoingBatch batch : extractedBatches) {
OutgoingBatchWithPayload batchWithPayload = new OutgoingBatchWithPayload(batch, payloadType);
batchWithPayload.setPayload(writer.getPayloadMap().get(batch.getBatchId()));
batchWithPayload.setPayloadType(payloadType);
batchesWithPayload.add(batchWithPayload);
}
return batchesWithPayload;
}
}
return Collections.emptyList();
}
Aggregations