Examples with FileTriggerRouter org.jumpmind.symmetric.model.FileTriggerRouter used on opensource projects

Example 6 with FileTriggerRouter

use of org.jumpmind.symmetric.model.FileTriggerRouter in project symmetric-ds by JumpMind.

the class FileSyncService method acknowledgeFiles.

public void acknowledgeFiles(OutgoingBatch outgoingBatch) {
    log.debug("Acknowledging file_sync outgoing batch-{}", outgoingBatch.getBatchId());
    List<File> filesToDelete = new ArrayList<File>();
    Table snapshotTable = platform.getTableFromCache(TableConstants.getTableName(tablePrefix, TableConstants.SYM_FILE_SNAPSHOT), false);
    ISqlReadCursor<Data> cursor = engine.getDataService().selectDataFor(outgoingBatch.getBatchId(), outgoingBatch.getChannelId());
    Data data = cursor.next();
    while (data != null) {
        if (data.getDataEventType() == DataEventType.INSERT || data.getDataEventType() == DataEventType.UPDATE) {
            Map<String, String> columnData = data.toColumnNameValuePairs(snapshotTable.getColumnNames(), CsvData.ROW_DATA);
            FileSnapshot fileSnapshot = new FileSnapshot();
            fileSnapshot.setTriggerId(columnData.get("TRIGGER_ID"));
            fileSnapshot.setRouterId(columnData.get("ROUTER_ID"));
            fileSnapshot.setFileModifiedTime(Long.parseLong(columnData.get("FILE_MODIFIED_TIME")));
            fileSnapshot.setFileName(columnData.get("FILE_NAME"));
            fileSnapshot.setRelativeDir(columnData.get("RELATIVE_DIR"));
            fileSnapshot.setLastEventType(LastEventType.fromCode(columnData.get("LAST_EVENT_TYPE")));
            FileTriggerRouter triggerRouter = this.getFileTriggerRouter(fileSnapshot.getTriggerId(), fileSnapshot.getRouterId());
            if (triggerRouter != null) {
                FileTrigger fileTrigger = triggerRouter.getFileTrigger();
                if (fileTrigger.isDeleteAfterSync()) {
                    File file = fileTrigger.createSourceFile(fileSnapshot);
                    if (!file.isDirectory()) {
                        filesToDelete.add(file);
                        if (fileTrigger.isSyncOnCtlFile()) {
                            filesToDelete.add(this.getControleFile(file));
                        }
                    }
                } else if (parameterService.is(ParameterConstants.FILE_SYNC_DELETE_CTL_FILE_AFTER_SYNC, false)) {
                    File file = fileTrigger.createSourceFile(fileSnapshot);
                    if (!file.isDirectory()) {
                        if (fileTrigger.isSyncOnCtlFile()) {
                            filesToDelete.add(this.getControleFile(file));
                        }
                    }
                }
            }
        }
        data = cursor.next();
    }
    if (cursor != null) {
        cursor.close();
        cursor = null;
    }
    if (filesToDelete != null && filesToDelete.size() > 0) {
        for (File file : filesToDelete) {
            if (file != null && file.exists()) {
                log.debug("Deleting the '{}' file", file.getAbsolutePath());
                boolean deleted = FileUtils.deleteQuietly(file);
                if (!deleted) {
                    log.warn("Failed to 'delete on sync' the {} file", file.getAbsolutePath());
                }
            }
            file = null;
        }
        filesToDelete = null;
    }
}

Example 7 with FileTriggerRouter

use of org.jumpmind.symmetric.model.FileTriggerRouter in project symmetric-ds by JumpMind.

the class FileSyncService method trackChangesFastScan.

protected void trackChangesFastScan(ProcessInfo processInfo, boolean useCrc) {
    boolean isLocked = engine.getClusterService().lock(ClusterConstants.FILE_SYNC_SCAN);
    Lock lock = engine.getClusterService().findLocks().get(ClusterConstants.FILE_SYNC_SCAN);
    log.debug("File tracker range of " + lock.getLastLockTime() + " to " + lock.getLockTime() + ", isLocked=" + isLocked);
    int maxRowsBeforeCommit = engine.getParameterService().getInt(ParameterConstants.DATA_LOADER_MAX_ROWS_BEFORE_COMMIT);
    try {
        List<FileTriggerRouter> fileTriggerRouters = getFileTriggerRoutersForCurrentNode();
        for (final FileTriggerRouter fileTriggerRouter : fileTriggerRouters) {
            if (fileTriggerRouter.isEnabled()) {
                FileAlterationObserver observer = new FileAlterationObserver(fileTriggerRouter.getFileTrigger().getBaseDir(), fileTriggerRouter.getFileTrigger().createIOFileFilter());
                FileTriggerFileModifiedListener listener = new FileTriggerFileModifiedListener(fileTriggerRouter, lock.getLastLockTime(), lock.getLockTime(), processInfo, useCrc, new FileModifiedCallback(maxRowsBeforeCommit) {

                    public void commit(DirectorySnapshot dirSnapshot) {
                        saveDirectorySnapshot(fileTriggerRouter, dirSnapshot);
                    }

                    public DirectorySnapshot getLastDirectorySnapshot(String relativeDir) {
                        return getDirectorySnapshot(fileTriggerRouter, relativeDir);
                    }
                }, engine);
                observer.addListener(listener);
                observer.checkAndNotify();
            }
        }
        engine.getClusterService().unlock(ClusterConstants.FILE_SYNC_SCAN);
    } catch (Exception ex) {
        log.error("Failed to track changes", ex);
    }
}

Example 8 with FileTriggerRouter

use of org.jumpmind.symmetric.model.FileTriggerRouter in project symmetric-ds by JumpMind.

the class FileSyncService method trackChanges.

protected void trackChanges(ProcessInfo processInfo, boolean useCrc) {
    List<FileTriggerRouter> fileTriggerRouters = getFileTriggerRoutersForCurrentNode();
    for (FileTriggerRouter fileTriggerRouter : fileTriggerRouters) {
        if (fileTriggerRouter.isEnabled()) {
            try {
                FileTriggerTracker tracker = new FileTriggerTracker(fileTriggerRouter, getDirectorySnapshot(fileTriggerRouter), processInfo, useCrc, engine);
                DirectorySnapshot dirSnapshot = tracker.trackChanges();
                saveDirectorySnapshot(fileTriggerRouter, dirSnapshot);
            } catch (Exception ex) {
                log.error("Failed to track changes for file trigger router: " + fileTriggerRouter.getFileTrigger().getTriggerId() + "::" + fileTriggerRouter.getRouter().getRouterId(), ex);
            }
        }
    }
}

Example 9 with FileTriggerRouter

use of org.jumpmind.symmetric.model.FileTriggerRouter in project symmetric-ds by JumpMind.

the class FileSyncZipDataWriter method end.

public void end(Batch batch, boolean inError) {
    try {
        if (!inError) {
            if (zos == null) {
                zos = new ZipOutputStream(stagedResource.getOutputStream());
            }
            FileSyncZipScript script = createFileSyncZipScript(batch.getTargetNodeId());
            script.buildScriptStart(batch);
            Map<String, LastEventType> entries = new HashMap<String, LastEventType>();
            for (FileSnapshot snapshot : snapshotEvents) {
                FileTriggerRouter triggerRouter = fileSyncService.getFileTriggerRouter(snapshot.getTriggerId(), snapshot.getRouterId());
                if (triggerRouter != null) {
                    LastEventType eventType = snapshot.getLastEventType();
                    FileTrigger fileTrigger = triggerRouter.getFileTrigger();
                    String targetBaseDir = ((triggerRouter.getTargetBaseDir() == null) ? null : triggerRouter.getTargetBaseDir().replace('\\', '/'));
                    if (StringUtils.isBlank(targetBaseDir)) {
                        targetBaseDir = ((fileTrigger.getBaseDir() == null) ? null : fileTrigger.getBaseDir().replace('\\', '/'));
                    }
                    targetBaseDir = StringEscapeUtils.escapeJava(targetBaseDir);
                    StringBuilder entryName = new StringBuilder(Long.toString(batch.getBatchId()));
                    entryName.append("/");
                    if (!snapshot.getRelativeDir().equals(".")) {
                        entryName.append(snapshot.getRelativeDir()).append("/");
                    }
                    entryName.append(snapshot.getFileName());
                    File file = fileTrigger.createSourceFile(snapshot);
                    if (file.isDirectory()) {
                        entryName.append("/");
                    }
                    String targetFile = "targetBaseDir + \"/\" + targetRelativeDir + \"/\" + targetFileName";
                    LastEventType previousEventForEntry = entries.get(entryName.toString());
                    boolean process = true;
                    if (previousEventForEntry != null) {
                        if ((previousEventForEntry == eventType) || (previousEventForEntry == LastEventType.CREATE && eventType == LastEventType.MODIFY)) {
                            process = false;
                        }
                    }
                    if (process) {
                        if (eventType != LastEventType.DELETE) {
                            if (file.exists()) {
                                byteCount += file.length();
                                ZipEntry entry = new ZipEntry(entryName.toString());
                                entry.setSize(file.length());
                                entry.setTime(file.lastModified());
                                zos.putNextEntry(entry);
                                if (file.isFile()) {
                                    FileInputStream fis = new FileInputStream(file);
                                    try {
                                        IOUtils.copy(fis, zos);
                                    } finally {
                                        IOUtils.closeQuietly(fis);
                                    }
                                }
                                zos.closeEntry();
                                entries.put(entryName.toString(), eventType);
                            } else {
                                log.warn("Could not find the {} file to package for synchronization.  Skipping it.", file.getAbsolutePath());
                            }
                        }
                        script.buildScriptFileSnapshot(batch, snapshot, triggerRouter, fileTrigger, file, targetBaseDir, targetFile);
                    }
                } else {
                    log.error("Could not locate the file trigger ({}) router ({}) to process a snapshot event.  The event will be ignored", snapshot.getTriggerId(), snapshot.getRouterId());
                }
            }
            script.buildScriptEnd(batch);
            ZipEntry entry = new ZipEntry(batch.getBatchId() + "/" + script.getScriptFileName(batch));
            zos.putNextEntry(entry);
            IOUtils.write(script.getScript().toString(), zos);
            zos.closeEntry();
            entry = new ZipEntry(batch.getBatchId() + "/batch-info.txt");
            zos.putNextEntry(entry);
            IOUtils.write(batch.getChannelId(), zos);
            zos.closeEntry();
        }
    } catch (IOException e) {
        throw new IoException(e);
    }
}

Example 10 with FileTriggerRouter

use of org.jumpmind.symmetric.model.FileTriggerRouter in project symmetric-ds by JumpMind.

the class FileSyncZipDataWriter method write.

public void write(CsvData data) {
    DataEventType eventType = data.getDataEventType();
    if (eventType == DataEventType.INSERT || eventType == DataEventType.UPDATE) {
        Map<String, String> columnData = data.toColumnNameValuePairs(snapshotTable.getColumnNames(), CsvData.ROW_DATA);
        Map<String, String> oldColumnData = data.toColumnNameValuePairs(snapshotTable.getColumnNames(), CsvData.OLD_DATA);
        FileSnapshot snapshot = new FileSnapshot();
        snapshot.setTriggerId(columnData.get("TRIGGER_ID"));
        snapshot.setRouterId(columnData.get("ROUTER_ID"));
        snapshot.setFileModifiedTime(Long.parseLong(columnData.get("FILE_MODIFIED_TIME")));
        snapshot.setCrc32Checksum(Long.parseLong(columnData.get("CRC32_CHECKSUM")));
        String oldChecksum = oldColumnData.get("CRC32_CHECKSUM");
        if (StringUtils.isNotBlank(oldChecksum)) {
            snapshot.setOldCrc32Checksum(Long.parseLong(oldChecksum));
        }
        snapshot.setFileSize(Long.parseLong(columnData.get("FILE_SIZE")));
        snapshot.setLastUpdateBy(columnData.get("LAST_UPDATE_BY"));
        snapshot.setFileName(columnData.get("FILE_NAME"));
        snapshot.setRelativeDir(columnData.get("RELATIVE_DIR"));
        snapshot.setLastEventType(LastEventType.fromCode(columnData.get("LAST_EVENT_TYPE")));
        snapshotEvents.add(snapshot);
    } else if (eventType == DataEventType.RELOAD) {
        String targetNodeId = context.getBatch().getTargetNodeId();
        Node targetNode = nodeService.findNode(targetNodeId);
        List<FileTriggerRouter> fileTriggerRouters = fileSyncService.getFileTriggerRoutersForCurrentNode();
        for (FileTriggerRouter fileTriggerRouter : fileTriggerRouters) {
            if (fileTriggerRouter.isEnabled() && fileTriggerRouter.isInitialLoadEnabled() && fileTriggerRouter.getRouter().getNodeGroupLink().getTargetNodeGroupId().equals(targetNode.getNodeGroupId())) {
                DirectorySnapshot directorySnapshot = fileSyncService.getDirectorySnapshot(fileTriggerRouter);
                snapshotEvents.addAll(directorySnapshot);
            }
        }
    }
}