Examples with FileTrigger org.jumpmind.symmetric.model.FileTrigger used on opensource projects

Example 1 with FileTrigger

use of org.jumpmind.symmetric.model.FileTrigger in project symmetric-ds by JumpMind.

the class AbstractFileParsingRouter method deleteFileIfNecessary.

public void deleteFileIfNecessary(DataMetaData dataMetaData) {
    Data data = dataMetaData.getData();
    Table snapshotTable = dataMetaData.getTable();
    if (data.getDataEventType() == DataEventType.INSERT || data.getDataEventType() == DataEventType.UPDATE) {
        List<File> filesToDelete = new ArrayList<File>();
        Map<String, String> columnData = data.toColumnNameValuePairs(snapshotTable.getColumnNames(), CsvData.ROW_DATA);
        FileSnapshot fileSnapshot = new FileSnapshot();
        fileSnapshot.setTriggerId(columnData.get("TRIGGER_ID"));
        fileSnapshot.setRouterId(columnData.get("ROUTER_ID"));
        fileSnapshot.setFileModifiedTime(Long.parseLong(columnData.get("FILE_MODIFIED_TIME")));
        fileSnapshot.setFileName(columnData.get("FILE_NAME"));
        fileSnapshot.setRelativeDir(columnData.get("RELATIVE_DIR"));
        fileSnapshot.setLastEventType(LastEventType.fromCode(columnData.get("LAST_EVENT_TYPE")));
        FileTriggerRouter triggerRouter = getEngine().getFileSyncService().getFileTriggerRouter(fileSnapshot.getTriggerId(), fileSnapshot.getRouterId());
        if (triggerRouter != null) {
            FileTrigger fileTrigger = triggerRouter.getFileTrigger();
            if (fileTrigger.isDeleteAfterSync()) {
                File file = fileTrigger.createSourceFile(fileSnapshot);
                if (!file.isDirectory()) {
                    filesToDelete.add(file);
                    if (fileTrigger.isSyncOnCtlFile()) {
                        File ctlFile = getEngine().getFileSyncService().getControleFile(file);
                        filesToDelete.add(ctlFile);
                    }
                }
            } else if (getEngine().getParameterService().is(ParameterConstants.FILE_SYNC_DELETE_CTL_FILE_AFTER_SYNC, false)) {
                File file = fileTrigger.createSourceFile(fileSnapshot);
                if (!file.isDirectory()) {
                    if (fileTrigger.isSyncOnCtlFile()) {
                        File ctlFile = getEngine().getFileSyncService().getControleFile(file);
                        filesToDelete.add(ctlFile);
                    }
                }
            }
        }
        if (filesToDelete != null && filesToDelete.size() > 0) {
            for (File file : filesToDelete) {
                if (file != null && file.exists()) {
                    log.debug("Deleting the '{}' file", file.getAbsolutePath());
                    boolean deleted = FileUtils.deleteQuietly(file);
                    if (!deleted) {
                        log.warn("Failed to 'delete on sync' the {} file", file.getAbsolutePath());
                    }
                }
                file = null;
            }
            filesToDelete = null;
        }
    }
}

Example 2 with FileTrigger

use of org.jumpmind.symmetric.model.FileTrigger in project symmetric-ds by JumpMind.

the class FileSyncService method acknowledgeFiles.

public void acknowledgeFiles(OutgoingBatch outgoingBatch) {
    log.debug("Acknowledging file_sync outgoing batch-{}", outgoingBatch.getBatchId());
    ISqlReadCursor<Data> cursor = engine.getDataService().selectDataFor(outgoingBatch.getBatchId(), outgoingBatch.getChannelId());
    Data data = null;
    List<File> filesToDelete = new ArrayList<File>();
    Table snapshotTable = platform.getTableFromCache(TableConstants.getTableName(tablePrefix, TableConstants.SYM_FILE_SNAPSHOT), false);
    for (int i = 0; i < outgoingBatch.getInsertEventCount(); i++) {
        data = cursor.next();
        if (data != null && (data.getDataEventType() == DataEventType.INSERT || data.getDataEventType() == DataEventType.UPDATE)) {
            Map<String, String> columnData = data.toColumnNameValuePairs(snapshotTable.getColumnNames(), CsvData.ROW_DATA);
            FileSnapshot fileSnapshot = new FileSnapshot();
            fileSnapshot.setTriggerId(columnData.get("TRIGGER_ID"));
            fileSnapshot.setRouterId(columnData.get("ROUTER_ID"));
            fileSnapshot.setFileModifiedTime(Long.parseLong(columnData.get("FILE_MODIFIED_TIME")));
            fileSnapshot.setFileName(columnData.get("FILE_NAME"));
            fileSnapshot.setRelativeDir(columnData.get("RELATIVE_DIR"));
            fileSnapshot.setLastEventType(LastEventType.fromCode(columnData.get("LAST_EVENT_TYPE")));
            FileTriggerRouter triggerRouter = this.getFileTriggerRouter(fileSnapshot.getTriggerId(), fileSnapshot.getRouterId());
            if (triggerRouter != null) {
                FileTrigger fileTrigger = triggerRouter.getFileTrigger();
                if (fileTrigger.isDeleteAfterSync()) {
                    File file = fileTrigger.createSourceFile(fileSnapshot);
                    if (!file.isDirectory()) {
                        filesToDelete.add(file);
                        if (fileTrigger.isSyncOnCtlFile()) {
                            filesToDelete.add(new File(file.getAbsolutePath() + ".ctl"));
                        }
                    }
                }
            }
        }
    }
    if (cursor != null) {
        cursor.close();
        cursor = null;
    }
    if (filesToDelete != null && filesToDelete.size() > 0) {
        for (File file : filesToDelete) {
            if (file != null && file.exists()) {
                log.debug("Deleting the '{}' file", file.getAbsolutePath());
                boolean deleted = FileUtils.deleteQuietly(file);
                if (!deleted) {
                    log.warn("Failed to 'delete on sync' the {} file", file.getAbsolutePath());
                }
            }
            file = null;
        }
        filesToDelete = null;
    }
}

Example 3 with FileTrigger

use of org.jumpmind.symmetric.model.FileTrigger in project symmetric-ds by JumpMind.

the class FileTriggerTrackerTest method testTakeSnapshotRecursiveTestDelete.

@Test
public void testTakeSnapshotRecursiveTestDelete() throws Exception {
    FileTrigger fileTrigger = new FileTrigger(directory.getAbsolutePath(), true, null, null);
    Router router = new Router();
    FileTriggerRouter fileTriggerRouter = new FileTriggerRouter(fileTrigger, router);
    FileTriggerTracker tracker = new FileTriggerTracker(fileTriggerRouter, null);
    tracker.trackChanges();
    FileUtils.deleteQuietly(fileInDirectory1);
    DirectorySnapshot snapshot = tracker.trackChanges();
    assertEquals(1, snapshot.size());
    FileSnapshot change = snapshot.get(0);
    assertEquals(change.getFileName(), FileSyncUtils.getRelativePath(fileInDirectory1, directory));
    assertEquals(change.getLastEventType(), LastEventType.DELETE);
}

Example 4 with FileTrigger

use of org.jumpmind.symmetric.model.FileTrigger in project symmetric-ds by JumpMind.

the class FileSyncZipDataWriter method end.

public void end(Batch batch, boolean inError) {
    try {
        if (!inError) {
            if (zos == null) {
                zos = new ZipOutputStream(stagedResource.getOutputStream());
            }
            Map<String, LastEventType> entries = new HashMap<String, LastEventType>();
            StringBuilder script = new StringBuilder("fileList = new HashMap();\n");
            for (FileSnapshot snapshot : snapshotEvents) {
                FileTriggerRouter triggerRouter = fileSyncService.getFileTriggerRouter(snapshot.getTriggerId(), snapshot.getRouterId());
                if (triggerRouter != null) {
                    StringBuilder command = new StringBuilder("\n");
                    LastEventType eventType = snapshot.getLastEventType();
                    FileTrigger fileTrigger = triggerRouter.getFileTrigger();
                    String targetBaseDir = ((triggerRouter.getTargetBaseDir() == null) ? null : triggerRouter.getTargetBaseDir().replace('\\', '/'));
                    if (StringUtils.isBlank(targetBaseDir)) {
                        targetBaseDir = ((fileTrigger.getBaseDir() == null) ? null : fileTrigger.getBaseDir().replace('\\', '/'));
                    }
                    targetBaseDir = StringEscapeUtils.escapeJava(targetBaseDir);
                    command.append("targetBaseDir = \"").append(targetBaseDir).append("\";\n");
                    command.append("processFile = true;\n");
                    command.append("sourceFileName = \"").append(snapshot.getFileName()).append("\";\n");
                    command.append("targetRelativeDir = \"");
                    if (!snapshot.getRelativeDir().equals(".")) {
                        command.append(StringEscapeUtils.escapeJava(snapshot.getRelativeDir()));
                        command.append("\";\n");
                    } else {
                        command.append("\";\n");
                    }
                    command.append("targetFileName = sourceFileName;\n");
                    command.append("sourceFilePath = \"");
                    command.append(StringEscapeUtils.escapeJava(snapshot.getRelativeDir())).append("\";\n");
                    StringBuilder entryName = new StringBuilder(Long.toString(batch.getBatchId()));
                    entryName.append("/");
                    if (!snapshot.getRelativeDir().equals(".")) {
                        entryName.append(snapshot.getRelativeDir()).append("/");
                    }
                    entryName.append(snapshot.getFileName());
                    File file = fileTrigger.createSourceFile(snapshot);
                    if (file.isDirectory()) {
                        entryName.append("/");
                    }
                    if (StringUtils.isNotBlank(fileTrigger.getBeforeCopyScript())) {
                        command.append(fileTrigger.getBeforeCopyScript()).append("\n");
                    }
                    command.append("if (processFile) {\n");
                    String targetFile = "targetBaseDir + \"/\" + targetRelativeDir + \"/\" + targetFileName";
                    switch(eventType) {
                        case CREATE:
                        case MODIFY:
                            if (file.exists()) {
                                command.append("  File targetBaseDirFile = new File(targetBaseDir);\n");
                                command.append("  if (!targetBaseDirFile.exists()) {\n");
                                command.append("    targetBaseDirFile.mkdirs();\n");
                                command.append("  }\n");
                                command.append("  java.io.File sourceFile = new java.io.File(batchDir + \"/\"");
                                if (!snapshot.getRelativeDir().equals(".")) {
                                    command.append(" + sourceFilePath + \"/\"");
                                }
                                command.append(" + sourceFileName");
                                command.append(");\n");
                                command.append("  java.io.File targetFile = new java.io.File(");
                                command.append(targetFile);
                                command.append(");\n");
                                // no need to copy directory if it already exists
                                command.append("  if (targetFile.exists() && targetFile.isDirectory()) {\n");
                                command.append("      processFile = false;\n");
                                command.append("  }\n");
                                // conflict resolution
                                FileConflictStrategy conflictStrategy = triggerRouter.getConflictStrategy();
                                if (conflictStrategy == FileConflictStrategy.TARGET_WINS || conflictStrategy == FileConflictStrategy.MANUAL) {
                                    command.append("  if (targetFile.exists() && !targetFile.isDirectory()) {\n");
                                    command.append("    long targetChecksum = org.apache.commons.io.FileUtils.checksumCRC32(targetFile);\n");
                                    command.append("    if (targetChecksum != " + snapshot.getOldCrc32Checksum() + "L) {\n");
                                    if (conflictStrategy == FileConflictStrategy.MANUAL) {
                                        command.append("      throw new org.jumpmind.symmetric.file.FileConflictException(targetFileName + \" was in conflict \");\n");
                                    } else {
                                        command.append("      processFile = false;\n");
                                    }
                                    command.append("    }\n");
                                    command.append("  }\n");
                                }
                                command.append("  if (processFile) {\n");
                                command.append("    if (sourceFile.isDirectory()) {\n");
                                command.append("      org.apache.commons.io.FileUtils.copyDirectory(sourceFile, targetFile, true);\n");
                                command.append("    } else {\n");
                                command.append("      org.apache.commons.io.FileUtils.copyFile(sourceFile, targetFile, true);\n");
                                command.append("    }\n");
                                command.append("  }\n");
                                command.append("  fileList.put(").append(targetFile).append(",\"");
                                command.append(eventType.getCode());
                                command.append("\");\n");
                            }
                            break;
                        case DELETE:
                            command.append("  org.apache.commons.io.FileUtils.deleteQuietly(new java.io.File(");
                            command.append(targetFile);
                            command.append("));\n");
                            command.append("  fileList.put(").append(targetFile).append(",\"");
                            command.append(eventType.getCode());
                            command.append("\");\n");
                            break;
                        default:
                            break;
                    }
                    if (StringUtils.isNotBlank(fileTrigger.getAfterCopyScript())) {
                        command.append(fileTrigger.getAfterCopyScript()).append("\n");
                    }
                    LastEventType previousEventForEntry = entries.get(entryName.toString());
                    boolean process = true;
                    if (previousEventForEntry != null) {
                        if ((previousEventForEntry == eventType) || (previousEventForEntry == LastEventType.CREATE && eventType == LastEventType.MODIFY)) {
                            process = false;
                        }
                    }
                    if (process) {
                        if (eventType != LastEventType.DELETE) {
                            if (file.exists()) {
                                byteCount += file.length();
                                ZipEntry entry = new ZipEntry(entryName.toString());
                                entry.setSize(file.length());
                                entry.setTime(file.lastModified());
                                zos.putNextEntry(entry);
                                if (file.isFile()) {
                                    FileInputStream fis = new FileInputStream(file);
                                    try {
                                        IOUtils.copy(fis, zos);
                                    } finally {
                                        IOUtils.closeQuietly(fis);
                                    }
                                }
                                zos.closeEntry();
                                entries.put(entryName.toString(), eventType);
                            } else {
                                log.warn("Could not find the {} file to package for synchronization.  Skipping it.", file.getAbsolutePath());
                            }
                        }
                        command.append("}\n\n");
                        script.append(command.toString());
                    }
                } else {
                    log.error("Could not locate the file trigger ({}) router ({}) to process a snapshot event.  The event will be ignored", snapshot.getTriggerId(), snapshot.getRouterId());
                }
            }
            script.append("return fileList;\n");
            ZipEntry entry = new ZipEntry(batch.getBatchId() + "/sync.bsh");
            zos.putNextEntry(entry);
            IOUtils.write(script.toString(), zos);
            zos.closeEntry();
            entry = new ZipEntry(batch.getBatchId() + "/batch-info.txt");
            zos.putNextEntry(entry);
            IOUtils.write(batch.getChannelId(), zos);
            zos.closeEntry();
        }
    } catch (IOException e) {
        throw new IoException(e);
    }
}

Example 5 with FileTrigger

use of org.jumpmind.symmetric.model.FileTrigger in project symmetric-ds by JumpMind.

the class FileTriggerTrackerTest method testTakeFullSnapshotIncludes.

@Test
public void testTakeFullSnapshotIncludes() throws Exception {
    FileTrigger fileTrigger = new FileTrigger(directory.getAbsolutePath(), false, "*.txt", null);
    Router router = new Router();
    FileTriggerRouter fileTriggerRouter = new FileTriggerRouter(fileTrigger, router);
    FileTriggerTracker tracker = new FileTriggerTracker(fileTriggerRouter, null);
    DirectorySnapshot snapshot = new DirectorySnapshot(fileTriggerRouter);
    tracker.takeFullSnapshot(snapshot);
    assertEquals(1, snapshot.size());
    assertEquals(snapshot.get(0).getFileName(), FileSyncUtils.getRelativePath(fileInDirectory1, directory));
}