Examples with RegexFileFilter org.apache.commons.io.filefilter.RegexFileFilter used on opensource projects

Example 1 with RegexFileFilter

use of org.apache.commons.io.filefilter.RegexFileFilter in project robovm by robovm.

the class IOSTarget method copyToIndexedDir.

/**
     * Copies the dSYM and the executable to {@code ~/Library/Developer/Xcode/
     * DerivedData/RoboVM/Build/Products/<appname>_<timestamp>/}.
     */
private void copyToIndexedDir(File dir, String executable, File dsymDir, File exePath) {
    SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
    final File indexedDir = new File(System.getProperty("user.home"), "Library/Developer/Xcode/DerivedData/RoboVM/Build/Products/" + FilenameUtils.removeExtension(dir.getName()) + "_" + sdf.format(new Date()));
    indexedDir.mkdirs();
    File indexedDSymDir = new File(indexedDir, dsymDir.getName());
    File indexedAppDir = new File(indexedDir, dir.getName());
    indexedAppDir.mkdirs();
    try {
        // No need to copy the whole .app folder. Just the exe
        // is enough to make symbolication happy.
        FileUtils.copyFile(exePath, new File(indexedAppDir, executable));
    } catch (IOException e) {
        config.getLogger().error("Failed to copy %s to indexed dir %s: %s", exePath.getAbsolutePath(), indexedAppDir.getAbsolutePath(), e.getMessage());
    }
    try {
        FileUtils.copyDirectory(dsymDir, indexedDSymDir);
    } catch (IOException e) {
        config.getLogger().error("Failed to copy %s to indexed dir %s: %s", dsymDir.getAbsolutePath(), indexedDir.getAbsolutePath(), e.getMessage());
    }
    // Now do some cleanup and delete all but the 3 most recent dirs
    List<File> dirs = new ArrayList<>(Arrays.asList(indexedDir.getParentFile().listFiles((FileFilter) new AndFileFilter(new PrefixFileFilter(FilenameUtils.removeExtension(dir.getName())), new RegexFileFilter(".*_\\d{14}")))));
    Collections.sort(dirs, new Comparator<File>() {

        public int compare(File o1, File o2) {
            return Long.compare(o1.lastModified(), o2.lastModified());
        }
    });
    if (dirs.size() > 3) {
        for (File f : dirs.subList(0, dirs.size() - 3)) {
            try {
                FileUtils.deleteDirectory(f);
            } catch (IOException e) {
                config.getLogger().error("Failed to delete diretcory %s", f.getAbsolutePath(), e.getMessage());
            }
        }
    }
}

Example 2 with RegexFileFilter

use of org.apache.commons.io.filefilter.RegexFileFilter in project geode by apache.

the class IncrementalBackupDUnitTest method testMissingMemberInBaseline.

/**
   * Successful if a member performs a full backup when its backup data is not present in the
   * baseline (for whatever reason). This also tests what happens when a member is offline during
   * the baseline backup.
   * 
   * The test is regarded as successful when all of the missing members oplog files are backed up
   * during an incremental backup. This means that the member peformed a full backup because its
   * oplogs were missing in the baseline.
   */
@Test
public void testMissingMemberInBaseline() throws Exception {
    // Simulate the missing member by forcing a persistent member
    // to go offline.
    final PersistentID missingMember = disconnect(Host.getHost(0).getVM(0), Host.getHost(0).getVM(1));
    /*
     * Perform baseline and make sure that the list of offline disk stores contains our missing
     * member.
     */
    BackupStatus baselineStatus = performBaseline();
    assertBackupStatus(baselineStatus);
    assertNotNull(baselineStatus.getOfflineDiskStores());
    assertEquals(2, baselineStatus.getOfflineDiskStores().size());
    // Find all of the member's oplogs in the missing member's diskstore directory structure
    // (*.crf,*.krf,*.drf)
    Collection<File> missingMemberOplogFiles = FileUtils.listFiles(new File(missingMember.getDirectory()), new RegexFileFilter(OPLOG_REGEX), DirectoryFileFilter.DIRECTORY);
    assertFalse(missingMemberOplogFiles.isEmpty());
    /*
     * Restart our missing member and make sure it is back online and part of the distributed system
     */
    openCache(Host.getHost(0).getVM(0));
    /*
     * After reconnecting make sure the other members agree that the missing member is back online.
     */
    final Set<PersistentID> missingMembers = new HashSet<>();
    Wait.waitForCriterion(new WaitCriterion() {

        @Override
        public boolean done() {
            missingMembers.clear();
            missingMembers.addAll(getMissingMembers(Host.getHost(0).getVM(1)));
            return !missingMembers.contains(missingMember);
        }

        @Override
        public String description() {
            return "[testMissingMemberInBasline] Wait for missing member.";
        }
    }, 10000, 500, false);
    assertEquals(0, missingMembers.size());
    /*
     * Peform incremental and make sure we have no offline disk stores.
     */
    BackupStatus incrementalStatus = performIncremental();
    assertBackupStatus(incrementalStatus);
    assertNotNull(incrementalStatus.getOfflineDiskStores());
    assertEquals(0, incrementalStatus.getOfflineDiskStores().size());
    // Get the missing member's member id which is different from the PersistentID
    String memberId = getMemberId(Host.getHost(0).getVM(0));
    assertNotNull(memberId);
    // Get list of backed up oplog files in the incremental backup for the missing member
    File incrementalMemberDir = getBackupDirForMember(getIncrementalDir(), memberId);
    Collection<File> backupOplogFiles = FileUtils.listFiles(incrementalMemberDir, new RegexFileFilter(OPLOG_REGEX), DirectoryFileFilter.DIRECTORY);
    assertFalse(backupOplogFiles.isEmpty());
    // Transform missing member oplogs to just their file names.
    List<String> missingMemberOplogNames = new LinkedList<>();
    TransformUtils.transform(missingMemberOplogFiles, missingMemberOplogNames, TransformUtils.fileNameTransformer);
    // Transform missing member's incremental backup oplogs to just their file names.
    List<String> backupOplogNames = new LinkedList<>();
    TransformUtils.transform(backupOplogFiles, backupOplogNames, TransformUtils.fileNameTransformer);
    /*
     * Make sure that the incremental backup for the missing member contains all of the operation
     * logs for that member. This proves that a full backup was performed for that member.
     */
    assertTrue(backupOplogNames.containsAll(missingMemberOplogNames));
}

Example 3 with RegexFileFilter

use of org.apache.commons.io.filefilter.RegexFileFilter in project geode by apache.

the class IncrementalBackupDUnitTest method testMissingBaseline.

/**
   * Successful if all members perform a full backup when they share the baseline directory and it
   * is missing.
   */
@Test
public void testMissingBaseline() throws Exception {
    /*
     * Get the member ID for VM 1 and perform a baseline.
     */
    String memberId = getMemberId(Host.getHost(0).getVM(1));
    assertBackupStatus(performBaseline());
    /*
     * Find all of the member's oplogs in the baseline (*.crf,*.krf,*.drf)
     */
    Collection<File> memberBaselineOplogs = FileUtils.listFiles(getBackupDirForMember(getBaselineDir(), memberId), new RegexFileFilter(OPLOG_REGEX), DirectoryFileFilter.DIRECTORY);
    assertFalse(memberBaselineOplogs.isEmpty());
    List<String> memberBaselineOplogNames = new LinkedList<>();
    TransformUtils.transform(memberBaselineOplogs, memberBaselineOplogNames, TransformUtils.fileNameTransformer);
    /*
     * Custom incremental backup callable that retrieves the current baseline before deletion.
     */
    SerializableCallable callable = new SerializableCallable("Backup all members.") {

        private final File baselineDir = getBaselineBackupDir();

        @Override
        public Object call() {
            AdminDistributedSystem adminDS = null;
            try {
                DistributedSystemConfig config = AdminDistributedSystemFactory.defineDistributedSystem(getSystem(), "");
                adminDS = AdminDistributedSystemFactory.getDistributedSystem(config);
                adminDS.connect();
                return adminDS.backupAllMembers(getIncrementalDir(), this.baselineDir);
            } catch (AdminException e) {
                throw new RuntimeException(e);
            } finally {
                if (adminDS != null) {
                    adminDS.disconnect();
                }
            }
        }
    };
    /*
     * Do an incremental after deleting the baseline. It should discover that the baseline is gone
     * and backup all of the operation logs that are in the baseline.
     */
    FileUtils.deleteDirectory(getBaselineDir());
    Host.getHost(0).getVM(1).invoke(callable);
    /*
     * Find all of the member's oplogs in the incremental (*.crf,*.krf,*.drf)
     */
    Collection<File> memberIncrementalOplogs = FileUtils.listFiles(getBackupDirForMember(getIncrementalDir(), memberId), new RegexFileFilter(OPLOG_REGEX), DirectoryFileFilter.DIRECTORY);
    assertFalse(memberIncrementalOplogs.isEmpty());
    List<String> memberIncrementalOplogNames = new LinkedList<>();
    TransformUtils.transform(memberIncrementalOplogs, memberIncrementalOplogNames, TransformUtils.fileNameTransformer);
    /*
     * Assert that all of the baseline operation logs are in the incremental backup. If so, then the
     * missing baseline was discovered by the incremental backup process.
     */
    for (String oplog : memberBaselineOplogNames) {
        assertTrue(memberIncrementalOplogNames.contains(oplog));
    }
}

Example 4 with RegexFileFilter

use of org.apache.commons.io.filefilter.RegexFileFilter in project geode by apache.

the class IncrementalBackupDUnitTest method testIncrementalBackup.

/**
   * This tests the basic features of incremental backup. This means that operation logs that are
   * present in both the baseline and member's disk store should not be copied during the
   * incremental backup. Additionally, the restore script should reference and copy operation logs
   * from the baseline backup.
   */
@Test
public void testIncrementalBackup() throws Exception {
    String memberId = getMemberId(Host.getHost(0).getVM(1));
    File memberDir = getVMDir(Host.getHost(0).getVM(1));
    assertNotNull(memberDir);
    // Find all of the member's oplogs in the disk directory (*.crf,*.krf,*.drf)
    Collection<File> memberOplogFiles = FileUtils.listFiles(memberDir, new RegexFileFilter(OPLOG_REGEX), DirectoryFileFilter.DIRECTORY);
    assertFalse(memberOplogFiles.isEmpty());
    // Perform a full backup and wait for it to finish
    assertBackupStatus(performBaseline());
    waitForBackup(Host.getHost(0).getVM(1));
    // Find all of the member's oplogs in the baseline (*.crf,*.krf,*.drf)
    Collection<File> memberBaselineOplogs = FileUtils.listFiles(getBackupDirForMember(getBaselineDir(), memberId), new RegexFileFilter(OPLOG_REGEX), DirectoryFileFilter.DIRECTORY);
    assertFalse(memberBaselineOplogs.isEmpty());
    List<String> memberBaselineOplogNames = new LinkedList<>();
    TransformUtils.transform(memberBaselineOplogs, memberBaselineOplogNames, TransformUtils.fileNameTransformer);
    // Peform and incremental backup and wait for it to finish
    // Doing this preserves the new oplogs created by the baseline backup
    loadMoreData();
    assertBackupStatus(performIncremental());
    waitForBackup(Host.getHost(0).getVM(1));
    // Find all of the member's oplogs in the incremental (*.crf,*.krf,*.drf)
    Collection<File> memberIncrementalOplogs = FileUtils.listFiles(getBackupDirForMember(getIncrementalDir(), memberId), new RegexFileFilter(OPLOG_REGEX), DirectoryFileFilter.DIRECTORY);
    assertFalse(memberIncrementalOplogs.isEmpty());
    List<String> memberIncrementalOplogNames = new LinkedList<>();
    TransformUtils.transform(memberIncrementalOplogs, memberIncrementalOplogNames, TransformUtils.fileNameTransformer);
    log("BASELINE OPLOGS = " + memberBaselineOplogNames);
    log("INCREMENTAL OPLOGS = " + memberIncrementalOplogNames);
    /*
     * Assert that the incremental backup does not contain baseline operation logs that the member
     * still has copies of.
     */
    for (String oplog : memberBaselineOplogNames) {
        assertFalse(memberIncrementalOplogNames.contains(oplog));
    }
    // Perform a second incremental and wait for it to finish.
    // Doing this preserves the new oplogs created by the incremental backup
    loadMoreData();
    assertBackupStatus(performIncremental2());
    waitForBackup(Host.getHost(0).getVM(1));
    Collection<File> memberIncremental2Oplogs = FileUtils.listFiles(getBackupDirForMember(getIncremental2Dir(), memberId), new RegexFileFilter(OPLOG_REGEX), DirectoryFileFilter.DIRECTORY);
    assertFalse(memberIncremental2Oplogs.isEmpty());
    List<String> memberIncremental2OplogNames = new LinkedList<>();
    TransformUtils.transform(memberIncremental2Oplogs, memberIncremental2OplogNames, TransformUtils.fileNameTransformer);
    log("INCREMENTAL 2 OPLOGS = " + memberIncremental2OplogNames);
    /*
     * Assert that the second incremental backup does not contain operation logs copied into the
     * baseline.
     */
    for (String oplog : memberBaselineOplogNames) {
        assertFalse(memberIncremental2OplogNames.contains(oplog));
    }
    /*
     * Also assert that the second incremental backup does not contain operation logs copied into
     * the member's first incremental backup.
     */
    for (String oplog : memberIncrementalOplogNames) {
        assertFalse(memberIncremental2OplogNames.contains(oplog));
    }
    // Shut down our member so we can perform a restore
    PersistentID id = getPersistentID(Host.getHost(0).getVM(1));
    closeCache(Host.getHost(0).getVM(1));
    // Execute the restore
    performRestore(new File(id.getDirectory()), getBackupDirForMember(getIncremental2Dir(), memberId));
    /*
     * Collect all of the restored operation logs.
     */
    Collection<File> restoredOplogs = FileUtils.listFiles(new File(id.getDirectory()), new RegexFileFilter(OPLOG_REGEX), DirectoryFileFilter.DIRECTORY);
    assertFalse(restoredOplogs.isEmpty());
    List<String> restoredOplogNames = new LinkedList<>();
    TransformUtils.transform(restoredOplogs, restoredOplogNames, TransformUtils.fileNameTransformer);
    /*
     * Assert that baseline operation logs have been copied over to the member's disk directory.
     */
    for (String oplog : memberBaselineOplogNames) {
        assertTrue(restoredOplogNames.contains(oplog));
    }
    /*
     * Assert that the incremental operation logs have been copied over to the member's disk
     * directory.
     */
    for (String oplog : memberIncrementalOplogNames) {
        assertTrue(restoredOplogNames.contains(oplog));
    }
    /*
     * Assert that the second incremental operation logs have been copied over to the member's disk
     * directory.
     */
    for (String oplog : memberIncremental2OplogNames) {
        assertTrue(restoredOplogNames.contains(oplog));
    }
    /*
     * Reconnect the member.
     */
    openCache(Host.getHost(0).getVM(1));
}

Example 5 with RegexFileFilter

use of org.apache.commons.io.filefilter.RegexFileFilter in project lucene-solr by apache.

the class ZkCLITest method testUpConfigLinkConfigClearZk.

@Test
public void testUpConfigLinkConfigClearZk() throws Exception {
    File tmpDir = createTempDir().toFile();
    // test upconfig
    String confsetname = "confsetone";
    final String[] upconfigArgs;
    if (random().nextBoolean()) {
        upconfigArgs = new String[] { "-zkhost", zkServer.getZkAddress(), "-cmd", ZkCLI.UPCONFIG, "-confdir", ExternalPaths.TECHPRODUCTS_CONFIGSET, "-confname", confsetname };
    } else {
        final String excluderegexOption = (random().nextBoolean() ? "--" + ZkCLI.EXCLUDE_REGEX : "-" + ZkCLI.EXCLUDE_REGEX_SHORT);
        upconfigArgs = new String[] { "-zkhost", zkServer.getZkAddress(), "-cmd", ZkCLI.UPCONFIG, excluderegexOption, ZkCLI.EXCLUDE_REGEX_DEFAULT, "-confdir", ExternalPaths.TECHPRODUCTS_CONFIGSET, "-confname", confsetname };
    }
    ZkCLI.main(upconfigArgs);
    assertTrue(zkClient.exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + confsetname, true));
    // print help
    // ZkCLI.main(new String[0]);
    // test linkconfig
    String[] args = new String[] { "-zkhost", zkServer.getZkAddress(), "-cmd", "linkconfig", "-collection", "collection1", "-confname", confsetname };
    ZkCLI.main(args);
    ZkNodeProps collectionProps = ZkNodeProps.load(zkClient.getData(ZkStateReader.COLLECTIONS_ZKNODE + "/collection1", null, null, true));
    assertTrue(collectionProps.containsKey("configName"));
    assertEquals(confsetname, collectionProps.getStr("configName"));
    // test down config
    File confDir = new File(tmpDir, "solrtest-confdropspot-" + this.getClass().getName() + "-" + System.nanoTime());
    assertFalse(confDir.exists());
    args = new String[] { "-zkhost", zkServer.getZkAddress(), "-cmd", "downconfig", "-confdir", confDir.getAbsolutePath(), "-confname", confsetname };
    ZkCLI.main(args);
    File[] files = confDir.listFiles();
    List<String> zkFiles = zkClient.getChildren(ZkConfigManager.CONFIGS_ZKNODE + "/" + confsetname, null, true);
    assertEquals(files.length, zkFiles.size());
    File sourceConfDir = new File(ExternalPaths.TECHPRODUCTS_CONFIGSET);
    // filter out all directories starting with . (e.g. .svn)
    Collection<File> sourceFiles = FileUtils.listFiles(sourceConfDir, TrueFileFilter.INSTANCE, new RegexFileFilter("[^\\.].*"));
    for (File sourceFile : sourceFiles) {
        int indexOfRelativePath = sourceFile.getAbsolutePath().lastIndexOf("sample_techproducts_configs" + File.separator + "conf");
        String relativePathofFile = sourceFile.getAbsolutePath().substring(indexOfRelativePath + 33, sourceFile.getAbsolutePath().length());
        File downloadedFile = new File(confDir, relativePathofFile);
        if (ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_PATTERN.matcher(relativePathofFile).matches()) {
            assertFalse(sourceFile.getAbsolutePath() + " exists in ZK, downloaded:" + downloadedFile.getAbsolutePath(), downloadedFile.exists());
        } else {
            assertTrue(downloadedFile.getAbsolutePath() + " does not exist source:" + sourceFile.getAbsolutePath(), downloadedFile.exists());
            assertTrue(relativePathofFile + " content changed", FileUtils.contentEquals(sourceFile, downloadedFile));
        }
    }
    // test reset zk
    args = new String[] { "-zkhost", zkServer.getZkAddress(), "-cmd", "clear", "/" };
    ZkCLI.main(args);
    assertEquals(0, zkClient.getChildren("/", null, true).size());
}