Examples with TempFolder io.hetu.core.common.filesystem.TempFolder used on opensource projects

Example 16 with TempFolder

use of io.hetu.core.common.filesystem.TempFolder in project hetu-core by openlookeng.

the class TestMinMaxIndex method testPersistHelper.

void testPersistHelper(Comparable min, Comparable max) throws IOException {
    MinMaxIndex index = new MinMaxIndex(min, max);
    try (TempFolder folder = new TempFolder()) {
        folder.create();
        File file = folder.newFile();
        String path = file.getCanonicalPath() + ".minmax";
        assertTrue(file.renameTo(new File(path)));
        file.deleteOnExit();
        if (!file.getParentFile().exists()) {
            assertTrue(file.getParentFile().mkdirs());
        }
        try (OutputStream os = new FileOutputStream(file)) {
            index.serialize(os);
        }
        assertTrue(file.isFile());
        MinMaxIndex loadedIndex = new MinMaxIndex();
        try (InputStream is = new FileInputStream(file)) {
            loadedIndex.deserialize(is);
        }
        assertEquals(index, loadedIndex);
    }
}

Example 17 with TempFolder

use of io.hetu.core.common.filesystem.TempFolder in project hetu-core by openlookeng.

the class HindexQueryRunner method createQueryRunner.

public static DistributedQueryRunner createQueryRunner() throws Exception {
    // Use this to return if no exceptions
    DistributedQueryRunner queryRunner;
    // this means we have to specify the coordinator port
    for (int i = 0; i < 10; i++) {
        int port = -1;
        try {
            TempFolder testFolder = new TempFolder().create();
            Runtime.getRuntime().addShutdownHook(new Thread(() -> testFolder.close()));
            port = SslSocketUtil.getAvailablePort();
            Map<String, String> configs = new HashMap<>();
            configs.put("http-server.http.port", Integer.toString(port));
            configs.put("hetu.heuristicindex.filter.enabled", "true");
            configs.put("hetu.heuristicindex.filter.cache.max-memory", "5GB");
            configs.put("hetu.heuristicindex.filter.cache.loading-delay", "0ms");
            configs.put("hetu.heuristicindex.filter.cache.autoload-default", "false");
            configs.put("hetu.heuristicindex.indexstore.uri", testFolder.newFolder("indexstore").getAbsolutePath());
            configs.put("hetu.heuristicindex.indexstore.filesystem.profile", "__test__hdfs__");
            Map<String, String> metastoreConfig = new HashMap<>();
            metastoreConfig.put("hetu.metastore.type", "hetufilesystem");
            metastoreConfig.put("hetu.metastore.hetufilesystem.profile-name", "default");
            metastoreConfig.put("hetu.metastore.hetufilesystem.path", testFolder.newFolder("metastore").getAbsolutePath());
            queryRunner = createQueryRunner(configs, metastoreConfig, Collections.emptyMap());
            return queryRunner;
        } catch (Exception portException) {
            System.out.printf("Failed to start testing server with port %d. Retrying.%n", port);
        }
    }
    throw new Exception("Reached maximum attempt to start testing server.");
}

Example 18 with TempFolder

use of io.hetu.core.common.filesystem.TempFolder in project hetu-core by openlookeng.

the class TestIndexServiceUtils method testArchive.

@Test
public void testArchive() throws IOException {
    try (TempFolder folder = new TempFolder()) {
        folder.create();
        Path tarPath = Paths.get(folder.getRoot().getAbsolutePath(), IndexConstants.LAST_MODIFIED_FILE_PREFIX + "123456.tar");
        folder.newFile("100000.bloom");
        folder.newFile("200000.bloom");
        IndexServiceUtils.writeToHdfs(LOCAL_FS_CLIENT, LOCAL_FS_CLIENT, folder.getRoot().toPath(), tarPath);
        assertTrue(Files.exists(tarPath));
        Set<String> filesInTar = new HashSet<>();
        try (TarArchiveInputStream i = new TarArchiveInputStream(Files.newInputStream(tarPath))) {
            ArchiveEntry entry;
            while ((entry = i.getNextEntry()) != null) {
                if (!i.canReadEntryData(entry)) {
                    throw new FileSystemException("Unable to read archive entry: " + entry.toString());
                }
                String filename = entry.getName();
                filesInTar.add(filename);
            }
        }
        assertEquals(filesInTar.size(), 2);
        assertTrue(filesInTar.contains("100000.bloom"));
        assertTrue(filesInTar.contains("200000.bloom"));
    }
}

Example 19 with TempFolder

use of io.hetu.core.common.filesystem.TempFolder in project hetu-core by openlookeng.

the class TestBitmapIndex method testLargeEntries.

@Test
public void testLargeEntries() throws IOException {
    try (TempFolder folder = new TempFolder();
        BitmapIndex bitmapIndexWrite = new BitmapIndex();
        BitmapIndex bitmapIndexRead = new BitmapIndex()) {
        folder.create();
        File file = folder.newFile();
        String columnName = "column";
        Long[] values = new Long[10000000];
        for (int i = 0; i < values.length; i++) {
            values[i] = Long.valueOf(i);
        }
        List<Object> columnValues = Arrays.asList(values);
        bitmapIndexWrite.setExpectedNumOfEntries(columnValues.size());
        bitmapIndexWrite.addValues(Collections.singletonList(new Pair<>(columnName, columnValues)));
        try (FileOutputStream os = new FileOutputStream(file);
            FileInputStream is = new FileInputStream(file)) {
            bitmapIndexWrite.serialize(os);
            bitmapIndexRead.deserialize(is);
        }
        assertEquals(iteratorToList(bitmapIndexRead.lookUp(Domain.create(ValueSet.ofRanges(equal(IntegerType.INTEGER, 3L)), false))), ImmutableList.of(3));
        assertEquals(iteratorToList(bitmapIndexRead.lookUp(Domain.create(ValueSet.ofRanges(equal(IntegerType.INTEGER, 2048L)), false))), ImmutableList.of(2048));
        assertEquals(iteratorToList(bitmapIndexRead.lookUp(Domain.create(ValueSet.ofRanges(equal(IntegerType.INTEGER, Long.valueOf(values.length))), false))), ImmutableList.of());
    }
}

Example 20 with TempFolder

use of io.hetu.core.common.filesystem.TempFolder in project hetu-core by openlookeng.

the class TestBloomIndex method testLoadEmpty.

@Test
public void testLoadEmpty() throws IOException {
    try (TempFolder folder = new TempFolder();
        BloomIndex objectBloomIndex = new BloomIndex()) {
        folder.create();
        File testFile = folder.newFile();
        try (InputStream is = new FileInputStream(testFile)) {
            assertThrows(IOException.class, () -> objectBloomIndex.deserialize(is));
        }
    }
}