Examples with LocalConfig io.hetu.core.filesystem.LocalConfig used on opensource projects

Example 11 with LocalConfig

use of io.hetu.core.filesystem.LocalConfig in project hetu-core by openlookeng.

the class TestHetuFsMetastore method setUp.

/**
 * setUp
 *
 * @throws Exception Exception
 */
@BeforeClass
public void setUp() {
    try {
        Map<String, String> config = new ImmutableMap.Builder<String, String>().put("hetu.metastore.hetufilesystem.path", path).put("hetu.metastore.hetufilesystem.profile-name", "local-config-catalog").build();
        LocalConfig localConfig = new LocalConfig(null);
        client = new HetuLocalFileSystemClient(localConfig, Paths.get(path));
        if (!client.exists(Paths.get(path))) {
            client.createDirectories(Paths.get(path));
        }
        client.deleteRecursively(Paths.get(path));
        String type = LOCAL;
        Bootstrap app = new Bootstrap(new MBeanModule(), new MBeanServerModule(), new HetuFsMetastoreModule(client, type));
        Injector injector = app.strictConfig().doNotInitializeLogging().setRequiredConfigurationProperties(config).initialize();
        metastore = injector.getInstance(HetuMetastore.class);
    } catch (Exception ex) {
        throwIfUnchecked(ex);
        throw new PrestoException(HETU_METASTORE_CODE, "init hetu metastore module failed.");
    }
    defaultCatalog = CatalogEntity.builder().setCatalogName("hetu").setOwner("hetu1").build();
    metastore.createCatalog(defaultCatalog);
    defaultDatabase = DatabaseEntity.builder().setCatalogName(defaultCatalog.getName()).setDatabaseName("db1").build();
    metastore.createDatabase(defaultDatabase);
}

Example 12 with LocalConfig

use of io.hetu.core.filesystem.LocalConfig in project hetu-core by openlookeng.

the class TestFileSingleStreamSpiller method prepare.

@BeforeClass
public void prepare() throws IOException {
    fs = getLocalHdfs();
    if (fs.exists(Paths.get(rootPath))) {
        fs.delete(Paths.get(rootPath));
    }
    fs.createDirectories(Paths.get(rootPath));
    when(fileSystemClientManager.getFileSystemClient(any(Path.class))).thenReturn(new HetuLocalFileSystemClient(new LocalConfig(new Properties()), Paths.get(spillPath.getCanonicalPath())));
    when(fileSystemClientManager.getFileSystemClient(any(String.class), any(Path.class))).thenReturn(fs);
}

Example 13 with LocalConfig

use of io.hetu.core.filesystem.LocalConfig in project hetu-core by openlookeng.

the class TestFileSingleStreamSpiller method assertSpillBenchmark.

private void assertSpillBenchmark(boolean compression, boolean encryption, String pageSize, int fileCount, boolean useKryo, boolean useDirectSerde, boolean spillToHdfs, String spillProfile) throws Exception {
    List<FileSingleStreamSpiller> spillers = new ArrayList<>();
    when(fileSystemClientManager.getFileSystemClient(any(Path.class))).thenReturn(new HetuLocalFileSystemClient(new LocalConfig(new Properties()), Paths.get(spillPath.getCanonicalPath())));
    FileSingleStreamSpillerFactory spillerFactory = new FileSingleStreamSpillerFactory(// executor won't be closed, because we don't call destroy() on the spiller factory
    executorBenchmark, (useKryo) ? createTestMetadataManager().getFunctionAndTypeManager().getBlockKryoEncodingSerde() : createTestMetadataManager().getFunctionAndTypeManager().getBlockEncodingSerde(), new SpillerStats(), ImmutableList.of(spillPath.toPath()), 1.0, compression, encryption, useDirectSerde, 1, useKryo, spillToHdfs, spillProfile, fileSystemClientManager);
    LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext("test");
    long startTime = System.currentTimeMillis();
    Stopwatch spillTimer = Stopwatch.createStarted();
    long numberOfPages = pageSize.equals("1GB") ? 262144 : 512;
    Page page = buildPageBenchmark();
    Path finalSpillPath = spillToHdfs ? spillerFactory.getSpillPaths().get(0) : spillPath.toPath();
    for (int j = 1; j <= fileCount; j++) {
        SingleStreamSpiller singleStreamSpiller = spillerFactory.create(TYPES, bytes -> {
        }, memoryContext);
        assertTrue(singleStreamSpiller instanceof FileSingleStreamSpiller);
        FileSingleStreamSpiller spiller = (FileSingleStreamSpiller) singleStreamSpiller;
        spillers.add(spiller);
        // The spillers will reserve memory in their constructors
        assertEquals(memoryContext.getBytes(), 4096);
        Iterator<Page> pageIterator = new Iterator<Page>() {

            private final long pageCount = numberOfPages;

            private long counter = 1;

            @Override
            public boolean hasNext() {
                return counter <= pageCount;
            }

            @Override
            public Page next() {
                if (counter > pageCount) {
                    throw new RuntimeException();
                }
                counter++;
                return page;
            }
        };
        spiller.spill(pageIterator).get();
        assertEquals(listFiles(finalSpillPath).stream().filter(path -> path.toString().endsWith(".bin")).count(), j);
    }
    spillTimer.stop();
    System.out.println("Time To Spill: " + spillTimer.elapsed(TimeUnit.MILLISECONDS) + " ms Traditional Timer: " + (System.currentTimeMillis() - startTime));
    log.debug("TimeTakenToSpill = " + (System.currentTimeMillis() - startTime));
    startTime = System.currentTimeMillis();
    spillTimer.reset();
    spillTimer.start();
    for (int i = 0; i < fileCount; i++) {
        Iterator<Page> spilledPagesIterator = spillers.get(i).getSpilledPages();
        assertEquals(memoryContext.getBytes(), FileSingleStreamSpiller.BUFFER_SIZE);
        spilledPagesIterator.forEachRemaining(page1 -> page1.getLoadedPage());
    }
    spillTimer.stop();
    log.debug("TimeTakenReadingFromSpill = " + (System.currentTimeMillis() - startTime));
    System.out.println("Time To Unspill: " + spillTimer.elapsed(TimeUnit.MILLISECONDS) + " ms, Traditional Timer: " + (System.currentTimeMillis() - startTime));
    spillers.stream().forEach(spiller -> spiller.close());
    assertEquals(listFiles(finalSpillPath).stream().filter(path -> path.toString().endsWith(".bin")).count(), 0);
    assertEquals(memoryContext.getBytes(), 0);
}

Example 14 with LocalConfig

use of io.hetu.core.filesystem.LocalConfig in project hetu-core by openlookeng.

the class TestQuerySnapshotManager method setup.

@BeforeMethod
public void setup() throws Exception {
    // Set up snapshot config
    snapshotConfig = new SnapshotConfig();
    // Set up mock file system client manager
    fileSystemClientManager = mock(FileSystemClientManager.class);
    when(fileSystemClientManager.getFileSystemClient(any(Path.class))).thenReturn(new HetuLocalFileSystemClient(new LocalConfig(new Properties()), Paths.get(SNAPSHOT_FILE_SYSTEM_DIR)));
    snapshotUtils = new SnapshotUtils(fileSystemClientManager, snapshotConfig, new InMemoryNodeManager());
    snapshotUtils.rootPath = SNAPSHOT_FILE_SYSTEM_DIR;
    snapshotUtils.initialize();
}

Example 15 with LocalConfig

use of io.hetu.core.filesystem.LocalConfig in project hetu-core by openlookeng.

the class TestSnapshotFileBasedClient method testSnapshotStateWithKryo.

/**
 * Test store, load snapshot state
 * @throws Exception
 */
@Test
public void testSnapshotStateWithKryo() throws Exception {
    SnapshotFileBasedClient client = new SnapshotFileBasedClient(new HetuLocalFileSystemClient(new LocalConfig(new Properties()), Paths.get(ROOT_PATH_STR)), Paths.get(ROOT_PATH_STR), new FileSystemClientManager(), null, false, true);
    String queryId = "query1";
    TaskId taskId = new TaskId(queryId, 1, 1);
    SnapshotStateId snapshotStateId = new SnapshotStateId(2, taskId, 10);
    LinkedHashMap<Long, SnapshotResult> map = new LinkedHashMap<>();
    map.put(3L, SnapshotResult.SUCCESSFUL);
    map.put(1L, SnapshotResult.FAILED);
    map.put(5L, SnapshotResult.FAILED_FATAL);
    map.put(8L, SnapshotResult.SUCCESSFUL);
    // Test store and Load
    client.storeState(snapshotStateId, map, null);
    client.loadState(snapshotStateId, null);
    Assert.assertEquals(map, client.loadState(snapshotStateId, null).get());
}