Examples with HetuLocalFileSystemClient io.hetu.core.filesystem.HetuLocalFileSystemClient used on opensource projects

Example 6 with HetuLocalFileSystemClient

use of io.hetu.core.filesystem.HetuLocalFileSystemClient in project hetu-core by openlookeng.

the class TestBinaryFileSpiller method setUp.

@BeforeMethod
public void setUp() throws IOException {
    Metadata metadata = createTestMetadataManager();
    FileSystemClientManager fileSystemClientManager = mock(FileSystemClientManager.class);
    when(fileSystemClientManager.getFileSystemClient(any(Path.class))).thenReturn(new HetuLocalFileSystemClient(new LocalConfig(new Properties()), Paths.get(spillPath.getCanonicalPath())));
    spillerStats = new SpillerStats();
    FeaturesConfig featuresConfig = new FeaturesConfig();
    try {
        featuresConfig.setSpillerSpillPaths(spillPath.getCanonicalPath());
    } catch (IOException e) {
        System.out.println(e.getStackTrace());
    }
    featuresConfig.setSpillMaxUsedSpaceThreshold(1.0);
    NodeSpillConfig nodeSpillConfig = new NodeSpillConfig();
    singleStreamSpillerFactory = new FileSingleStreamSpillerFactory(metadata, spillerStats, featuresConfig, nodeSpillConfig, fileSystemClientManager);
    factory = new GenericSpillerFactory(singleStreamSpillerFactory);
    PagesSerdeFactory pagesSerdeFactory = new PagesSerdeFactory(metadata.getFunctionAndTypeManager().getBlockEncodingSerde(), nodeSpillConfig.isSpillCompressionEnabled());
    pagesSerde = pagesSerdeFactory.createPagesSerde();
    memoryContext = newSimpleAggregatedMemoryContext();
}

Example 7 with HetuLocalFileSystemClient

use of io.hetu.core.filesystem.HetuLocalFileSystemClient in project hetu-core by openlookeng.

the class TestSnapshotFileBasedClient method testSnapshotStateWithJava.

/**
 * Test store, load snapshot state
 * @throws Exception
 */
@Test
public void testSnapshotStateWithJava() throws Exception {
    SnapshotFileBasedClient client = new SnapshotFileBasedClient(new HetuLocalFileSystemClient(new LocalConfig(new Properties()), Paths.get(ROOT_PATH_STR)), Paths.get(ROOT_PATH_STR), new FileSystemClientManager(), null, false, false);
    String queryId = "query1";
    TaskId taskId = new TaskId(queryId, 1, 1);
    SnapshotStateId snapshotStateId = new SnapshotStateId(2, taskId, 10);
    LinkedHashMap<Long, SnapshotResult> map = new LinkedHashMap<>();
    map.put(3L, SnapshotResult.SUCCESSFUL);
    map.put(1L, SnapshotResult.FAILED);
    map.put(5L, SnapshotResult.FAILED_FATAL);
    map.put(8L, SnapshotResult.SUCCESSFUL);
    // Test store and Load
    client.storeState(snapshotStateId, map, null);
    client.loadState(snapshotStateId, null);
    Assert.assertEquals(map, client.loadState(snapshotStateId, null).get());
}

Example 8 with HetuLocalFileSystemClient

use of io.hetu.core.filesystem.HetuLocalFileSystemClient in project hetu-core by openlookeng.

the class TestSnapshotFileBasedClient method testSnapshotResult.

/**
 * Test store, load, and delete snapshot result
 * @throws Exception
 */
@Test
public void testSnapshotResult() throws Exception {
    SnapshotFileBasedClient client = new SnapshotFileBasedClient(new HetuLocalFileSystemClient(new LocalConfig(new Properties()), Paths.get(ROOT_PATH_STR)), Paths.get(ROOT_PATH_STR), new FileSystemClientManager(), null, false, false);
    String queryId = "query1";
    LinkedHashMap<Long, SnapshotInfo> map = new LinkedHashMap<>();
    map.put(3L, SnapshotInfo.withStatus(SnapshotResult.SUCCESSFUL));
    map.put(1L, SnapshotInfo.withStatus(SnapshotResult.FAILED));
    map.put(5L, SnapshotInfo.withStatus(SnapshotResult.FAILED_FATAL));
    map.put(8L, SnapshotInfo.withStatus(SnapshotResult.SUCCESSFUL));
    // Test store and Load
    client.storeSnapshotResult(queryId, map);
    LinkedHashMap<Long, SnapshotInfo> resultMap = (LinkedHashMap<Long, SnapshotInfo>) client.loadSnapshotResult(queryId);
    Assert.assertEquals(map.get(3L).getSnapshotResult(), resultMap.get(3L).getSnapshotResult());
    Assert.assertEquals(map.get(1L).getSnapshotResult(), resultMap.get(1L).getSnapshotResult());
    Assert.assertEquals(map.get(5L).getSnapshotResult(), resultMap.get(5L).getSnapshotResult());
    Assert.assertEquals(map.get(8L).getSnapshotResult(), resultMap.get(8L).getSnapshotResult());
}

Example 9 with HetuLocalFileSystemClient

use of io.hetu.core.filesystem.HetuLocalFileSystemClient in project hetu-core by openlookeng.

the class TestHeuristicIndexClient method testDeleteSelectedColumnsHelper.

@Test
public void testDeleteSelectedColumnsHelper() throws IOException {
    String tableName = "catalog.schema.UT_test";
    try (TempFolder folder = new TempFolder()) {
        folder.create();
        File tableFolder = new File(folder.getRoot().getPath(), tableName);
        assertTrue(tableFolder.mkdir());
        File columnFolder = new File(tableFolder, "test_column");
        assertTrue(columnFolder.mkdirs());
        File indexTypeFolder = new File(columnFolder, "BLOOM");
        assertTrue(indexTypeFolder.mkdirs());
        assertTrue(new File(indexTypeFolder, "testIndex.index").createNewFile());
        HetuFileSystemClient fs = new HetuLocalFileSystemClient(new LocalConfig(new Properties()), folder.getRoot().toPath());
        HetuMetastore testMetaStore = new HetuFsMetastore(new HetuFsMetastoreConfig().setHetuFileSystemMetastorePath(folder.getRoot().getPath()), fs);
        HeuristicIndexClient client = new HeuristicIndexClient(fs, testMetaStore, folder.getRoot().toPath());
        client.addIndexRecord(new CreateIndexMetadata("idx1", tableName, "BLOOM", 0L, Collections.singletonList(new Pair<>("test_column", VARCHAR)), Collections.emptyList(), new Properties(), "user", CreateIndexMetadata.Level.UNDEFINED));
        client.deleteIndex("idx1", Collections.emptyList());
        assertFalse(indexTypeFolder.exists());
    }
}

Example 10 with HetuLocalFileSystemClient

use of io.hetu.core.filesystem.HetuLocalFileSystemClient in project hetu-core by openlookeng.

the class TestHetuMetastoreCacheLocal method setUp.

/**
 * setUp
 *
 * @throws Exception Exception
 */
@BeforeClass
public void setUp() throws Throwable {
    try {
        Map<String, String> config = new ImmutableMap.Builder<String, String>().put("hetu.metastore.hetufilesystem.path", path).put("hetu.metastore.hetufilesystem.profile-name", "local-config-catalog").put("hetu.metastore.cache.size", "10000").put("hetu.metastore.cache.ttl", "10h").build();
        LocalConfig localConfig = new LocalConfig(null);
        client = new HetuLocalFileSystemClient(localConfig, Paths.get(path));
        if (!client.exists(Paths.get(path))) {
            client.createDirectories(Paths.get(path));
        }
        client.deleteRecursively(Paths.get(path));
        StateStore stateStore = createMockStateStore();
        String type = LOCAL;
        Bootstrap app = new Bootstrap(new MBeanModule(), new MBeanServerModule(), new HetuFsMetastoreModule(client, stateStore, type));
        Injector injector = app.strictConfig().doNotInitializeLogging().setRequiredConfigurationProperties(config).initialize();
        metastore = injector.getInstance(HetuMetastore.class);
    } catch (Exception ex) {
        throwIfUnchecked(ex);
        throw new PrestoException(HETU_METASTORE_CODE, "init hetu metastore module failed.");
    }
    // create default catalog and database
    defaultCatalog = CatalogEntity.builder().setCatalogName("hetu1").setOwner("hetu1").build();
    metastore.createCatalog(defaultCatalog);
    defaultDatabase = DatabaseEntity.builder().setCatalogName(defaultCatalog.getName()).setDatabaseName("db1").build();
    metastore.createDatabase(defaultDatabase);
    // get metastore catalog cache
    Field catalogCacheField = metastore.getClass().getSuperclass().getDeclaredField("catalogCache");
    catalogCacheField.setAccessible(true);
    catalogCache = (HetuCache<String, Optional<CatalogEntity>>) catalogCacheField.get(metastore);
    Field catalogsCacheField = metastore.getClass().getSuperclass().getDeclaredField("catalogsCache");
    catalogsCacheField.setAccessible(true);
    catalogsCache = (HetuCache<String, List<CatalogEntity>>) catalogsCacheField.get(metastore);
    // get metastore database cache
    Field databaseCacheField = metastore.getClass().getSuperclass().getDeclaredField("databaseCache");
    databaseCacheField.setAccessible(true);
    databaseCache = (HetuCache<String, Optional<DatabaseEntity>>) databaseCacheField.get(metastore);
    Field databasesCacheField = metastore.getClass().getSuperclass().getDeclaredField("databasesCache");
    databasesCacheField.setAccessible(true);
    databasesCache = (HetuCache<String, List<DatabaseEntity>>) databasesCacheField.get(metastore);
    // get metastore table cache
    Field tableCacheField = metastore.getClass().getSuperclass().getDeclaredField("tableCache");
    tableCacheField.setAccessible(true);
    tableCache = (HetuCache<String, Optional<TableEntity>>) tableCacheField.get(metastore);
    Field tablesCacheField = metastore.getClass().getSuperclass().getDeclaredField("tablesCache");
    tablesCacheField.setAccessible(true);
    tablesCache = (HetuCache<String, List<TableEntity>>) tablesCacheField.get(metastore);
}