Search in sources :

Example 6 with HetuLocalFileSystemClient

use of io.hetu.core.filesystem.HetuLocalFileSystemClient in project hetu-core by openlookeng.

the class TestBinaryFileSpiller method setUp.

@BeforeMethod
public void setUp() throws IOException {
    Metadata metadata = createTestMetadataManager();
    FileSystemClientManager fileSystemClientManager = mock(FileSystemClientManager.class);
    when(fileSystemClientManager.getFileSystemClient(any(Path.class))).thenReturn(new HetuLocalFileSystemClient(new LocalConfig(new Properties()), Paths.get(spillPath.getCanonicalPath())));
    spillerStats = new SpillerStats();
    FeaturesConfig featuresConfig = new FeaturesConfig();
    try {
        featuresConfig.setSpillerSpillPaths(spillPath.getCanonicalPath());
    } catch (IOException e) {
        System.out.println(e.getStackTrace());
    }
    featuresConfig.setSpillMaxUsedSpaceThreshold(1.0);
    NodeSpillConfig nodeSpillConfig = new NodeSpillConfig();
    singleStreamSpillerFactory = new FileSingleStreamSpillerFactory(metadata, spillerStats, featuresConfig, nodeSpillConfig, fileSystemClientManager);
    factory = new GenericSpillerFactory(singleStreamSpillerFactory);
    PagesSerdeFactory pagesSerdeFactory = new PagesSerdeFactory(metadata.getFunctionAndTypeManager().getBlockEncodingSerde(), nodeSpillConfig.isSpillCompressionEnabled());
    pagesSerde = pagesSerdeFactory.createPagesSerde();
    memoryContext = newSimpleAggregatedMemoryContext();
}
Also used : Path(java.nio.file.Path) LocalConfig(io.hetu.core.filesystem.LocalConfig) FeaturesConfig(io.prestosql.sql.analyzer.FeaturesConfig) Metadata(io.prestosql.metadata.Metadata) IOException(java.io.IOException) Properties(java.util.Properties) FileSystemClientManager(io.prestosql.filesystem.FileSystemClientManager) TestingPagesSerdeFactory(io.prestosql.testing.TestingPagesSerdeFactory) PagesSerdeFactory(io.hetu.core.transport.execution.buffer.PagesSerdeFactory) HetuLocalFileSystemClient(io.hetu.core.filesystem.HetuLocalFileSystemClient) BeforeMethod(org.testng.annotations.BeforeMethod)

Example 7 with HetuLocalFileSystemClient

use of io.hetu.core.filesystem.HetuLocalFileSystemClient in project hetu-core by openlookeng.

the class TestSnapshotFileBasedClient method testSnapshotStateWithJava.

/**
 * Test store, load snapshot state
 * @throws Exception
 */
@Test
public void testSnapshotStateWithJava() throws Exception {
    SnapshotFileBasedClient client = new SnapshotFileBasedClient(new HetuLocalFileSystemClient(new LocalConfig(new Properties()), Paths.get(ROOT_PATH_STR)), Paths.get(ROOT_PATH_STR), new FileSystemClientManager(), null, false, false);
    String queryId = "query1";
    TaskId taskId = new TaskId(queryId, 1, 1);
    SnapshotStateId snapshotStateId = new SnapshotStateId(2, taskId, 10);
    LinkedHashMap<Long, SnapshotResult> map = new LinkedHashMap<>();
    map.put(3L, SnapshotResult.SUCCESSFUL);
    map.put(1L, SnapshotResult.FAILED);
    map.put(5L, SnapshotResult.FAILED_FATAL);
    map.put(8L, SnapshotResult.SUCCESSFUL);
    // Test store and Load
    client.storeState(snapshotStateId, map, null);
    client.loadState(snapshotStateId, null);
    Assert.assertEquals(map, client.loadState(snapshotStateId, null).get());
}
Also used : TaskId(io.prestosql.execution.TaskId) LocalConfig(io.hetu.core.filesystem.LocalConfig) Properties(java.util.Properties) HetuLocalFileSystemClient(io.hetu.core.filesystem.HetuLocalFileSystemClient) FileSystemClientManager(io.prestosql.filesystem.FileSystemClientManager) LinkedHashMap(java.util.LinkedHashMap) Test(org.testng.annotations.Test)

Example 8 with HetuLocalFileSystemClient

use of io.hetu.core.filesystem.HetuLocalFileSystemClient in project hetu-core by openlookeng.

the class TestSnapshotFileBasedClient method testSnapshotResult.

/**
 * Test store, load, and delete snapshot result
 * @throws Exception
 */
@Test
public void testSnapshotResult() throws Exception {
    SnapshotFileBasedClient client = new SnapshotFileBasedClient(new HetuLocalFileSystemClient(new LocalConfig(new Properties()), Paths.get(ROOT_PATH_STR)), Paths.get(ROOT_PATH_STR), new FileSystemClientManager(), null, false, false);
    String queryId = "query1";
    LinkedHashMap<Long, SnapshotInfo> map = new LinkedHashMap<>();
    map.put(3L, SnapshotInfo.withStatus(SnapshotResult.SUCCESSFUL));
    map.put(1L, SnapshotInfo.withStatus(SnapshotResult.FAILED));
    map.put(5L, SnapshotInfo.withStatus(SnapshotResult.FAILED_FATAL));
    map.put(8L, SnapshotInfo.withStatus(SnapshotResult.SUCCESSFUL));
    // Test store and Load
    client.storeSnapshotResult(queryId, map);
    LinkedHashMap<Long, SnapshotInfo> resultMap = (LinkedHashMap<Long, SnapshotInfo>) client.loadSnapshotResult(queryId);
    Assert.assertEquals(map.get(3L).getSnapshotResult(), resultMap.get(3L).getSnapshotResult());
    Assert.assertEquals(map.get(1L).getSnapshotResult(), resultMap.get(1L).getSnapshotResult());
    Assert.assertEquals(map.get(5L).getSnapshotResult(), resultMap.get(5L).getSnapshotResult());
    Assert.assertEquals(map.get(8L).getSnapshotResult(), resultMap.get(8L).getSnapshotResult());
}
Also used : LocalConfig(io.hetu.core.filesystem.LocalConfig) Properties(java.util.Properties) HetuLocalFileSystemClient(io.hetu.core.filesystem.HetuLocalFileSystemClient) FileSystemClientManager(io.prestosql.filesystem.FileSystemClientManager) LinkedHashMap(java.util.LinkedHashMap) Test(org.testng.annotations.Test)

Example 9 with HetuLocalFileSystemClient

use of io.hetu.core.filesystem.HetuLocalFileSystemClient in project hetu-core by openlookeng.

the class TestHeuristicIndexClient method testDeleteSelectedColumnsHelper.

@Test
public void testDeleteSelectedColumnsHelper() throws IOException {
    String tableName = "catalog.schema.UT_test";
    try (TempFolder folder = new TempFolder()) {
        folder.create();
        File tableFolder = new File(folder.getRoot().getPath(), tableName);
        assertTrue(tableFolder.mkdir());
        File columnFolder = new File(tableFolder, "test_column");
        assertTrue(columnFolder.mkdirs());
        File indexTypeFolder = new File(columnFolder, "BLOOM");
        assertTrue(indexTypeFolder.mkdirs());
        assertTrue(new File(indexTypeFolder, "testIndex.index").createNewFile());
        HetuFileSystemClient fs = new HetuLocalFileSystemClient(new LocalConfig(new Properties()), folder.getRoot().toPath());
        HetuMetastore testMetaStore = new HetuFsMetastore(new HetuFsMetastoreConfig().setHetuFileSystemMetastorePath(folder.getRoot().getPath()), fs);
        HeuristicIndexClient client = new HeuristicIndexClient(fs, testMetaStore, folder.getRoot().toPath());
        client.addIndexRecord(new CreateIndexMetadata("idx1", tableName, "BLOOM", 0L, Collections.singletonList(new Pair<>("test_column", VARCHAR)), Collections.emptyList(), new Properties(), "user", CreateIndexMetadata.Level.UNDEFINED));
        client.deleteIndex("idx1", Collections.emptyList());
        assertFalse(indexTypeFolder.exists());
    }
}
Also used : CreateIndexMetadata(io.prestosql.spi.connector.CreateIndexMetadata) HetuFsMetastoreConfig(io.hetu.core.metastore.hetufilesystem.HetuFsMetastoreConfig) LocalConfig(io.hetu.core.filesystem.LocalConfig) Properties(java.util.Properties) HetuFsMetastore(io.hetu.core.metastore.hetufilesystem.HetuFsMetastore) HetuMetastore(io.prestosql.spi.metastore.HetuMetastore) HetuFileSystemClient(io.prestosql.spi.filesystem.HetuFileSystemClient) TempFolder(io.hetu.core.common.filesystem.TempFolder) File(java.io.File) HetuLocalFileSystemClient(io.hetu.core.filesystem.HetuLocalFileSystemClient) Test(org.testng.annotations.Test)

Example 10 with HetuLocalFileSystemClient

use of io.hetu.core.filesystem.HetuLocalFileSystemClient in project hetu-core by openlookeng.

the class TestHetuMetastoreCacheLocal method setUp.

/**
 * setUp
 *
 * @throws Exception Exception
 */
@BeforeClass
public void setUp() throws Throwable {
    try {
        Map<String, String> config = new ImmutableMap.Builder<String, String>().put("hetu.metastore.hetufilesystem.path", path).put("hetu.metastore.hetufilesystem.profile-name", "local-config-catalog").put("hetu.metastore.cache.size", "10000").put("hetu.metastore.cache.ttl", "10h").build();
        LocalConfig localConfig = new LocalConfig(null);
        client = new HetuLocalFileSystemClient(localConfig, Paths.get(path));
        if (!client.exists(Paths.get(path))) {
            client.createDirectories(Paths.get(path));
        }
        client.deleteRecursively(Paths.get(path));
        StateStore stateStore = createMockStateStore();
        String type = LOCAL;
        Bootstrap app = new Bootstrap(new MBeanModule(), new MBeanServerModule(), new HetuFsMetastoreModule(client, stateStore, type));
        Injector injector = app.strictConfig().doNotInitializeLogging().setRequiredConfigurationProperties(config).initialize();
        metastore = injector.getInstance(HetuMetastore.class);
    } catch (Exception ex) {
        throwIfUnchecked(ex);
        throw new PrestoException(HETU_METASTORE_CODE, "init hetu metastore module failed.");
    }
    // create default catalog and database
    defaultCatalog = CatalogEntity.builder().setCatalogName("hetu1").setOwner("hetu1").build();
    metastore.createCatalog(defaultCatalog);
    defaultDatabase = DatabaseEntity.builder().setCatalogName(defaultCatalog.getName()).setDatabaseName("db1").build();
    metastore.createDatabase(defaultDatabase);
    // get metastore catalog cache
    Field catalogCacheField = metastore.getClass().getSuperclass().getDeclaredField("catalogCache");
    catalogCacheField.setAccessible(true);
    catalogCache = (HetuCache<String, Optional<CatalogEntity>>) catalogCacheField.get(metastore);
    Field catalogsCacheField = metastore.getClass().getSuperclass().getDeclaredField("catalogsCache");
    catalogsCacheField.setAccessible(true);
    catalogsCache = (HetuCache<String, List<CatalogEntity>>) catalogsCacheField.get(metastore);
    // get metastore database cache
    Field databaseCacheField = metastore.getClass().getSuperclass().getDeclaredField("databaseCache");
    databaseCacheField.setAccessible(true);
    databaseCache = (HetuCache<String, Optional<DatabaseEntity>>) databaseCacheField.get(metastore);
    Field databasesCacheField = metastore.getClass().getSuperclass().getDeclaredField("databasesCache");
    databasesCacheField.setAccessible(true);
    databasesCache = (HetuCache<String, List<DatabaseEntity>>) databasesCacheField.get(metastore);
    // get metastore table cache
    Field tableCacheField = metastore.getClass().getSuperclass().getDeclaredField("tableCache");
    tableCacheField.setAccessible(true);
    tableCache = (HetuCache<String, Optional<TableEntity>>) tableCacheField.get(metastore);
    Field tablesCacheField = metastore.getClass().getSuperclass().getDeclaredField("tablesCache");
    tablesCacheField.setAccessible(true);
    tablesCache = (HetuCache<String, List<TableEntity>>) tablesCacheField.get(metastore);
}
Also used : MBeanModule(org.weakref.jmx.guice.MBeanModule) Optional(java.util.Optional) LocalConfig(io.hetu.core.filesystem.LocalConfig) StateStore(io.prestosql.spi.statestore.StateStore) PrestoException(io.prestosql.spi.PrestoException) ImmutableMap(com.google.common.collect.ImmutableMap) PrestoException(io.prestosql.spi.PrestoException) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) IOException(java.io.IOException) HetuMetastore(io.prestosql.spi.metastore.HetuMetastore) Field(java.lang.reflect.Field) MBeanServerModule(io.prestosql.plugin.base.jmx.MBeanServerModule) Injector(com.google.inject.Injector) Bootstrap(io.airlift.bootstrap.Bootstrap) List(java.util.List) HetuLocalFileSystemClient(io.hetu.core.filesystem.HetuLocalFileSystemClient) HetuFsMetastoreModule(io.hetu.core.metastore.hetufilesystem.HetuFsMetastoreModule) BeforeClass(org.testng.annotations.BeforeClass)

Aggregations

HetuLocalFileSystemClient (io.hetu.core.filesystem.HetuLocalFileSystemClient)19 LocalConfig (io.hetu.core.filesystem.LocalConfig)19 Properties (java.util.Properties)15 BeforeMethod (org.testng.annotations.BeforeMethod)8 Path (java.nio.file.Path)7 FileSystemClientManager (io.prestosql.filesystem.FileSystemClientManager)6 BeforeClass (org.testng.annotations.BeforeClass)5 Test (org.testng.annotations.Test)5 HetuMetastore (io.prestosql.spi.metastore.HetuMetastore)4 IOException (java.io.IOException)4 ImmutableMap (com.google.common.collect.ImmutableMap)3 Injector (com.google.inject.Injector)3 Bootstrap (io.airlift.bootstrap.Bootstrap)3 MBeanServerModule (io.prestosql.plugin.base.jmx.MBeanServerModule)3 PrestoException (io.prestosql.spi.PrestoException)3 StateStore (io.prestosql.spi.statestore.StateStore)3 MBeanModule (org.weakref.jmx.guice.MBeanModule)3 JsonProcessingException (com.fasterxml.jackson.core.JsonProcessingException)2 TempFolder (io.hetu.core.common.filesystem.TempFolder)2 HetuFsMetastore (io.hetu.core.metastore.hetufilesystem.HetuFsMetastore)2