use of io.hetu.core.filesystem.LocalConfig in project hetu-core by openlookeng.
the class TestBinaryFileSpiller method setUp.
@BeforeMethod
public void setUp() throws IOException {
Metadata metadata = createTestMetadataManager();
FileSystemClientManager fileSystemClientManager = mock(FileSystemClientManager.class);
when(fileSystemClientManager.getFileSystemClient(any(Path.class))).thenReturn(new HetuLocalFileSystemClient(new LocalConfig(new Properties()), Paths.get(spillPath.getCanonicalPath())));
spillerStats = new SpillerStats();
FeaturesConfig featuresConfig = new FeaturesConfig();
try {
featuresConfig.setSpillerSpillPaths(spillPath.getCanonicalPath());
} catch (IOException e) {
System.out.println(e.getStackTrace());
}
featuresConfig.setSpillMaxUsedSpaceThreshold(1.0);
NodeSpillConfig nodeSpillConfig = new NodeSpillConfig();
singleStreamSpillerFactory = new FileSingleStreamSpillerFactory(metadata, spillerStats, featuresConfig, nodeSpillConfig, fileSystemClientManager);
factory = new GenericSpillerFactory(singleStreamSpillerFactory);
PagesSerdeFactory pagesSerdeFactory = new PagesSerdeFactory(metadata.getFunctionAndTypeManager().getBlockEncodingSerde(), nodeSpillConfig.isSpillCompressionEnabled());
pagesSerde = pagesSerdeFactory.createPagesSerde();
memoryContext = newSimpleAggregatedMemoryContext();
}
use of io.hetu.core.filesystem.LocalConfig in project hetu-core by openlookeng.
the class TestSnapshotFileBasedClient method testSnapshotStateWithJava.
/**
* Test store, load snapshot state
* @throws Exception
*/
@Test
public void testSnapshotStateWithJava() throws Exception {
SnapshotFileBasedClient client = new SnapshotFileBasedClient(new HetuLocalFileSystemClient(new LocalConfig(new Properties()), Paths.get(ROOT_PATH_STR)), Paths.get(ROOT_PATH_STR), new FileSystemClientManager(), null, false, false);
String queryId = "query1";
TaskId taskId = new TaskId(queryId, 1, 1);
SnapshotStateId snapshotStateId = new SnapshotStateId(2, taskId, 10);
LinkedHashMap<Long, SnapshotResult> map = new LinkedHashMap<>();
map.put(3L, SnapshotResult.SUCCESSFUL);
map.put(1L, SnapshotResult.FAILED);
map.put(5L, SnapshotResult.FAILED_FATAL);
map.put(8L, SnapshotResult.SUCCESSFUL);
// Test store and Load
client.storeState(snapshotStateId, map, null);
client.loadState(snapshotStateId, null);
Assert.assertEquals(map, client.loadState(snapshotStateId, null).get());
}
use of io.hetu.core.filesystem.LocalConfig in project hetu-core by openlookeng.
the class TestSnapshotFileBasedClient method testSnapshotResult.
/**
* Test store, load, and delete snapshot result
* @throws Exception
*/
@Test
public void testSnapshotResult() throws Exception {
SnapshotFileBasedClient client = new SnapshotFileBasedClient(new HetuLocalFileSystemClient(new LocalConfig(new Properties()), Paths.get(ROOT_PATH_STR)), Paths.get(ROOT_PATH_STR), new FileSystemClientManager(), null, false, false);
String queryId = "query1";
LinkedHashMap<Long, SnapshotInfo> map = new LinkedHashMap<>();
map.put(3L, SnapshotInfo.withStatus(SnapshotResult.SUCCESSFUL));
map.put(1L, SnapshotInfo.withStatus(SnapshotResult.FAILED));
map.put(5L, SnapshotInfo.withStatus(SnapshotResult.FAILED_FATAL));
map.put(8L, SnapshotInfo.withStatus(SnapshotResult.SUCCESSFUL));
// Test store and Load
client.storeSnapshotResult(queryId, map);
LinkedHashMap<Long, SnapshotInfo> resultMap = (LinkedHashMap<Long, SnapshotInfo>) client.loadSnapshotResult(queryId);
Assert.assertEquals(map.get(3L).getSnapshotResult(), resultMap.get(3L).getSnapshotResult());
Assert.assertEquals(map.get(1L).getSnapshotResult(), resultMap.get(1L).getSnapshotResult());
Assert.assertEquals(map.get(5L).getSnapshotResult(), resultMap.get(5L).getSnapshotResult());
Assert.assertEquals(map.get(8L).getSnapshotResult(), resultMap.get(8L).getSnapshotResult());
}
use of io.hetu.core.filesystem.LocalConfig in project hetu-core by openlookeng.
the class TestHeuristicIndexClient method testDeleteSelectedColumnsHelper.
@Test
public void testDeleteSelectedColumnsHelper() throws IOException {
String tableName = "catalog.schema.UT_test";
try (TempFolder folder = new TempFolder()) {
folder.create();
File tableFolder = new File(folder.getRoot().getPath(), tableName);
assertTrue(tableFolder.mkdir());
File columnFolder = new File(tableFolder, "test_column");
assertTrue(columnFolder.mkdirs());
File indexTypeFolder = new File(columnFolder, "BLOOM");
assertTrue(indexTypeFolder.mkdirs());
assertTrue(new File(indexTypeFolder, "testIndex.index").createNewFile());
HetuFileSystemClient fs = new HetuLocalFileSystemClient(new LocalConfig(new Properties()), folder.getRoot().toPath());
HetuMetastore testMetaStore = new HetuFsMetastore(new HetuFsMetastoreConfig().setHetuFileSystemMetastorePath(folder.getRoot().getPath()), fs);
HeuristicIndexClient client = new HeuristicIndexClient(fs, testMetaStore, folder.getRoot().toPath());
client.addIndexRecord(new CreateIndexMetadata("idx1", tableName, "BLOOM", 0L, Collections.singletonList(new Pair<>("test_column", VARCHAR)), Collections.emptyList(), new Properties(), "user", CreateIndexMetadata.Level.UNDEFINED));
client.deleteIndex("idx1", Collections.emptyList());
assertFalse(indexTypeFolder.exists());
}
}
use of io.hetu.core.filesystem.LocalConfig in project hetu-core by openlookeng.
the class TestHetuMetastoreCacheLocal method setUp.
/**
* setUp
*
* @throws Exception Exception
*/
@BeforeClass
public void setUp() throws Throwable {
try {
Map<String, String> config = new ImmutableMap.Builder<String, String>().put("hetu.metastore.hetufilesystem.path", path).put("hetu.metastore.hetufilesystem.profile-name", "local-config-catalog").put("hetu.metastore.cache.size", "10000").put("hetu.metastore.cache.ttl", "10h").build();
LocalConfig localConfig = new LocalConfig(null);
client = new HetuLocalFileSystemClient(localConfig, Paths.get(path));
if (!client.exists(Paths.get(path))) {
client.createDirectories(Paths.get(path));
}
client.deleteRecursively(Paths.get(path));
StateStore stateStore = createMockStateStore();
String type = LOCAL;
Bootstrap app = new Bootstrap(new MBeanModule(), new MBeanServerModule(), new HetuFsMetastoreModule(client, stateStore, type));
Injector injector = app.strictConfig().doNotInitializeLogging().setRequiredConfigurationProperties(config).initialize();
metastore = injector.getInstance(HetuMetastore.class);
} catch (Exception ex) {
throwIfUnchecked(ex);
throw new PrestoException(HETU_METASTORE_CODE, "init hetu metastore module failed.");
}
// create default catalog and database
defaultCatalog = CatalogEntity.builder().setCatalogName("hetu1").setOwner("hetu1").build();
metastore.createCatalog(defaultCatalog);
defaultDatabase = DatabaseEntity.builder().setCatalogName(defaultCatalog.getName()).setDatabaseName("db1").build();
metastore.createDatabase(defaultDatabase);
// get metastore catalog cache
Field catalogCacheField = metastore.getClass().getSuperclass().getDeclaredField("catalogCache");
catalogCacheField.setAccessible(true);
catalogCache = (HetuCache<String, Optional<CatalogEntity>>) catalogCacheField.get(metastore);
Field catalogsCacheField = metastore.getClass().getSuperclass().getDeclaredField("catalogsCache");
catalogsCacheField.setAccessible(true);
catalogsCache = (HetuCache<String, List<CatalogEntity>>) catalogsCacheField.get(metastore);
// get metastore database cache
Field databaseCacheField = metastore.getClass().getSuperclass().getDeclaredField("databaseCache");
databaseCacheField.setAccessible(true);
databaseCache = (HetuCache<String, Optional<DatabaseEntity>>) databaseCacheField.get(metastore);
Field databasesCacheField = metastore.getClass().getSuperclass().getDeclaredField("databasesCache");
databasesCacheField.setAccessible(true);
databasesCache = (HetuCache<String, List<DatabaseEntity>>) databasesCacheField.get(metastore);
// get metastore table cache
Field tableCacheField = metastore.getClass().getSuperclass().getDeclaredField("tableCache");
tableCacheField.setAccessible(true);
tableCache = (HetuCache<String, Optional<TableEntity>>) tableCacheField.get(metastore);
Field tablesCacheField = metastore.getClass().getSuperclass().getDeclaredField("tablesCache");
tablesCacheField.setAccessible(true);
tablesCache = (HetuCache<String, List<TableEntity>>) tablesCacheField.get(metastore);
}
Aggregations