Examples with TestableCachingCatalog org.apache.iceberg.TestableCachingCatalog used on opensource projects

Example 1 with TestableCachingCatalog

use of org.apache.iceberg.TestableCachingCatalog in project iceberg by apache.

the class TestCachingCatalog method testDeadlock.

@Test
public void testDeadlock() throws IOException, InterruptedException {
    HadoopCatalog underlyingCatalog = hadoopCatalog();
    TestableCachingCatalog catalog = TestableCachingCatalog.wrap(underlyingCatalog, Duration.ofSeconds(1), ticker);
    Namespace namespace = Namespace.of("db", "ns1", "ns2");
    int numThreads = 20;
    List<TableIdentifier> createdTables = Lists.newArrayList();
    for (int i = 0; i < numThreads; i++) {
        TableIdentifier tableIdent = TableIdentifier.of(namespace, "tbl" + i);
        catalog.createTable(tableIdent, SCHEMA, SPEC, ImmutableMap.of("key", "value"));
        createdTables.add(tableIdent);
    }
    Cache<TableIdentifier, Table> cache = catalog.cache();
    AtomicInteger cacheGetCount = new AtomicInteger(0);
    AtomicInteger cacheCleanupCount = new AtomicInteger(0);
    ExecutorService executor = Executors.newFixedThreadPool(numThreads);
    for (int i = 0; i < numThreads; i++) {
        if (i % 2 == 0) {
            String table = "tbl" + i;
            executor.submit(() -> {
                ticker.advance(Duration.ofSeconds(2));
                cache.get(TableIdentifier.of(namespace, table), underlyingCatalog::loadTable);
                cacheGetCount.incrementAndGet();
            });
        } else {
            executor.submit(() -> {
                ticker.advance(Duration.ofSeconds(2));
                cache.cleanUp();
                cacheCleanupCount.incrementAndGet();
            });
        }
    }
    executor.awaitTermination(2, TimeUnit.SECONDS);
    Assertions.assertThat(cacheGetCount).hasValue(numThreads / 2);
    Assertions.assertThat(cacheCleanupCount).hasValue(numThreads / 2);
    executor.shutdown();
    createdTables.forEach(table -> catalog.dropTable(table, true));
}

Example 2 with TestableCachingCatalog

use of org.apache.iceberg.TestableCachingCatalog in project iceberg by apache.

the class TestCachingCatalog method testCatalogExpirationTtlRefreshesAfterAccessViaCatalog.

@Test
public void testCatalogExpirationTtlRefreshesAfterAccessViaCatalog() throws IOException {
    TestableCachingCatalog catalog = TestableCachingCatalog.wrap(hadoopCatalog(), EXPIRATION_TTL, ticker);
    Namespace namespace = Namespace.of("db", "ns1", "ns2");
    TableIdentifier tableIdent = TableIdentifier.of(namespace, "tbl");
    catalog.createTable(tableIdent, SCHEMA, SPEC, ImmutableMap.of("key", "value"));
    Assertions.assertThat(catalog.cache().asMap()).containsKey(tableIdent);
    Assertions.assertThat(catalog.ageOf(tableIdent)).isPresent().get().isEqualTo(Duration.ZERO);
    ticker.advance(HALF_OF_EXPIRATION);
    Assertions.assertThat(catalog.cache().asMap()).containsKey(tableIdent);
    Assertions.assertThat(catalog.ageOf(tableIdent)).isPresent().get().isEqualTo(HALF_OF_EXPIRATION);
    Assertions.assertThat(catalog.remainingAgeFor(tableIdent)).isPresent().get().isEqualTo(HALF_OF_EXPIRATION);
    Duration oneMinute = Duration.ofMinutes(1L);
    ticker.advance(oneMinute);
    Assertions.assertThat(catalog.cache().asMap()).containsKey(tableIdent);
    Assertions.assertThat(catalog.ageOf(tableIdent)).isPresent().get().isEqualTo(HALF_OF_EXPIRATION.plus(oneMinute));
    Assertions.assertThat(catalog.remainingAgeFor(tableIdent)).get().isEqualTo(HALF_OF_EXPIRATION.minus(oneMinute));
    // Access the table via the catalog, which should refresh the TTL
    Table table = catalog.loadTable(tableIdent);
    Assertions.assertThat(catalog.ageOf(tableIdent)).get().isEqualTo(Duration.ZERO);
    Assertions.assertThat(catalog.remainingAgeFor(tableIdent)).get().isEqualTo(EXPIRATION_TTL);
    ticker.advance(HALF_OF_EXPIRATION);
    Assertions.assertThat(catalog.ageOf(tableIdent)).get().isEqualTo(HALF_OF_EXPIRATION);
    Assertions.assertThat(catalog.remainingAgeFor(tableIdent)).get().isEqualTo(HALF_OF_EXPIRATION);
    // Check that accessing the table object directly does not affect the cache TTL
    table.refresh();
    Assertions.assertThat(catalog.ageOf(tableIdent)).get().isEqualTo(HALF_OF_EXPIRATION);
    Assertions.assertThat(catalog.remainingAgeFor(tableIdent)).get().isEqualTo(HALF_OF_EXPIRATION);
    table.newAppend().appendFile(FILE_A).commit();
    Assertions.assertThat(catalog.ageOf(tableIdent)).get().isEqualTo(HALF_OF_EXPIRATION);
    Assertions.assertThat(catalog.remainingAgeFor(tableIdent)).get().isEqualTo(HALF_OF_EXPIRATION);
}

Example 3 with TestableCachingCatalog

use of org.apache.iceberg.TestableCachingCatalog in project iceberg by apache.

the class TestCachingCatalog method testTableExpiresAfterInterval.

@Test
public void testTableExpiresAfterInterval() throws IOException {
    TestableCachingCatalog catalog = TestableCachingCatalog.wrap(hadoopCatalog(), EXPIRATION_TTL, ticker);
    Namespace namespace = Namespace.of("db", "ns1", "ns2");
    TableIdentifier tableIdent = TableIdentifier.of(namespace, "tbl");
    catalog.createTable(tableIdent, SCHEMA, SPEC, ImmutableMap.of("key", "value"));
    // Ensure table is cached with full ttl remaining upon creation
    Assertions.assertThat(catalog.cache().asMap()).containsKey(tableIdent);
    Assertions.assertThat(catalog.remainingAgeFor(tableIdent)).isPresent().get().isEqualTo(EXPIRATION_TTL);
    ticker.advance(HALF_OF_EXPIRATION);
    Assertions.assertThat(catalog.cache().asMap()).containsKey(tableIdent);
    Assertions.assertThat(catalog.ageOf(tableIdent)).isPresent().get().isEqualTo(HALF_OF_EXPIRATION);
    ticker.advance(HALF_OF_EXPIRATION.plus(Duration.ofSeconds(10)));
    Assertions.assertThat(catalog.cache().asMap()).doesNotContainKey(tableIdent);
    Assert.assertNotSame("CachingCatalog should return a new instance after expiration", table, catalog.loadTable(tableIdent));
}

Example 4 with TestableCachingCatalog

use of org.apache.iceberg.TestableCachingCatalog in project iceberg by apache.

the class TestCachingCatalog method testCacheExpirationEagerlyRemovesMetadataTables.

@Test
public void testCacheExpirationEagerlyRemovesMetadataTables() throws IOException {
    TestableCachingCatalog catalog = TestableCachingCatalog.wrap(hadoopCatalog(), EXPIRATION_TTL, ticker);
    Namespace namespace = Namespace.of("db", "ns1", "ns2");
    TableIdentifier tableIdent = TableIdentifier.of(namespace, "tbl");
    Table table = catalog.createTable(tableIdent, SCHEMA, SPEC, ImmutableMap.of("key2", "value2"));
    Assertions.assertThat(catalog.cache().asMap()).containsKey(tableIdent);
    table.newAppend().appendFile(FILE_A).commit();
    Assertions.assertThat(catalog.cache().asMap()).containsKey(tableIdent);
    Assertions.assertThat(catalog.ageOf(tableIdent)).get().isEqualTo(Duration.ZERO);
    ticker.advance(HALF_OF_EXPIRATION);
    Assertions.assertThat(catalog.cache().asMap()).containsKey(tableIdent);
    Assertions.assertThat(catalog.ageOf(tableIdent)).get().isEqualTo(HALF_OF_EXPIRATION);
    // Load the metadata tables for the first time. Their age should be zero as they're new entries.
    Arrays.stream(metadataTables(tableIdent)).forEach(catalog::loadTable);
    Assertions.assertThat(catalog.cache().asMap()).containsKeys(metadataTables(tableIdent));
    Assertions.assertThat(Arrays.stream(metadataTables(tableIdent)).map(catalog::ageOf)).isNotEmpty().allMatch(age -> age.isPresent() && age.get().equals(Duration.ZERO));
    Assert.assertEquals("Loading a non-cached metadata table should refresh the main table's age", Optional.of(EXPIRATION_TTL), catalog.remainingAgeFor(tableIdent));
    // Move time forward and access already cached metadata tables.
    ticker.advance(HALF_OF_EXPIRATION);
    Arrays.stream(metadataTables(tableIdent)).forEach(catalog::loadTable);
    Assertions.assertThat(Arrays.stream(metadataTables(tableIdent)).map(catalog::ageOf)).isNotEmpty().allMatch(age -> age.isPresent() && age.get().equals(Duration.ZERO));
    Assert.assertEquals("Accessing a cached metadata table should not affect the main table's age", Optional.of(HALF_OF_EXPIRATION), catalog.remainingAgeFor(tableIdent));
    // Move time forward so the data table drops.
    ticker.advance(HALF_OF_EXPIRATION);
    Assertions.assertThat(catalog.cache().asMap()).doesNotContainKey(tableIdent);
    Arrays.stream(metadataTables(tableIdent)).forEach(metadataTable -> Assert.assertFalse("When a data table expires, its metadata tables should expire regardless of age", catalog.cache().asMap().containsKey(metadataTable)));
}

Example 5 with TestableCachingCatalog

use of org.apache.iceberg.TestableCachingCatalog in project iceberg by apache.

the class TestCachingCatalog method testCacheExpirationIsDisabledByANegativeValue.

@Test
public void testCacheExpirationIsDisabledByANegativeValue() throws IOException {
    TestableCachingCatalog catalog = TestableCachingCatalog.wrap(hadoopCatalog(), Duration.ofMillis(CatalogProperties.CACHE_EXPIRATION_INTERVAL_MS_OFF), ticker);
    Assert.assertFalse("When a negative value is used as the expiration interval, the cache should not expire entries based on a TTL", catalog.isCacheExpirationEnabled());
}