Examples with NessieApiV1 org.projectnessie.client.api.NessieApiV1 used on opensource projects

Example 11 with NessieApiV1

use of org.projectnessie.client.api.NessieApiV1 in project nessie by projectnessie.

the class BaseClientAuthTest method api.

protected NessieApiV1 api() {
    if (api != null) {
        return api;
    }
    HttpClientBuilder builder = HttpClientBuilder.builder().withUri("http://localhost:19121/api/v1");
    if (customizer != null) {
        customizer.accept(builder);
    }
    api = builder.build(NessieApiV1.class);
    return api;
}

Example 12 with NessieApiV1

use of org.projectnessie.client.api.NessieApiV1 in project iceberg by apache.

the class BaseTestIceberg method beforeEach.

@BeforeEach
public void beforeEach(@NessieUri URI nessieUri) throws IOException {
    this.uri = nessieUri.toString();
    this.api = HttpClientBuilder.builder().withUri(this.uri).build(NessieApiV1.class);
    resetData();
    try {
        api.createReference().reference(Branch.of(branch, null)).create();
    } catch (Exception e) {
    // ignore, already created. Can't run this in BeforeAll as quarkus hasn't disabled auth
    }
    hadoopConfig = new Configuration();
    catalog = initCatalog(branch);
}

Example 13 with NessieApiV1

use of org.projectnessie.client.api.NessieApiV1 in project nessie by projectnessie.

the class TestTranslatingVersionNessieApi method serialization.

@ParameterizedTest
@MethodSource("serialization")
void serialization(Class<?> modelClass, Object modelObj, String modelJson) throws Exception {
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try (TranslatingVersionNessieApi translating = new TranslatingVersionNessieApi(createOldVersionNessieAPi(), NessieApiV1.class, oldVersionClassLoader)) {
        Object translatedObject = translating.translateObject(modelObj, oldVersionClassLoader, contextClassLoader);
        assertThat(translating.serializeWith(contextClassLoader, modelObj)).isEqualTo(translating.serializeWith(oldVersionClassLoader, translatedObject)).isEqualTo(modelJson);
        assertThat(translating.deserializeWith(contextClassLoader, modelJson, modelClass.getName())).isEqualTo(modelObj);
        assertThat(translating.deserializeWith(oldVersionClassLoader, modelJson, modelClass.getName())).isEqualTo(translatedObject);
        assertThat(translating.reserialize(modelObj)).isEqualTo(translatedObject).extracting(b -> b.getClass().getClassLoader()).isSameAs(oldVersionClassLoader);
        assertThat(translating.reserialize(translatedObject)).isEqualTo(modelObj).extracting(b -> b.getClass().getClassLoader()).isSameAs(contextClassLoader);
    }
}

Example 14 with NessieApiV1

use of org.projectnessie.client.api.NessieApiV1 in project nessie by projectnessie.

the class TestTranslatingVersionNessieApi method translate.

@ParameterizedTest
@MethodSource("translate")
void translate(String expectedClassName, Object modelObj) throws Exception {
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try (TranslatingVersionNessieApi translating = new TranslatingVersionNessieApi(createOldVersionNessieAPi(), NessieApiV1.class, oldVersionClassLoader)) {
        assertThat(translating.translateObject(null, oldVersionClassLoader, contextClassLoader)).isNull();
        assertThat(translating.translateArgs(null, oldVersionClassLoader, contextClassLoader)).isNull();
        assertThat(translating.translateArgs(new Object[] { null, null }, oldVersionClassLoader, contextClassLoader)).containsExactly(null, null);
        Object translatedObject = translating.translateObject(modelObj, oldVersionClassLoader, contextClassLoader);
        assertThat(translatedObject).extracting(Object::getClass).matches(c -> c.getName().equals(expectedClassName)).matches(c -> c.getClassLoader() == oldVersionClassLoader);
        Object[] translatedArgs = translating.translateArgs(new Object[] { null, modelObj, Collections.singletonList(modelObj), Collections.singleton(modelObj), Collections.singletonMap("key", modelObj) }, oldVersionClassLoader, contextClassLoader);
        assertThat(translatedArgs).hasSize(5);
        assertThat(translatedArgs[0]).isNull();
        assertThat(translatedArgs[1]).extracting(Object::getClass).matches(c -> c.getName().equals(expectedClassName)).matches(c -> c.getClassLoader() == oldVersionClassLoader);
        assertThat(translatedArgs[2]).asInstanceOf(InstanceOfAssertFactories.list(Object.class)).hasSize(1).allMatch(o -> o.getClass().getClassLoader() == oldVersionClassLoader);
        assertThat(translatedArgs[3]).asInstanceOf(InstanceOfAssertFactories.collection(Object.class)).hasSize(1).allMatch(o -> o.getClass().getClassLoader() == oldVersionClassLoader);
        assertThat(translatedArgs[4]).asInstanceOf(InstanceOfAssertFactories.map(String.class, Object.class)).hasSize(1).containsKey("key").extractingByKey("key").matches(o -> o.getClass().getClassLoader() == oldVersionClassLoader);
    }
}

Example 15 with NessieApiV1

use of org.projectnessie.client.api.NessieApiV1 in project nessie by projectnessie.

the class GCImpl method identifyExpiredContents.

/**
 * Identify the expired contents using a two-step traversal algorithm.
 *
 * <h2>Algorithm for identifying the live contents and return the bloom filter per content-id</h2>
 *
 * <p>Walk through each reference(both live and dead) distributively (one spark task for each
 * reference).
 *
 * <p>While traversing from the head commit in a reference(use DETACHED reference to fetch commits
 * from dead reference), for each live commit (commit that is not expired based on cutoff time)
 * add the contents of put operation to bloom filter.
 *
 * <p>Collect the live content keys for this reference just before cutoff time (at first expired
 * commit head). Which is used to identify the commit head for each live content key at the time
 * of cutoff time to support the time travel.
 *
 * <p>While traversing the expired commits (commit that is expired based on cutoff time), if it is
 * a head commit content for its key, add it to bloom filter. Else move to next expired commit.
 *
 * <p>Stop traversing the expired commits if each live content key has processed one live commit
 * for it. This is an optimization to avoid traversing all the commits.
 *
 * <p>Collect bloom filter per content id from each task and merge them.
 *
 * <h2>Algorithm for identifying the expired contents and return the list of globally expired
 * contents per content id per reference </h2>
 *
 * <p>Walk through each reference(both live and dead) distributively (one spark task for each
 * reference).
 *
 * <p>For each commit in the reference (use DETACHED reference to fetch commits from dead
 * reference) check it against bloom filter to decide whether its contents in put operation are
 * globally expired or not. If globally expired, Add the contents to the expired output for this
 * content id for this reference.
 *
 * <p>Overall the contents after or equal to cutoff time and the contents that are mapped to
 * commit head of live keys at the time of cutoff timestamp will be retained.
 *
 * @param session spark session for distributed computation
 * @return {@link IdentifiedResult} object having expired contents per content id.
 */
public IdentifiedResult identifyExpiredContents(SparkSession session) {
    try (NessieApiV1 api = GCUtil.getApi(gcParams.getNessieClientConfigs())) {
        DistributedIdentifyContents distributedIdentifyContents = new DistributedIdentifyContents(session, gcParams);
        List<Reference> liveReferences = api.getAllReferences().get().getReferences();
        Map<String, Instant> droppedReferenceTimeMap = collectDeadReferences(api);
        // As this list of references is passed from Spark driver to executor,
        // using available Immutables JSON serialization instead of adding java serialization to the
        // classes.
        List<String> allRefs = liveReferences.stream().map(GCUtil::serializeReference).collect(Collectors.toList());
        if (droppedReferenceTimeMap.size() > 0) {
            allRefs.addAll(droppedReferenceTimeMap.keySet());
        }
        long bloomFilterSize = gcParams.getBloomFilterExpectedEntries() == null ? getTotalCommitsInDefaultReference(api) : gcParams.getBloomFilterExpectedEntries();
        // Identify the live contents and return the bloom filter per content-id
        Map<String, ContentBloomFilter> liveContentsBloomFilterMap = distributedIdentifyContents.getLiveContentsBloomFilters(allRefs, bloomFilterSize, droppedReferenceTimeMap);
        // Identify the expired contents
        return distributedIdentifyContents.getIdentifiedResults(liveContentsBloomFilterMap, allRefs);
    }
}