Examples with Content org.projectnessie.model.Content used on opensource projects

Example 16 with Content

use of org.projectnessie.model.Content in project nessie by projectnessie.

the class IdentifyContentsPerExecutor method handleCommitForExpiredContents.

private static void handleCommitForExpiredContents(Reference reference, LogResponse.LogEntry logEntry, Map<String, ContentBloomFilter> liveContentsBloomFilterMap, IdentifiedResult result) {
    if (logEntry.getOperations() != null) {
        logEntry.getOperations().stream().filter(operation -> operation instanceof Operation.Put).forEach(operation -> {
            Content content = ((Operation.Put) operation).getContent();
            ContentBloomFilter bloomFilter = liveContentsBloomFilterMap.get(content.getId());
            // But live contents never be considered as expired.
            if (bloomFilter == null || !bloomFilter.mightContain(content)) {
                result.addContent(reference.getName(), content);
            }
        });
    }
}

Example 17 with Content

use of org.projectnessie.model.Content in project nessie by projectnessie.

the class IdentifyContentsPerExecutor method handleLiveCommit.

private void handleLiveCommit(GCStateParamsPerTask gcStateParamsPerTask, LogResponse.LogEntry logEntry, Map<String, ContentBloomFilter> bloomFilterMap, MutableBoolean foundAllLiveCommitHeadsBeforeCutoffTime, Set<ContentKey> liveContentKeys) {
    if (logEntry.getOperations() != null) {
        boolean isExpired = !gcStateParamsPerTask.getLiveCommitPredicate().test(logEntry.getCommitMeta());
        if (isExpired && liveContentKeys.isEmpty()) {
            // as it is the first expired commit. Time travel is supported till this state.
            try {
                gcStateParamsPerTask.getApi().getEntries().refName(Detached.REF_NAME).hashOnRef(logEntry.getCommitMeta().getHash()).get().getEntries().forEach(entries -> liveContentKeys.add(entries.getName()));
            } catch (NessieNotFoundException e) {
                throw new RuntimeException(e);
            }
        }
        logEntry.getOperations().stream().filter(operation -> operation instanceof Operation.Put).forEach(operation -> {
            boolean addContent;
            if (liveContentKeys.contains(operation.getKey())) {
                // commit head of this key
                addContent = true;
                liveContentKeys.remove(operation.getKey());
                if (liveContentKeys.isEmpty()) {
                    // found all the live commit heads before cutoff time.
                    foundAllLiveCommitHeadsBeforeCutoffTime.setTrue();
                }
            } else {
                addContent = !isExpired;
            }
            if (addContent) {
                Content content = ((Operation.Put) operation).getContent();
                bloomFilterMap.computeIfAbsent(content.getId(), k -> new ContentBloomFilter(gcStateParamsPerTask.getBloomFilterSize(), gcParams.getBloomFilterFpp())).put(content);
            }
        });
    }
}

Example 18 with Content

use of org.projectnessie.model.Content in project nessie by projectnessie.

the class AbstractRestGC method fillExpectedContents.

void fillExpectedContents(Branch branch, int numCommits, IdentifiedResult expected) throws NessieNotFoundException {
    fetchLogEntries(branch, numCommits).stream().map(LogEntry::getOperations).filter(Objects::nonNull).flatMap(Collection::stream).filter(op -> op instanceof Put).forEach(op -> {
        Content content = ((Put) op).getContent();
        expected.addContent(branch.getName(), content);
    });
}

Example 19 with Content

use of org.projectnessie.model.Content in project nessie by projectnessie.

the class ITUpgradePath method keysUpgradeAddCommits.

@Test
@Order(203)
void keysUpgradeAddCommits() throws Exception {
    keysUpgradeBranch = (Branch) api.getReference().refName(keysUpgradeBranch.getName()).get();
    Map<ContentKey, IcebergTable> currentKeyValues = keysUpgradeAtHash.getOrDefault(keysUpgradeBranch.getHash(), Collections.emptyMap());
    for (int i = 0; i < keysUpgradeCommitsPerVersion; i++) {
        ContentKey key = ContentKey.of("keys.upgrade.table" + i);
        if ((i % 10) == 9) {
            keysUpgradeBranch = commitMaybeRetry(api.commitMultipleOperations().branch(keysUpgradeBranch).commitMeta(CommitMeta.fromMessage("Commit #" + i + "/delete from Nessie version " + version)).operation(Delete.of(key)));
            Map<ContentKey, IcebergTable> newKeyValues = new HashMap<>(currentKeyValues);
            newKeyValues.remove(key);
            keysUpgradeAtHash.put(keysUpgradeBranch.getHash(), newKeyValues);
        }
        Content currentContent = api.getContent().refName(keysUpgradeBranch.getName()).key(key).get().get(key);
        String cid = currentContent == null ? "table-" + i + "-" + version : currentContent.getId();
        IcebergTable newContent = IcebergTable.of("pointer-" + version + "-commit-" + i, keysUpgradeSequence++, i, i, i, cid);
        Put put = currentContent != null ? Put.of(key, newContent, currentContent) : Put.of(key, newContent);
        keysUpgradeBranch = commitMaybeRetry(api.commitMultipleOperations().branch(keysUpgradeBranch).commitMeta(CommitMeta.fromMessage("Commit #" + i + "/put from Nessie version " + version)).operation(put));
        Map<ContentKey, IcebergTable> newKeyValues = new HashMap<>(currentKeyValues);
        newKeyValues.remove(key);
        keysUpgradeAtHash.put(keysUpgradeBranch.getHash(), newKeyValues);
    }
}

Example 20 with Content

use of org.projectnessie.model.Content in project nessie by projectnessie.

the class ITUpgradePath method commitLog.

@Test
@Order(104)
void commitLog() {
    assertThat(api.getAllReferences().get().getReferences().stream().filter(r -> r.getName().startsWith(VERSION_BRANCH_PREFIX))).isNotEmpty().allSatisfy(ref -> {
        String versionFromRef = ref.getName().substring(VERSION_BRANCH_PREFIX.length());
        LogResponse commitLog = api.getCommitLog().refName(ref.getName()).get();
        String commitMessage = "hello world " + versionFromRef;
        assertThat(commitLog.getLogEntries()).hasSize(1).map(LogEntry::getCommitMeta).map(CommitMeta::getMessage).containsExactly(commitMessage);
    }).allSatisfy(ref -> {
        String versionFromRef = ref.getName().substring(VERSION_BRANCH_PREFIX.length());
        ContentKey key = ContentKey.of("my", "tables", "table_name");
        IcebergTable content = IcebergTable.of("metadata-location", 42L, 43, 44, 45, "content-id-" + versionFromRef);
        Map<ContentKey, Content> contents = api.getContent().reference(ref).key(key).get();
        assertThat(contents).containsExactly(entry(key, content));
    });
}