Examples with Operation org.projectnessie.model.Operation used on opensource projects

Example 6 with Operation

use of org.projectnessie.model.Operation in project nessie by projectnessie.

the class IdentifyContentsPerExecutor method handleLiveCommit.

private void handleLiveCommit(GCStateParamsPerTask gcStateParamsPerTask, LogResponse.LogEntry logEntry, Map<String, ContentBloomFilter> bloomFilterMap, MutableBoolean foundAllLiveCommitHeadsBeforeCutoffTime, Set<ContentKey> liveContentKeys) {
    if (logEntry.getOperations() != null) {
        boolean isExpired = !gcStateParamsPerTask.getLiveCommitPredicate().test(logEntry.getCommitMeta());
        if (isExpired && liveContentKeys.isEmpty()) {
            // as it is the first expired commit. Time travel is supported till this state.
            try {
                gcStateParamsPerTask.getApi().getEntries().refName(Detached.REF_NAME).hashOnRef(logEntry.getCommitMeta().getHash()).get().getEntries().forEach(entries -> liveContentKeys.add(entries.getName()));
            } catch (NessieNotFoundException e) {
                throw new RuntimeException(e);
            }
        }
        logEntry.getOperations().stream().filter(operation -> operation instanceof Operation.Put).forEach(operation -> {
            boolean addContent;
            if (liveContentKeys.contains(operation.getKey())) {
                // commit head of this key
                addContent = true;
                liveContentKeys.remove(operation.getKey());
                if (liveContentKeys.isEmpty()) {
                    // found all the live commit heads before cutoff time.
                    foundAllLiveCommitHeadsBeforeCutoffTime.setTrue();
                }
            } else {
                addContent = !isExpired;
            }
            if (addContent) {
                Content content = ((Operation.Put) operation).getContent();
                bloomFilterMap.computeIfAbsent(content.getId(), k -> new ContentBloomFilter(gcStateParamsPerTask.getBloomFilterSize(), gcParams.getBloomFilterFpp())).put(content);
            }
        });
    }
}

Example 7 with Operation

use of org.projectnessie.model.Operation in project nessie by projectnessie.

the class AbstractRestContents method verifyAllContentAndOperationTypes.

@Test
public void verifyAllContentAndOperationTypes() throws BaseNessieClientServerException {
    Branch branch = createBranch("contentAndOperationAll");
    CommitMultipleOperationsBuilder commit = getApi().commitMultipleOperations().branch(branch).commitMeta(CommitMeta.fromMessage("verifyAllContentAndOperationTypes"));
    contentAndOperationTypes().flatMap(c -> c.globalOperation == null ? Stream.of(c.operation) : Stream.of(c.operation, c.globalOperation)).forEach(commit::operation);
    commit.commit();
    List<Entry> entries = getApi().getEntries().refName(branch.getName()).get().getEntries();
    List<Entry> expect = contentAndOperationTypes().filter(c -> c.operation instanceof Put).map(c -> Entry.builder().type(c.type).name(c.operation.getKey()).build()).collect(Collectors.toList());
    assertThat(entries).containsExactlyInAnyOrderElementsOf(expect);
}

Example 8 with Operation

use of org.projectnessie.model.Operation in project nessie by projectnessie.

the class AbstractRestReferences method referenceNames.

@ParameterizedTest
@ValueSource(strings = { "normal", "with-no_space", "slash/thing" })
public void referenceNames(String refNamePart) throws BaseNessieClientServerException {
    String tagName = "tag" + refNamePart;
    String branchName = "branch" + refNamePart;
    String branchName2 = "branch2" + refNamePart;
    String root = "ref_name_" + refNamePart.replaceAll("[^a-z]", "");
    Branch main = createBranch(root);
    IcebergTable meta = IcebergTable.of("meep", 42, 42, 42, 42);
    main = getApi().commitMultipleOperations().branchName(main.getName()).hash(main.getHash()).commitMeta(CommitMeta.builder().message("common-merge-ancestor").properties(ImmutableMap.of("prop1", "val1", "prop2", "val2")).build()).operation(Operation.Put.of(ContentKey.of("meep"), meta)).commit();
    String someHash = main.getHash();
    Reference createdTag = getApi().createReference().sourceRefName(main.getName()).reference(Tag.of(tagName, someHash)).create();
    assertEquals(Tag.of(tagName, someHash), createdTag);
    Reference createdBranch1 = getApi().createReference().sourceRefName(main.getName()).reference(Branch.of(branchName, someHash)).create();
    assertEquals(Branch.of(branchName, someHash), createdBranch1);
    Reference createdBranch2 = getApi().createReference().sourceRefName(main.getName()).reference(Branch.of(branchName2, someHash)).create();
    assertEquals(Branch.of(branchName2, someHash), createdBranch2);
    Map<String, Reference> references = getApi().getAllReferences().get().getReferences().stream().filter(r -> root.equals(r.getName()) || r.getName().endsWith(refNamePart)).collect(Collectors.toMap(Reference::getName, Function.identity()));
    assertThat(references).containsAllEntriesOf(ImmutableMap.of(main.getName(), main, createdTag.getName(), createdTag, createdBranch1.getName(), createdBranch1, createdBranch2.getName(), createdBranch2));
    assertThat(references.get(main.getName())).isInstanceOf(Branch.class);
    assertThat(references.get(createdTag.getName())).isInstanceOf(Tag.class);
    assertThat(references.get(createdBranch1.getName())).isInstanceOf(Branch.class);
    assertThat(references.get(createdBranch2.getName())).isInstanceOf(Branch.class);
    Reference tagRef = references.get(tagName);
    Reference branchRef = references.get(branchName);
    Reference branchRef2 = references.get(branchName2);
    String tagHash = tagRef.getHash();
    String branchHash = branchRef.getHash();
    String branchHash2 = branchRef2.getHash();
    assertThat(getApi().getReference().refName(tagName).get()).isEqualTo(tagRef);
    assertThat(getApi().getReference().refName(branchName).get()).isEqualTo(branchRef);
    EntriesResponse entries = getApi().getEntries().refName(tagName).get();
    assertThat(entries).isNotNull();
    entries = getApi().getEntries().refName(branchName).get();
    assertThat(entries).isNotNull();
    LogResponse log = getApi().getCommitLog().refName(tagName).get();
    assertThat(log).isNotNull();
    log = getApi().getCommitLog().refName(branchName).get();
    assertThat(log).isNotNull();
    // Need to have at least one op, otherwise all following operations (assignTag/Branch, merge,
    // delete) will fail
    meta = IcebergTable.of("foo", 42, 42, 42, 42);
    getApi().commitMultipleOperations().branchName(branchName).hash(branchHash).operation(Put.of(ContentKey.of("some-key"), meta)).commitMeta(CommitMeta.fromMessage("One dummy op")).commit();
    log = getApi().getCommitLog().refName(branchName).get();
    String newHash = log.getLogEntries().get(0).getCommitMeta().getHash();
    getApi().assignTag().tagName(tagName).hash(tagHash).assignTo(Branch.of(branchName, newHash)).assign();
    getApi().assignBranch().branchName(branchName).hash(newHash).assignTo(Branch.of(branchName, newHash)).assign();
    getApi().mergeRefIntoBranch().branchName(branchName2).hash(branchHash2).fromRefName(branchName).fromHash(newHash).merge();
}

Example 9 with Operation

use of org.projectnessie.model.Operation in project nessie by projectnessie.

the class ReadCommits method execute.

@Override
public void execute() throws NessieNotFoundException {
    try (NessieApiV1 api = createNessieApiInstance()) {
        spec.commandLine().getOut().printf("Reading commits for ref '%s'\n\n", ref);
        FetchOption fetchOption = isVerbose() ? FetchOption.ALL : FetchOption.MINIMAL;
        LogResponse logResponse = api.getCommitLog().refName(ref).fetch(fetchOption).get();
        for (LogResponse.LogEntry logEntry : logResponse.getLogEntries()) {
            CommitMeta commitMeta = logEntry.getCommitMeta();
            spec.commandLine().getOut().printf("%s\t%s\t%s [%s]\n", Objects.requireNonNull(commitMeta.getHash()).substring(0, 8), commitMeta.getAuthorTime(), commitMeta.getMessage(), commitMeta.getAuthor());
            List<Operation> operations = logEntry.getOperations();
            if (operations != null) {
                for (Operation op : operations) {
                    spec.commandLine().getOut().printf("  %s\n", op);
                    if (isVerbose()) {
                        List<String> key = op.getKey().getElements();
                        for (int i = 0; i < key.size(); i++) {
                            spec.commandLine().getOut().printf("    key[%d]: %s\n", i, key.get(i));
                        }
                    }
                }
            }
        }
        spec.commandLine().getOut().printf("\nDone reading commits for ref '%s'\n\n", ref);
    }
}

Example 10 with Operation

use of org.projectnessie.model.Operation in project nessie by projectnessie.

the class NessieViewOperations method drop.

@Override
public void drop(String viewIdentifier) {
    reference.checkMutable();
    IcebergView existingView = view(toCatalogTableIdentifier(viewIdentifier));
    if (existingView == null) {
        return;
    }
    CommitMultipleOperationsBuilder commitBuilderBase = api.commitMultipleOperations().commitMeta(NessieUtil.buildCommitMetadata(String.format("Iceberg delete view %s", viewIdentifier), catalogOptions)).operation(Operation.Delete.of(NessieUtil.toKey(toCatalogTableIdentifier(viewIdentifier))));
    // We try to drop the view. Simple retry after ref update.
    try {
        Tasks.foreach(commitBuilderBase).retry(5).stopRetryOn(NessieNotFoundException.class).throwFailureWhenFinished().onFailure((o, exception) -> refresh()).run(commitBuilder -> {
            Branch branch = commitBuilder.branch(reference.getAsBranch()).commit();
            reference.updateReference(branch);
        }, BaseNessieClientServerException.class);
    } catch (NessieConflictException e) {
        LOG.error("Cannot drop view: failed after retry (update ref and retry)", e);
    } catch (NessieNotFoundException e) {
        LOG.error("Cannot drop view: ref is no longer valid.", e);
    } catch (BaseNessieClientServerException e) {
        LOG.error("Cannot drop view: unknown error", e);
    }
}