Search in sources :

Example 11 with IcebergTable

use of org.projectnessie.model.IcebergTable in project nessie by projectnessie.

the class GenerateContent method execute.

@Override
public void execute() throws BaseNessieClientServerException {
    if (runtimeDuration != null) {
        if (runtimeDuration.isZero() || runtimeDuration.isNegative()) {
            throw new ParameterException(spec.commandLine(), "Duration must be absent to greater than zero.");
        }
    }
    Duration perCommitDuration = Optional.ofNullable(runtimeDuration).orElse(Duration.ZERO).dividedBy(numCommits);
    ThreadLocalRandom random = ThreadLocalRandom.current();
    String runStartTime = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss").format(LocalDateTime.now());
    List<ContentKey> tableNames = IntStream.range(0, numTables).mapToObj(i -> ContentKey.of(String.format("create-contents-%s", runStartTime), "contents", Integer.toString(i))).collect(Collectors.toList());
    try (NessieApiV1 api = createNessieApiInstance()) {
        Branch defaultBranch;
        if (defaultBranchName == null) {
            // Use the server's default branch.
            defaultBranch = api.getDefaultBranch();
        } else {
            // Use the specified default branch.
            try {
                defaultBranch = (Branch) api.getReference().refName(defaultBranchName).get();
            } catch (NessieReferenceNotFoundException e) {
                // Create branch if it does not exist.
                defaultBranch = api.getDefaultBranch();
                defaultBranch = (Branch) api.createReference().reference(Branch.of(defaultBranchName, defaultBranch.getHash())).sourceRefName(defaultBranch.getName()).create();
            }
        }
        List<String> branches = new ArrayList<>();
        branches.add(defaultBranch.getName());
        while (branches.size() < branchCount) {
            // Create a new branch
            String newBranchName = "branch-" + runStartTime + "_" + (branches.size() - 1);
            Branch branch = Branch.of(newBranchName, defaultBranch.getHash());
            spec.commandLine().getOut().printf("Creating branch '%s' from '%s' at %s%n", branch.getName(), defaultBranch.getName(), branch.getHash());
            api.createReference().reference(branch).sourceRefName(defaultBranch.getName()).create();
            branches.add(newBranchName);
        }
        spec.commandLine().getOut().printf("Starting contents generation, %d commits...%n", numCommits);
        for (int i = 0; i < numCommits; i++) {
            // Choose a random branch to commit to
            String branchName = branches.get(random.nextInt(branches.size()));
            Branch commitToBranch = (Branch) api.getReference().refName(branchName).get();
            ContentKey tableName = tableNames.get(random.nextInt(tableNames.size()));
            Content tableContents = api.getContent().refName(branchName).key(tableName).get().get(tableName);
            Content newContents = createContents(tableContents, random);
            spec.commandLine().getOut().printf("Committing content-key '%s' to branch '%s' at %s%n", tableName, commitToBranch.getName(), commitToBranch.getHash());
            CommitMultipleOperationsBuilder commit = api.commitMultipleOperations().branch(commitToBranch).commitMeta(CommitMeta.builder().message(String.format("%s table %s on %s, commit #%d of %d", tableContents != null ? "Update" : "Create", tableName, branchName, i, numCommits)).author(System.getProperty("user.name")).authorTime(Instant.now()).build());
            if (newContents instanceof IcebergTable || newContents instanceof IcebergView) {
                commit.operation(Put.of(tableName, newContents, tableContents));
            } else {
                commit.operation(Put.of(tableName, newContents));
            }
            Branch newHead = commit.commit();
            if (random.nextDouble() < newTagProbability) {
                Tag tag = Tag.of("new-tag-" + random.nextLong(), newHead.getHash());
                spec.commandLine().getOut().printf("Creating tag '%s' from '%s' at %s%n", tag.getName(), branchName, tag.getHash());
                api.createReference().reference(tag).sourceRefName(branchName).create();
            }
            try {
                TimeUnit.NANOSECONDS.sleep(perCommitDuration.toNanos());
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                break;
            }
        }
    }
    spec.commandLine().getOut().printf("Done creating contents.%n");
}
Also used : IntStream(java.util.stream.IntStream) ImmutableIcebergView(org.projectnessie.model.ImmutableIcebergView) Put(org.projectnessie.model.Operation.Put) LocalDateTime(java.time.LocalDateTime) NessieReferenceNotFoundException(org.projectnessie.error.NessieReferenceNotFoundException) ParameterException(picocli.CommandLine.ParameterException) Spec(picocli.CommandLine.Spec) ArrayList(java.util.ArrayList) Duration(java.time.Duration) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) Content(org.projectnessie.model.Content) CommitMeta(org.projectnessie.model.CommitMeta) Command(picocli.CommandLine.Command) Branch(org.projectnessie.model.Branch) ImmutableDeltaLakeTable(org.projectnessie.model.ImmutableDeltaLakeTable) Min(javax.validation.constraints.Min) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) NessieApiV1(org.projectnessie.client.api.NessieApiV1) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) ImmutableIcebergTable(org.projectnessie.model.ImmutableIcebergTable) Option(picocli.CommandLine.Option) IcebergView(org.projectnessie.model.IcebergView) IcebergTable(org.projectnessie.model.IcebergTable) DateTimeFormatter(java.time.format.DateTimeFormatter) Optional(java.util.Optional) Tag(org.projectnessie.model.Tag) BaseNessieClientServerException(org.projectnessie.error.BaseNessieClientServerException) ContentKey(org.projectnessie.model.ContentKey) CommandSpec(picocli.CommandLine.Model.CommandSpec) CommitMultipleOperationsBuilder(org.projectnessie.client.api.CommitMultipleOperationsBuilder) CommitMultipleOperationsBuilder(org.projectnessie.client.api.CommitMultipleOperationsBuilder) ArrayList(java.util.ArrayList) Duration(java.time.Duration) ImmutableIcebergView(org.projectnessie.model.ImmutableIcebergView) IcebergView(org.projectnessie.model.IcebergView) ContentKey(org.projectnessie.model.ContentKey) NessieReferenceNotFoundException(org.projectnessie.error.NessieReferenceNotFoundException) Branch(org.projectnessie.model.Branch) Content(org.projectnessie.model.Content) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) ImmutableIcebergTable(org.projectnessie.model.ImmutableIcebergTable) IcebergTable(org.projectnessie.model.IcebergTable) ParameterException(picocli.CommandLine.ParameterException) Tag(org.projectnessie.model.Tag) NessieApiV1(org.projectnessie.client.api.NessieApiV1)

Example 12 with IcebergTable

use of org.projectnessie.model.IcebergTable in project nessie by projectnessie.

the class AbstractRestMergeTransplant method transplant.

@ParameterizedTest
@ValueSource(booleans = { true, false })
public void transplant(boolean withDetachedCommit) throws BaseNessieClientServerException {
    Branch base = createBranch("transplant-base");
    Branch branch = createBranch("transplant-branch");
    IcebergTable table1 = IcebergTable.of("transplant-table1", 42, 42, 42, 42);
    IcebergTable table2 = IcebergTable.of("transplant-table2", 43, 43, 43, 43);
    Branch committed1 = getApi().commitMultipleOperations().branchName(branch.getName()).hash(branch.getHash()).commitMeta(CommitMeta.fromMessage("test-transplant-branch1")).operation(Put.of(ContentKey.of("key1"), table1)).commit();
    assertThat(committed1.getHash()).isNotNull();
    Branch committed2 = getApi().commitMultipleOperations().branchName(branch.getName()).hash(committed1.getHash()).commitMeta(CommitMeta.fromMessage("test-transplant-branch2")).operation(Put.of(ContentKey.of("key1"), table1, table1)).commit();
    assertThat(committed2.getHash()).isNotNull();
    int commitsToTransplant = 2;
    LogResponse logBranch = getApi().getCommitLog().refName(branch.getName()).untilHash(branch.getHash()).maxRecords(commitsToTransplant).get();
    getApi().commitMultipleOperations().branchName(base.getName()).hash(base.getHash()).commitMeta(CommitMeta.fromMessage("test-transplant-main")).operation(Put.of(ContentKey.of("key2"), table2)).commit();
    getApi().transplantCommitsIntoBranch().hashesToTransplant(ImmutableList.of(committed1.getHash(), committed2.getHash())).fromRefName(maybeAsDetachedName(withDetachedCommit, branch)).branch(base).transplant();
    LogResponse log = getApi().getCommitLog().refName(base.getName()).untilHash(base.getHash()).get();
    assertThat(log.getLogEntries().stream().map(LogEntry::getCommitMeta).map(CommitMeta::getMessage)).containsExactly("test-transplant-branch2", "test-transplant-branch1", "test-transplant-main");
    // Verify that the commit-timestamp was updated
    LogResponse logOfTransplanted = getApi().getCommitLog().refName(base.getName()).maxRecords(commitsToTransplant).get();
    assertThat(logOfTransplanted.getLogEntries().stream().map(LogEntry::getCommitMeta).map(CommitMeta::getCommitTime)).isNotEqualTo(logBranch.getLogEntries().stream().map(LogEntry::getCommitMeta).map(CommitMeta::getCommitTime));
    assertThat(getApi().getEntries().refName(base.getName()).get().getEntries().stream().map(e -> e.getName().getName())).containsExactlyInAnyOrder("key1", "key2");
}
Also used : LogResponse(org.projectnessie.model.LogResponse) Branch(org.projectnessie.model.Branch) IcebergTable(org.projectnessie.model.IcebergTable) CommitMeta(org.projectnessie.model.CommitMeta) LogEntry(org.projectnessie.model.LogResponse.LogEntry) ValueSource(org.junit.jupiter.params.provider.ValueSource) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 13 with IcebergTable

use of org.projectnessie.model.IcebergTable in project nessie by projectnessie.

the class AbstractRestMisc method checkSpecialCharacterRoundTrip.

@Test
public void checkSpecialCharacterRoundTrip() throws BaseNessieClientServerException {
    Branch branch = createBranch("specialchar");
    // ContentKey k = ContentKey.of("/%国","国.国");
    ContentKey k = ContentKey.of("a.b", "c.txt");
    IcebergTable ta = IcebergTable.of("path1", 42, 42, 42, 42);
    getApi().commitMultipleOperations().branch(branch).operation(Put.of(k, ta)).commitMeta(CommitMeta.fromMessage("commit 1")).commit();
    assertThat(getApi().getContent().key(k).refName(branch.getName()).get()).containsEntry(k, ta);
    assertEquals(ta, getApi().getContent().key(k).refName(branch.getName()).get().get(k));
}
Also used : ContentKey(org.projectnessie.model.ContentKey) Branch(org.projectnessie.model.Branch) IcebergTable(org.projectnessie.model.IcebergTable) Test(org.junit.jupiter.api.Test)

Example 14 with IcebergTable

use of org.projectnessie.model.IcebergTable in project nessie by projectnessie.

the class AbstractRestNamespace method testNamespaceMergeWithConflict.

@Test
public void testNamespaceMergeWithConflict() throws BaseNessieClientServerException {
    Branch base = createBranch("merge-base");
    Branch branch = createBranch("merge-branch");
    Namespace ns = Namespace.parse("a.b.c");
    // create a namespace on the base branch
    getApi().createNamespace().namespace(ns).refName(base.getName()).create();
    base = (Branch) getApi().getReference().refName(base.getName()).get();
    // create a table with the same name on the other branch
    IcebergTable table = IcebergTable.of("merge-table1", 42, 42, 42, 42);
    branch = getApi().commitMultipleOperations().branchName(branch.getName()).hash(branch.getHash()).commitMeta(CommitMeta.fromMessage("test-merge-branch1")).operation(Put.of(ContentKey.of("a", "b", "c"), table)).commit();
    Branch finalBase = base;
    Branch finalBranch = branch;
    assertThatThrownBy(() -> getApi().mergeRefIntoBranch().branch(finalBase).fromRef(finalBranch).merge()).isInstanceOf(NessieReferenceConflictException.class).hasMessage("The following keys have been changed in conflict: 'a.b.c'");
    LogResponse log = getApi().getCommitLog().refName(base.getName()).untilHash(base.getHash()).get();
    // merging should not have been possible ("test-merge-branch1" shouldn't be in the commits)
    assertThat(log.getLogEntries().stream().map(LogEntry::getCommitMeta).map(CommitMeta::getMessage)).containsExactly("create namespace a.b.c");
    List<Entry> entries = getApi().getEntries().refName(base.getName()).get().getEntries();
    assertThat(entries.stream().map(Entry::getName)).containsExactly(ContentKey.of(ns.getElements()));
    assertThat(getApi().getNamespace().refName(base.getName()).namespace(ns).get()).isNotNull();
}
Also used : LogResponse(org.projectnessie.model.LogResponse) LogEntry(org.projectnessie.model.LogResponse.LogEntry) Entry(org.projectnessie.model.EntriesResponse.Entry) Branch(org.projectnessie.model.Branch) IcebergTable(org.projectnessie.model.IcebergTable) NessieReferenceConflictException(org.projectnessie.error.NessieReferenceConflictException) CommitMeta(org.projectnessie.model.CommitMeta) Namespace(org.projectnessie.model.Namespace) Test(org.junit.jupiter.api.Test)

Example 15 with IcebergTable

use of org.projectnessie.model.IcebergTable in project nessie by projectnessie.

the class AbstractRestRefLog method testReflog.

@Test
public void testReflog() throws BaseNessieClientServerException {
    String tagName = "tag1_test_reflog";
    String branch1 = "branch1_test_reflog";
    String branch2 = "branch2_test_reflog";
    String branch3 = "branch3_test_reflog";
    String root = "ref_name_test_reflog";
    List<Tuple> expectedEntries = new ArrayList<>(12);
    // reflog 1: creating the default branch0
    Branch branch0 = createBranch(root);
    expectedEntries.add(Tuple.tuple(root, "CREATE_REFERENCE"));
    // reflog 2: create tag1
    Reference createdTag = getApi().createReference().sourceRefName(branch0.getName()).reference(Tag.of(tagName, branch0.getHash())).create();
    expectedEntries.add(Tuple.tuple(tagName, "CREATE_REFERENCE"));
    // reflog 3: create branch1
    Reference createdBranch1 = getApi().createReference().sourceRefName(branch0.getName()).reference(Branch.of(branch1, branch0.getHash())).create();
    expectedEntries.add(Tuple.tuple(branch1, "CREATE_REFERENCE"));
    // reflog 4: create branch2
    Reference createdBranch2 = getApi().createReference().sourceRefName(branch0.getName()).reference(Branch.of(branch2, branch0.getHash())).create();
    expectedEntries.add(Tuple.tuple(branch2, "CREATE_REFERENCE"));
    // reflog 5: create branch2
    Branch createdBranch3 = (Branch) getApi().createReference().sourceRefName(branch0.getName()).reference(Branch.of(branch3, branch0.getHash())).create();
    expectedEntries.add(Tuple.tuple(branch3, "CREATE_REFERENCE"));
    // reflog 6: commit on default branch0
    IcebergTable meta = IcebergTable.of("meep", 42, 42, 42, 42);
    branch0 = getApi().commitMultipleOperations().branchName(branch0.getName()).hash(branch0.getHash()).commitMeta(CommitMeta.builder().message("dummy commit log").properties(ImmutableMap.of("prop1", "val1", "prop2", "val2")).build()).operation(Operation.Put.of(ContentKey.of("meep"), meta)).commit();
    expectedEntries.add(Tuple.tuple(root, "COMMIT"));
    // reflog 7: assign tag
    getApi().assignTag().tagName(tagName).hash(createdTag.getHash()).assignTo(branch0).assign();
    expectedEntries.add(Tuple.tuple(tagName, "ASSIGN_REFERENCE"));
    // reflog 8: assign ref
    getApi().assignBranch().branchName(branch1).hash(createdBranch1.getHash()).assignTo(branch0).assign();
    expectedEntries.add(Tuple.tuple(branch1, "ASSIGN_REFERENCE"));
    // reflog 9: merge
    getApi().mergeRefIntoBranch().branchName(branch2).hash(createdBranch2.getHash()).fromRefName(branch1).fromHash(branch0.getHash()).merge();
    expectedEntries.add(Tuple.tuple(branch2, "MERGE"));
    // reflog 10: transplant
    getApi().transplantCommitsIntoBranch().hashesToTransplant(ImmutableList.of(Objects.requireNonNull(branch0.getHash()))).fromRefName(branch1).branch(createdBranch3).transplant();
    expectedEntries.add(Tuple.tuple(branch3, "TRANSPLANT"));
    // reflog 11: delete branch
    getApi().deleteBranch().branchName(branch1).hash(branch0.getHash()).delete();
    expectedEntries.add(Tuple.tuple(branch1, "DELETE_REFERENCE"));
    // reflog 12: delete tag
    getApi().deleteTag().tagName(tagName).hash(branch0.getHash()).delete();
    expectedEntries.add(Tuple.tuple(tagName, "DELETE_REFERENCE"));
    // In the reflog output new entry will be the head. Hence, reverse the expected list
    Collections.reverse(expectedEntries);
    RefLogResponse refLogResponse = getApi().getRefLog().get();
    // verify reflog entries
    assertThat(refLogResponse.getLogEntries().subList(0, 12)).extracting(RefLogResponse.RefLogResponseEntry::getRefName, RefLogResponse.RefLogResponseEntry::getOperation).isEqualTo(expectedEntries);
    // verify pagination (limit and token)
    RefLogResponse refLogResponse1 = getApi().getRefLog().maxRecords(2).get();
    assertThat(refLogResponse1.getLogEntries()).isEqualTo(refLogResponse.getLogEntries().subList(0, 2));
    assertThat(refLogResponse1.isHasMore()).isTrue();
    RefLogResponse refLogResponse2 = getApi().getRefLog().pageToken(refLogResponse1.getToken()).get();
    // should start from the token.
    assertThat(refLogResponse2.getLogEntries().get(0).getRefLogId()).isEqualTo(refLogResponse1.getToken());
    assertThat(refLogResponse2.getLogEntries().subList(0, 10)).isEqualTo(refLogResponse.getLogEntries().subList(2, 12));
    // verify startHash and endHash
    RefLogResponse refLogResponse3 = getApi().getRefLog().fromHash(refLogResponse.getLogEntries().get(10).getRefLogId()).get();
    assertThat(refLogResponse3.getLogEntries().subList(0, 2)).isEqualTo(refLogResponse.getLogEntries().subList(10, 12));
    RefLogResponse refLogResponse4 = getApi().getRefLog().fromHash(refLogResponse.getLogEntries().get(3).getRefLogId()).untilHash(refLogResponse.getLogEntries().get(5).getRefLogId()).get();
    assertThat(refLogResponse4.getLogEntries()).isEqualTo(refLogResponse.getLogEntries().subList(3, 6));
    // use invalid reflog id f1234d75178d892a133a410355a5a990cf75d2f33eba25d575943d4df632f3a4
    // computed using Hash.of(
    // UnsafeByteOperations.unsafeWrap(newHasher().putString("invalid",
    // StandardCharsets.UTF_8).hash().asBytes()));
    assertThatThrownBy(() -> getApi().getRefLog().fromHash("f1234d75178d892a133a410355a5a990cf75d2f33eba25d575943d4df632f3a4").get()).isInstanceOf(NessieRefLogNotFoundException.class).hasMessageContaining("RefLog entry for 'f1234d75178d892a133a410355a5a990cf75d2f33eba25d575943d4df632f3a4' does not exist");
    // verify source hashes for assign reference
    assertThat(refLogResponse.getLogEntries().get(4).getSourceHashes()).isEqualTo(Collections.singletonList(createdBranch1.getHash()));
    // verify source hashes for merge
    assertThat(refLogResponse.getLogEntries().get(3).getSourceHashes()).isEqualTo(Collections.singletonList(branch0.getHash()));
    // verify source hashes for transplant
    assertThat(refLogResponse.getLogEntries().get(2).getSourceHashes()).isEqualTo(Collections.singletonList(branch0.getHash()));
    // test filter with stream
    List<RefLogResponse.RefLogResponseEntry> filteredResult = StreamingUtil.getReflogStream(getApi(), builder -> builder.filter("reflog.operation == 'ASSIGN_REFERENCE' " + "&& reflog.refName == 'tag1_test_reflog'"), OptionalInt.empty()).collect(Collectors.toList());
    assertThat(filteredResult.size()).isEqualTo(1);
    assertThat(filteredResult.get(0)).extracting(RefLogResponse.RefLogResponseEntry::getRefName, RefLogResponse.RefLogResponseEntry::getOperation).isEqualTo(expectedEntries.get(5).toList());
}
Also used : Operation(org.projectnessie.model.Operation) Tuple(org.assertj.core.groups.Tuple) ImmutableMap(com.google.common.collect.ImmutableMap) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) Branch(org.projectnessie.model.Branch) OptionalInt(java.util.OptionalInt) Collectors(java.util.stream.Collectors) Reference(org.projectnessie.model.Reference) NessieRefLogNotFoundException(org.projectnessie.error.NessieRefLogNotFoundException) ArrayList(java.util.ArrayList) Objects(java.util.Objects) Test(org.junit.jupiter.api.Test) List(java.util.List) Assertions.assertThatThrownBy(org.assertj.core.api.Assertions.assertThatThrownBy) ImmutableList(com.google.common.collect.ImmutableList) StreamingUtil(org.projectnessie.client.StreamingUtil) IcebergTable(org.projectnessie.model.IcebergTable) Tag(org.projectnessie.model.Tag) BaseNessieClientServerException(org.projectnessie.error.BaseNessieClientServerException) ContentKey(org.projectnessie.model.ContentKey) CommitMeta(org.projectnessie.model.CommitMeta) Collections(java.util.Collections) RefLogResponse(org.projectnessie.model.RefLogResponse) Branch(org.projectnessie.model.Branch) Reference(org.projectnessie.model.Reference) ArrayList(java.util.ArrayList) IcebergTable(org.projectnessie.model.IcebergTable) RefLogResponse(org.projectnessie.model.RefLogResponse) NessieRefLogNotFoundException(org.projectnessie.error.NessieRefLogNotFoundException) Tuple(org.assertj.core.groups.Tuple) Test(org.junit.jupiter.api.Test)

Aggregations

IcebergTable (org.projectnessie.model.IcebergTable)36 Branch (org.projectnessie.model.Branch)25 Test (org.junit.jupiter.api.Test)22 ContentKey (org.projectnessie.model.ContentKey)18 Content (org.projectnessie.model.Content)12 CommitMeta (org.projectnessie.model.CommitMeta)10 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)9 LogEntry (org.projectnessie.model.LogResponse.LogEntry)8 Reference (org.projectnessie.model.Reference)8 List (java.util.List)7 Collectors (java.util.stream.Collectors)7 Tag (org.projectnessie.model.Tag)7 LogResponse (org.projectnessie.model.LogResponse)6 Map (java.util.Map)5 BaseNessieClientServerException (org.projectnessie.error.BaseNessieClientServerException)5 NessieNotFoundException (org.projectnessie.error.NessieNotFoundException)5 Put (org.projectnessie.model.Operation.Put)5 ArrayList (java.util.ArrayList)4 Stream (java.util.stream.Stream)4 CommitFailedException (org.apache.iceberg.exceptions.CommitFailedException)4