use of org.projectnessie.model.IcebergView in project nessie by projectnessie.
the class GenerateContent method execute.
@Override
public void execute() throws BaseNessieClientServerException {
if (runtimeDuration != null) {
if (runtimeDuration.isZero() || runtimeDuration.isNegative()) {
throw new ParameterException(spec.commandLine(), "Duration must be absent to greater than zero.");
}
}
Duration perCommitDuration = Optional.ofNullable(runtimeDuration).orElse(Duration.ZERO).dividedBy(numCommits);
ThreadLocalRandom random = ThreadLocalRandom.current();
String runStartTime = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss").format(LocalDateTime.now());
List<ContentKey> tableNames = IntStream.range(0, numTables).mapToObj(i -> ContentKey.of(String.format("create-contents-%s", runStartTime), "contents", Integer.toString(i))).collect(Collectors.toList());
try (NessieApiV1 api = createNessieApiInstance()) {
Branch defaultBranch;
if (defaultBranchName == null) {
// Use the server's default branch.
defaultBranch = api.getDefaultBranch();
} else {
// Use the specified default branch.
try {
defaultBranch = (Branch) api.getReference().refName(defaultBranchName).get();
} catch (NessieReferenceNotFoundException e) {
// Create branch if it does not exist.
defaultBranch = api.getDefaultBranch();
defaultBranch = (Branch) api.createReference().reference(Branch.of(defaultBranchName, defaultBranch.getHash())).sourceRefName(defaultBranch.getName()).create();
}
}
List<String> branches = new ArrayList<>();
branches.add(defaultBranch.getName());
while (branches.size() < branchCount) {
// Create a new branch
String newBranchName = "branch-" + runStartTime + "_" + (branches.size() - 1);
Branch branch = Branch.of(newBranchName, defaultBranch.getHash());
spec.commandLine().getOut().printf("Creating branch '%s' from '%s' at %s%n", branch.getName(), defaultBranch.getName(), branch.getHash());
api.createReference().reference(branch).sourceRefName(defaultBranch.getName()).create();
branches.add(newBranchName);
}
spec.commandLine().getOut().printf("Starting contents generation, %d commits...%n", numCommits);
for (int i = 0; i < numCommits; i++) {
// Choose a random branch to commit to
String branchName = branches.get(random.nextInt(branches.size()));
Branch commitToBranch = (Branch) api.getReference().refName(branchName).get();
ContentKey tableName = tableNames.get(random.nextInt(tableNames.size()));
Content tableContents = api.getContent().refName(branchName).key(tableName).get().get(tableName);
Content newContents = createContents(tableContents, random);
spec.commandLine().getOut().printf("Committing content-key '%s' to branch '%s' at %s%n", tableName, commitToBranch.getName(), commitToBranch.getHash());
CommitMultipleOperationsBuilder commit = api.commitMultipleOperations().branch(commitToBranch).commitMeta(CommitMeta.builder().message(String.format("%s table %s on %s, commit #%d of %d", tableContents != null ? "Update" : "Create", tableName, branchName, i, numCommits)).author(System.getProperty("user.name")).authorTime(Instant.now()).build());
if (newContents instanceof IcebergTable || newContents instanceof IcebergView) {
commit.operation(Put.of(tableName, newContents, tableContents));
} else {
commit.operation(Put.of(tableName, newContents));
}
Branch newHead = commit.commit();
if (random.nextDouble() < newTagProbability) {
Tag tag = Tag.of("new-tag-" + random.nextLong(), newHead.getHash());
spec.commandLine().getOut().printf("Creating tag '%s' from '%s' at %s%n", tag.getName(), branchName, tag.getHash());
api.createReference().reference(tag).sourceRefName(branchName).create();
}
try {
TimeUnit.NANOSECONDS.sleep(perCommitDuration.toNanos());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
}
spec.commandLine().getOut().printf("Done creating contents.%n");
}
use of org.projectnessie.model.IcebergView in project nessie by projectnessie.
the class AbstractRestEntries method filterEntriesByType.
@ParameterizedTest
@EnumSource(ReferenceMode.class)
public void filterEntriesByType(ReferenceMode refMode) throws BaseNessieClientServerException {
Branch branch = createBranch("filterTypes");
ContentKey a = ContentKey.of("a");
ContentKey b = ContentKey.of("b");
IcebergTable tam = IcebergTable.of("path1", 42, 42, 42, 42);
IcebergView tb = IcebergView.of("pathx", 1, 1, "select * from table", "Dremio");
getApi().commitMultipleOperations().branch(branch).operation(Put.of(a, tam)).commitMeta(CommitMeta.fromMessage("commit 1")).commit();
branch = getApi().commitMultipleOperations().branch(branch).operation(Put.of(b, tb)).commitMeta(CommitMeta.fromMessage("commit 2")).commit();
List<Entry> entries = getApi().getEntries().reference(refMode.transform(branch)).get().getEntries();
List<Entry> expected = asList(Entry.builder().name(a).type(Type.ICEBERG_TABLE).build(), Entry.builder().name(b).type(Type.ICEBERG_VIEW).build());
assertThat(entries).containsExactlyInAnyOrderElementsOf(expected);
entries = getApi().getEntries().reference(refMode.transform(branch)).filter("entry.contentType=='ICEBERG_TABLE'").get().getEntries();
assertEquals(singletonList(expected.get(0)), entries);
entries = getApi().getEntries().reference(refMode.transform(branch)).filter("entry.contentType=='ICEBERG_VIEW'").get().getEntries();
assertEquals(singletonList(expected.get(1)), entries);
entries = getApi().getEntries().reference(refMode.transform(branch)).filter("entry.contentType in ['ICEBERG_TABLE', 'ICEBERG_VIEW']").get().getEntries();
assertThat(entries).containsExactlyInAnyOrderElementsOf(expected);
}
use of org.projectnessie.model.IcebergView in project nessie by projectnessie.
the class TableCommitMetaStoreWorker method toStoreGlobalState.
@Override
public ByteString toStoreGlobalState(Content content) {
ObjectTypes.Content.Builder builder = ObjectTypes.Content.newBuilder().setId(content.getId());
if (content instanceof IcebergTable) {
IcebergTable state = (IcebergTable) content;
ObjectTypes.IcebergMetadataPointer.Builder stateBuilder = ObjectTypes.IcebergMetadataPointer.newBuilder().setMetadataLocation(state.getMetadataLocation());
builder.setIcebergMetadataPointer(stateBuilder);
} else if (content instanceof IcebergView) {
IcebergView state = (IcebergView) content;
ObjectTypes.IcebergMetadataPointer.Builder stateBuilder = ObjectTypes.IcebergMetadataPointer.newBuilder().setMetadataLocation(state.getMetadataLocation());
builder.setIcebergMetadataPointer(stateBuilder);
} else {
throw new IllegalArgumentException("Unknown type " + content);
}
return builder.build().toByteString();
}
use of org.projectnessie.model.IcebergView in project nessie by projectnessie.
the class NessieViewOperations method drop.
@Override
public void drop(String viewIdentifier) {
reference.checkMutable();
IcebergView existingView = view(toCatalogTableIdentifier(viewIdentifier));
if (existingView == null) {
return;
}
CommitMultipleOperationsBuilder commitBuilderBase = api.commitMultipleOperations().commitMeta(NessieUtil.buildCommitMetadata(String.format("Iceberg delete view %s", viewIdentifier), catalogOptions)).operation(Operation.Delete.of(NessieUtil.toKey(toCatalogTableIdentifier(viewIdentifier))));
// We try to drop the view. Simple retry after ref update.
try {
Tasks.foreach(commitBuilderBase).retry(5).stopRetryOn(NessieNotFoundException.class).throwFailureWhenFinished().onFailure((o, exception) -> refresh()).run(commitBuilder -> {
Branch branch = commitBuilder.branch(reference.getAsBranch()).commit();
reference.updateReference(branch);
}, BaseNessieClientServerException.class);
} catch (NessieConflictException e) {
LOG.error("Cannot drop view: failed after retry (update ref and retry)", e);
} catch (NessieNotFoundException e) {
LOG.error("Cannot drop view: ref is no longer valid.", e);
} catch (BaseNessieClientServerException e) {
LOG.error("Cannot drop view: unknown error", e);
}
}
use of org.projectnessie.model.IcebergView in project nessie by projectnessie.
the class NessieViewOperations method commit.
@Override
public void commit(ViewVersionMetadata base, ViewVersionMetadata metadata, Map<String, String> properties) {
reference.checkMutable();
String newMetadataLocation = writeNewMetadata(metadata, currentVersion() + 1);
boolean delete = true;
try {
ImmutableIcebergView.Builder viewBuilder = ImmutableIcebergView.builder();
if (icebergView != null) {
viewBuilder.id(icebergView.getId());
}
IcebergView newView = viewBuilder.metadataLocation(newMetadataLocation).versionId(metadata.currentVersionId()).schemaId(metadata.definition().schema().schemaId()).dialect("TODO: needs to be defined in Iceberg ViewDefinition").sqlText(metadata.definition().sql()).build();
LOG.debug("Committing '{}' against '{}': {}", key, reference.getReference(), newView);
ImmutableCommitMeta.Builder builder = ImmutableCommitMeta.builder();
builder.message(buildCommitMsg(base, metadata) + " " + key.getName());
Branch branch = api.commitMultipleOperations().operation(Operation.Put.of(key, newView, icebergView)).commitMeta(NessieUtil.catalogOptions(builder, catalogOptions).build()).branch(reference.getAsBranch()).commit();
reference.updateReference(branch);
delete = false;
} catch (NessieConflictException ex) {
throw new CommitFailedException(ex, "Commit failed: Reference hash is out of date. " + "Update the reference %s and try again", reference.getName());
} catch (HttpClientException ex) {
// Intentionally catch all nessie-client-exceptions here and not just the "timeout" variant
// to catch all kinds of network errors (e.g. connection reset). Network code implementation
// details and all kinds of network devices can induce unexpected behavior. So better be
// safe than sorry.
delete = false;
throw new CommitStateUnknownException(ex);
} catch (NessieNotFoundException ex) {
throw new RuntimeException(String.format("Commit failed: Reference %s no longer exist", reference.getName()), ex);
} finally {
if (delete) {
io().deleteFile(newMetadataLocation);
}
}
}
Aggregations