Search in sources :

Example 11 with CreateSnapshotRequest

use of org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest in project crate by crate.

the class CreateSnapshotPlan method executeOrFail.

@Override
public void executeOrFail(DependencyCarrier dependencies, PlannerContext plannerContext, RowConsumer consumer, Row parameters, SubQueryResults subQueryResults) {
    CreateSnapshotRequest request = createRequest(createSnapshot, plannerContext.transactionContext(), dependencies.nodeContext(), parameters, subQueryResults, dependencies.schemas());
    var transportCreateSnapshotAction = dependencies.transportActionProvider().transportCreateSnapshotAction();
    transportCreateSnapshotAction.execute(request, new OneRowActionListener<>(consumer, response -> {
        SnapshotInfo snapshotInfo = response.getSnapshotInfo();
        if (// if wait_for_completion is false, the snapshotInfo is null
        snapshotInfo != null && snapshotInfo.state() == SnapshotState.FAILED) {
            // fail request if snapshot creation failed
            String reason = response.getSnapshotInfo().reason().replaceAll("Index", "Table").replaceAll("Indices", "Tables");
            consumer.accept(null, new CreateSnapshotException(createSnapshot.snapshot(), reason));
            return new Row1(-1L);
        } else {
            return new Row1(1L);
        }
    }));
}
Also used : Arrays(java.util.Arrays) Operation(io.crate.metadata.table.Operation) SnapshotInfo(org.elasticsearch.snapshots.SnapshotInfo) Function(java.util.function.Function) PartitionName(io.crate.metadata.PartitionName) DependencyCarrier(io.crate.planner.DependencyCarrier) HashSet(java.util.HashSet) WAIT_FOR_COMPLETION(io.crate.analyze.SnapshotSettings.WAIT_FOR_COMPLETION) SymbolEvaluator(io.crate.analyze.SymbolEvaluator) Settings(org.elasticsearch.common.settings.Settings) AnalyzedCreateSnapshot(io.crate.analyze.AnalyzedCreateSnapshot) IndicesOptions(org.elasticsearch.action.support.IndicesOptions) GenericPropertiesConverter(io.crate.analyze.GenericPropertiesConverter) SnapshotState(org.elasticsearch.snapshots.SnapshotState) OneRowActionListener(io.crate.execution.support.OneRowActionListener) PartitionUnknownException(io.crate.exceptions.PartitionUnknownException) PartitionPropertiesAnalyzer.toPartitionName(io.crate.analyze.PartitionPropertiesAnalyzer.toPartitionName) SchemaInfo(io.crate.metadata.table.SchemaInfo) DocTableInfo(io.crate.metadata.doc.DocTableInfo) TableInfo(io.crate.metadata.table.TableInfo) NodeContext(io.crate.metadata.NodeContext) CreateSnapshotException(io.crate.exceptions.CreateSnapshotException) Table(io.crate.sql.tree.Table) ResourceUnknownException(io.crate.exceptions.ResourceUnknownException) CreateSnapshotRequest(org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest) Lists2(io.crate.common.collections.Lists2) SnapshotSettings(io.crate.analyze.SnapshotSettings) RowConsumer(io.crate.data.RowConsumer) Logger(org.apache.logging.log4j.Logger) Row(io.crate.data.Row) Symbol(io.crate.expression.symbol.Symbol) PlannerContext(io.crate.planner.PlannerContext) IGNORE_UNAVAILABLE(io.crate.analyze.SnapshotSettings.IGNORE_UNAVAILABLE) Plan(io.crate.planner.Plan) SubQueryResults(io.crate.planner.operators.SubQueryResults) Schemas(io.crate.metadata.Schemas) VisibleForTesting(io.crate.common.annotations.VisibleForTesting) LogManager(org.apache.logging.log4j.LogManager) Row1(io.crate.data.Row1) CoordinatorTxnCtx(io.crate.metadata.CoordinatorTxnCtx) Row1(io.crate.data.Row1) SnapshotInfo(org.elasticsearch.snapshots.SnapshotInfo) CreateSnapshotException(io.crate.exceptions.CreateSnapshotException) CreateSnapshotRequest(org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest)

Example 12 with CreateSnapshotRequest

use of org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest in project crate by crate.

the class CreateSnapshotPlan method createRequest.

@VisibleForTesting
public static CreateSnapshotRequest createRequest(AnalyzedCreateSnapshot createSnapshot, CoordinatorTxnCtx txnCtx, NodeContext nodeCtx, Row parameters, SubQueryResults subQueryResults, Schemas schemas) {
    Function<? super Symbol, Object> eval = x -> SymbolEvaluator.evaluate(txnCtx, nodeCtx, x, parameters, subQueryResults);
    Settings settings = GenericPropertiesConverter.genericPropertiesToSettings(createSnapshot.properties().map(eval), SnapshotSettings.SETTINGS);
    boolean ignoreUnavailable = IGNORE_UNAVAILABLE.get(settings);
    final HashSet<String> snapshotIndices;
    final HashSet<String> templates = new HashSet<>();
    if (createSnapshot.tables().isEmpty()) {
        for (SchemaInfo schemaInfo : schemas) {
            for (TableInfo tableInfo : schemaInfo.getTables()) {
                // only check for user generated tables
                if (tableInfo instanceof DocTableInfo) {
                    Operation.blockedRaiseException(tableInfo, Operation.READ);
                }
            }
        }
        snapshotIndices = new HashSet<>(AnalyzedCreateSnapshot.ALL_INDICES);
    } else {
        snapshotIndices = new HashSet<>(createSnapshot.tables().size());
        for (Table<Symbol> table : createSnapshot.tables()) {
            DocTableInfo docTableInfo;
            try {
                docTableInfo = (DocTableInfo) schemas.resolveTableInfo(table.getName(), Operation.CREATE_SNAPSHOT, txnCtx.sessionContext().sessionUser(), txnCtx.sessionContext().searchPath());
            } catch (Exception e) {
                if (ignoreUnavailable && e instanceof ResourceUnknownException) {
                    LOGGER.info("Ignore unknown relation '{}' for the '{}' snapshot'", table.getName(), createSnapshot.snapshot());
                    continue;
                } else {
                    throw e;
                }
            }
            if (docTableInfo.isPartitioned()) {
                templates.add(PartitionName.templateName(docTableInfo.ident().schema(), docTableInfo.ident().name()));
            }
            if (table.partitionProperties().isEmpty()) {
                snapshotIndices.addAll(Arrays.asList(docTableInfo.concreteIndices()));
            } else {
                var partitionName = toPartitionName(docTableInfo, Lists2.map(table.partitionProperties(), x -> x.map(eval)));
                if (!docTableInfo.partitions().contains(partitionName)) {
                    if (!ignoreUnavailable) {
                        throw new PartitionUnknownException(partitionName);
                    } else {
                        LOGGER.info("ignoring unknown partition of table '{}' with ident '{}'", partitionName.relationName(), partitionName.ident());
                    }
                } else {
                    snapshotIndices.add(partitionName.asIndexName());
                }
            }
        }
    }
    return new CreateSnapshotRequest(createSnapshot.snapshot().getRepository(), createSnapshot.snapshot().getSnapshotId().getName()).includeGlobalState(createSnapshot.tables().isEmpty()).waitForCompletion(WAIT_FOR_COMPLETION.get(settings)).indices(snapshotIndices.toArray(new String[0])).indicesOptions(IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, IndicesOptions.lenientExpandOpen())).templates(templates.stream().toList()).settings(settings);
}
Also used : Arrays(java.util.Arrays) Operation(io.crate.metadata.table.Operation) SnapshotInfo(org.elasticsearch.snapshots.SnapshotInfo) Function(java.util.function.Function) PartitionName(io.crate.metadata.PartitionName) DependencyCarrier(io.crate.planner.DependencyCarrier) HashSet(java.util.HashSet) WAIT_FOR_COMPLETION(io.crate.analyze.SnapshotSettings.WAIT_FOR_COMPLETION) SymbolEvaluator(io.crate.analyze.SymbolEvaluator) Settings(org.elasticsearch.common.settings.Settings) AnalyzedCreateSnapshot(io.crate.analyze.AnalyzedCreateSnapshot) IndicesOptions(org.elasticsearch.action.support.IndicesOptions) GenericPropertiesConverter(io.crate.analyze.GenericPropertiesConverter) SnapshotState(org.elasticsearch.snapshots.SnapshotState) OneRowActionListener(io.crate.execution.support.OneRowActionListener) PartitionUnknownException(io.crate.exceptions.PartitionUnknownException) PartitionPropertiesAnalyzer.toPartitionName(io.crate.analyze.PartitionPropertiesAnalyzer.toPartitionName) SchemaInfo(io.crate.metadata.table.SchemaInfo) DocTableInfo(io.crate.metadata.doc.DocTableInfo) TableInfo(io.crate.metadata.table.TableInfo) NodeContext(io.crate.metadata.NodeContext) CreateSnapshotException(io.crate.exceptions.CreateSnapshotException) Table(io.crate.sql.tree.Table) ResourceUnknownException(io.crate.exceptions.ResourceUnknownException) CreateSnapshotRequest(org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest) Lists2(io.crate.common.collections.Lists2) SnapshotSettings(io.crate.analyze.SnapshotSettings) RowConsumer(io.crate.data.RowConsumer) Logger(org.apache.logging.log4j.Logger) Row(io.crate.data.Row) Symbol(io.crate.expression.symbol.Symbol) PlannerContext(io.crate.planner.PlannerContext) IGNORE_UNAVAILABLE(io.crate.analyze.SnapshotSettings.IGNORE_UNAVAILABLE) Plan(io.crate.planner.Plan) SubQueryResults(io.crate.planner.operators.SubQueryResults) Schemas(io.crate.metadata.Schemas) VisibleForTesting(io.crate.common.annotations.VisibleForTesting) LogManager(org.apache.logging.log4j.LogManager) Row1(io.crate.data.Row1) CoordinatorTxnCtx(io.crate.metadata.CoordinatorTxnCtx) DocTableInfo(io.crate.metadata.doc.DocTableInfo) PartitionUnknownException(io.crate.exceptions.PartitionUnknownException) Symbol(io.crate.expression.symbol.Symbol) ResourceUnknownException(io.crate.exceptions.ResourceUnknownException) PartitionUnknownException(io.crate.exceptions.PartitionUnknownException) CreateSnapshotException(io.crate.exceptions.CreateSnapshotException) ResourceUnknownException(io.crate.exceptions.ResourceUnknownException) CreateSnapshotRequest(org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest) DocTableInfo(io.crate.metadata.doc.DocTableInfo) TableInfo(io.crate.metadata.table.TableInfo) Settings(org.elasticsearch.common.settings.Settings) SnapshotSettings(io.crate.analyze.SnapshotSettings) HashSet(java.util.HashSet) SchemaInfo(io.crate.metadata.table.SchemaInfo) VisibleForTesting(io.crate.common.annotations.VisibleForTesting)

Aggregations

CreateSnapshotRequest (org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest)12 CrateDummyClusterServiceUnitTest (io.crate.test.integration.CrateDummyClusterServiceUnitTest)6 Test (org.junit.Test)6 IndicesOptions (org.elasticsearch.action.support.IndicesOptions)4 CreateSnapshotException (io.crate.exceptions.CreateSnapshotException)3 Settings (org.elasticsearch.common.settings.Settings)3 AnalyzedCreateSnapshot (io.crate.analyze.AnalyzedCreateSnapshot)2 GenericPropertiesConverter (io.crate.analyze.GenericPropertiesConverter)2 PartitionPropertiesAnalyzer.toPartitionName (io.crate.analyze.PartitionPropertiesAnalyzer.toPartitionName)2 SnapshotSettings (io.crate.analyze.SnapshotSettings)2 IGNORE_UNAVAILABLE (io.crate.analyze.SnapshotSettings.IGNORE_UNAVAILABLE)2 WAIT_FOR_COMPLETION (io.crate.analyze.SnapshotSettings.WAIT_FOR_COMPLETION)2 SymbolEvaluator (io.crate.analyze.SymbolEvaluator)2 VisibleForTesting (io.crate.common.annotations.VisibleForTesting)2 Lists2 (io.crate.common.collections.Lists2)2 Row (io.crate.data.Row)2 Row1 (io.crate.data.Row1)2 RowConsumer (io.crate.data.RowConsumer)2 PartitionUnknownException (io.crate.exceptions.PartitionUnknownException)2 ResourceUnknownException (io.crate.exceptions.ResourceUnknownException)2