Examples with DataStream org.opensearch.cluster.metadata.DataStream used on opensource projects

Example 1 with DataStream

use of org.opensearch.cluster.metadata.DataStream in project OpenSearch by opensearch-project.

the class GetDataStreamResponseTests method randomInstance.

private static DataStreamInfo randomInstance() {
    List<Index> indices = randomIndexInstances();
    long generation = indices.size() + randomLongBetween(1, 128);
    String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
    indices.add(new Index(getDefaultBackingIndexName(dataStreamName, generation), UUIDs.randomBase64UUID(random())));
    DataStream dataStream = new DataStream(dataStreamName, createTimestampField("@timestamp"), indices, generation);
    return new DataStreamInfo(dataStream, ClusterHealthStatus.YELLOW, randomAlphaOfLengthBetween(2, 10));
}

Example 2 with DataStream

use of org.opensearch.cluster.metadata.DataStream in project OpenSearch by opensearch-project.

the class GetDataStreamResponseTests method assertInstances.

@Override
protected void assertInstances(GetDataStreamAction.Response serverTestInstance, GetDataStreamResponse clientInstance) {
    assertEquals(serverTestInstance.getDataStreams().size(), clientInstance.getDataStreams().size());
    Iterator<DataStreamInfo> serverIt = serverTestInstance.getDataStreams().iterator();
    Iterator<org.opensearch.client.indices.DataStream> clientIt = clientInstance.getDataStreams().iterator();
    while (serverIt.hasNext()) {
        org.opensearch.client.indices.DataStream client = clientIt.next();
        DataStream server = serverIt.next().getDataStream();
        assertEquals(server.getName(), client.getName());
        assertEquals(server.getIndices().stream().map(Index::getName).collect(Collectors.toList()), client.getIndices());
        assertEquals(server.getTimeStampField().getName(), client.getTimeStampField());
        assertEquals(server.getGeneration(), client.getGeneration());
    }
}

Example 3 with DataStream

use of org.opensearch.cluster.metadata.DataStream in project OpenSearch by opensearch-project.

the class DataStreamRolloverIT method testDataStreamRollover.

public void testDataStreamRollover() throws Exception {
    createDataStreamIndexTemplate("demo-template", Collections.singletonList("logs-*"));
    createDataStream("logs-demo");
    DataStream dataStream;
    GetDataStreamAction.Response.DataStreamInfo dataStreamInfo;
    GetDataStreamAction.Response response;
    // Data stream before a rollover.
    response = getDataStreams("logs-demo");
    dataStreamInfo = response.getDataStreams().get(0);
    assertThat(dataStreamInfo.getIndexTemplate(), equalTo("demo-template"));
    dataStream = dataStreamInfo.getDataStream();
    assertThat(dataStream.getGeneration(), equalTo(1L));
    assertThat(dataStream.getIndices().size(), equalTo(1));
    assertThat(dataStream.getTimeStampField(), equalTo(new DataStream.TimestampField("@timestamp")));
    assertThat(dataStream.getIndices().stream().map(Index::getName).collect(Collectors.toList()), containsInAnyOrder(".ds-logs-demo-000001"));
    // Perform a rollover.
    RolloverResponse rolloverResponse = rolloverDataStream("logs-demo");
    assertThat(rolloverResponse.getOldIndex(), equalTo(".ds-logs-demo-000001"));
    assertThat(rolloverResponse.getNewIndex(), equalTo(".ds-logs-demo-000002"));
    // Data stream after a rollover.
    response = getDataStreams("logs-demo");
    dataStreamInfo = response.getDataStreams().get(0);
    assertThat(dataStreamInfo.getIndexTemplate(), equalTo("demo-template"));
    dataStream = dataStreamInfo.getDataStream();
    assertThat(dataStream.getGeneration(), equalTo(2L));
    assertThat(dataStream.getIndices().size(), equalTo(2));
    assertThat(dataStream.getTimeStampField(), equalTo(new DataStream.TimestampField("@timestamp")));
    assertThat(dataStream.getIndices().stream().map(Index::getName).collect(Collectors.toList()), containsInAnyOrder(".ds-logs-demo-000001", ".ds-logs-demo-000002"));
}

Example 4 with DataStream

use of org.opensearch.cluster.metadata.DataStream in project OpenSearch by opensearch-project.

the class TransportBulkAction method prohibitAppendWritesInBackingIndices.

static void prohibitAppendWritesInBackingIndices(DocWriteRequest<?> writeRequest, Metadata metadata) {
    IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(writeRequest.index());
    if (indexAbstraction == null) {
        return;
    }
    if (indexAbstraction.getType() != IndexAbstraction.Type.CONCRETE_INDEX) {
        return;
    }
    if (indexAbstraction.getParentDataStream() == null) {
        return;
    }
    DataStream dataStream = indexAbstraction.getParentDataStream().getDataStream();
    // At this point with write op is targeting a backing index of a data stream directly,
    // so checking if write op is append-only and if so fail.
    // (Updates and deletes are allowed to target a backing index)
    DocWriteRequest.OpType opType = writeRequest.opType();
    // that an engine level change is needed and for now this check is sufficient.)
    if (opType == DocWriteRequest.OpType.CREATE) {
        throw new IllegalArgumentException("index request with op_type=create targeting backing indices is disallowed, " + "target corresponding data stream [" + dataStream.getName() + "] instead");
    }
    if (opType == DocWriteRequest.OpType.INDEX && writeRequest.ifPrimaryTerm() == UNASSIGNED_PRIMARY_TERM && writeRequest.ifSeqNo() == UNASSIGNED_SEQ_NO) {
        throw new IllegalArgumentException("index request with op_type=index and no if_primary_term and if_seq_no set " + "targeting backing indices is disallowed, target corresponding data stream [" + dataStream.getName() + "] instead");
    }
}

Example 5 with DataStream

use of org.opensearch.cluster.metadata.DataStream in project OpenSearch by opensearch-project.

the class SnapshotsService method snapshottingDataStreams.

/**
 * Returns the data streams that are currently being snapshotted (with partial == false) and that are contained in the
 * indices-to-check set.
 */
public static Set<String> snapshottingDataStreams(final ClusterState currentState, final Set<String> dataStreamsToCheck) {
    final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
    if (snapshots == null) {
        return emptySet();
    }
    Map<String, DataStream> dataStreams = currentState.metadata().dataStreams();
    return snapshots.entries().stream().filter(e -> e.partial() == false).flatMap(e -> e.dataStreams().stream()).filter(ds -> dataStreams.containsKey(ds) && dataStreamsToCheck.contains(ds)).collect(Collectors.toSet());
}