use of org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY in project elasticsearch by elastic.
the class IndexShardTests method testTranslogRecoverySyncsTranslog.
public void testTranslogRecoverySyncsTranslog() throws IOException {
Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).build();
IndexMetaData metaData = IndexMetaData.builder("test").putMapping("test", "{ \"properties\": { \"foo\": { \"type\": \"text\"}}}").settings(settings).primaryTerm(0, 1).build();
IndexShard primary = newShard(new ShardId(metaData.getIndex(), 0), true, "n1", metaData, null);
recoveryShardFromStore(primary);
indexDoc(primary, "test", "0", "{\"foo\" : \"bar\"}");
IndexShard replica = newShard(primary.shardId(), false, "n2", metaData, null);
recoverReplica(replica, primary, (shard, discoveryNode) -> new RecoveryTarget(shard, discoveryNode, recoveryListener, aLong -> {
}) {
@Override
public void indexTranslogOperations(List<Translog.Operation> operations, int totalTranslogOps) {
super.indexTranslogOperations(operations, totalTranslogOps);
assertFalse(replica.getTranslog().syncNeeded());
}
}, true);
closeShards(primary, replica);
}
use of org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY in project elasticsearch by elastic.
the class IndexShardTests method testShardActiveDuringPeerRecovery.
public void testShardActiveDuringPeerRecovery() throws IOException {
Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).build();
IndexMetaData metaData = IndexMetaData.builder("test").putMapping("test", "{ \"properties\": { \"foo\": { \"type\": \"text\"}}}").settings(settings).primaryTerm(0, 1).build();
IndexShard primary = newShard(new ShardId(metaData.getIndex(), 0), true, "n1", metaData, null);
recoveryShardFromStore(primary);
indexDoc(primary, "test", "0", "{\"foo\" : \"bar\"}");
IndexShard replica = newShard(primary.shardId(), false, "n2", metaData, null);
DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
replica.markAsRecovering("for testing", new RecoveryState(replica.routingEntry(), localNode, localNode));
// Shard is still inactive since we haven't started recovering yet
assertFalse(replica.isActive());
recoverReplica(replica, primary, (shard, discoveryNode) -> new RecoveryTarget(shard, discoveryNode, recoveryListener, aLong -> {
}) {
@Override
public void prepareForTranslogOperations(int totalTranslogOps, long maxUnsafeAutoIdTimestamp) throws IOException {
super.prepareForTranslogOperations(totalTranslogOps, maxUnsafeAutoIdTimestamp);
// Shard is still inactive since we haven't started recovering yet
assertFalse(replica.isActive());
}
@Override
public void indexTranslogOperations(List<Translog.Operation> operations, int totalTranslogOps) {
super.indexTranslogOperations(operations, totalTranslogOps);
// Shard should now be active since we did recover:
assertTrue(replica.isActive());
}
}, false);
closeShards(primary, replica);
}
use of org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY in project crate by crate.
the class InternalEngineTests method testNoOps.
/*
* This test tests that a no-op does not generate a new sequence number, that no-ops can advance the local checkpoint, and that no-ops
* are correctly added to the translog.
*/
@Test
public void testNoOps() throws IOException {
engine.close();
InternalEngine noOpEngine = null;
final int maxSeqNo = randomIntBetween(0, 128);
final int localCheckpoint = randomIntBetween(0, maxSeqNo);
try {
final BiFunction<Long, Long, LocalCheckpointTracker> supplier = (ms, lcp) -> new LocalCheckpointTracker(maxSeqNo, localCheckpoint);
EngineConfig noopEngineConfig = copy(engine.config(), new SoftDeletesRetentionMergePolicy(Lucene.SOFT_DELETES_FIELD, () -> new MatchAllDocsQuery(), engine.config().getMergePolicy()));
noOpEngine = new InternalEngine(noopEngineConfig, supplier) {
@Override
protected long doGenerateSeqNoForOperation(Operation operation) {
throw new UnsupportedOperationException();
}
};
noOpEngine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
final int gapsFilled = noOpEngine.fillSeqNoGaps(primaryTerm.get());
final String reason = "filling gaps";
noOpEngine.noOp(new Engine.NoOp(maxSeqNo + 1, primaryTerm.get(), LOCAL_TRANSLOG_RECOVERY, System.nanoTime(), reason));
assertThat(noOpEngine.getProcessedLocalCheckpoint(), equalTo((long) (maxSeqNo + 1)));
assertThat(noOpEngine.getTranslog().stats().getUncommittedOperations(), equalTo(gapsFilled));
noOpEngine.noOp(new Engine.NoOp(maxSeqNo + 2, primaryTerm.get(), randomFrom(PRIMARY, REPLICA, PEER_RECOVERY), System.nanoTime(), reason));
assertThat(noOpEngine.getProcessedLocalCheckpoint(), equalTo((long) (maxSeqNo + 2)));
assertThat(noOpEngine.getTranslog().stats().getUncommittedOperations(), equalTo(gapsFilled + 1));
// skip to the op that we added to the translog
Translog.Operation op;
Translog.Operation last = null;
try (Translog.Snapshot snapshot = noOpEngine.getTranslog().newSnapshot()) {
while ((op = snapshot.next()) != null) {
last = op;
}
}
assertNotNull(last);
assertThat(last, instanceOf(Translog.NoOp.class));
final Translog.NoOp noOp = (Translog.NoOp) last;
assertThat(noOp.seqNo(), equalTo((long) (maxSeqNo + 2)));
assertThat(noOp.primaryTerm(), equalTo(primaryTerm.get()));
assertThat(noOp.reason(), equalTo(reason));
if (engine.engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
MapperService mapperService = createMapperService("test");
List<Translog.Operation> operationsFromLucene = readAllOperationsInLucene(noOpEngine, mapperService);
// fills n gap and 2 manual noop.
assertThat(operationsFromLucene, hasSize(maxSeqNo + 2 - localCheckpoint));
for (int i = 0; i < operationsFromLucene.size(); i++) {
assertThat(operationsFromLucene.get(i), equalTo(new Translog.NoOp(localCheckpoint + 1 + i, primaryTerm.get(), "filling gaps")));
}
assertConsistentHistoryBetweenTranslogAndLuceneIndex(noOpEngine, mapperService);
}
} finally {
IOUtils.close(noOpEngine);
}
}
use of org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY in project crate by crate.
the class InternalEngineTests method testConcurrentGetAndSetOnPrimary.
@Test
public void testConcurrentGetAndSetOnPrimary() throws IOException, InterruptedException {
Thread[] thread = new Thread[randomIntBetween(3, 5)];
CountDownLatch startGun = new CountDownLatch(thread.length);
final int opsPerThread = randomIntBetween(10, 20);
class OpAndVersion {
final long version;
final String removed;
final String added;
OpAndVersion(long version, String removed, String added) {
this.version = version;
this.removed = removed;
this.added = added;
}
}
final AtomicInteger idGenerator = new AtomicInteger();
final Queue<OpAndVersion> history = ConcurrentCollections.newQueue();
ParsedDocument doc = testParsedDocument("1", null, testDocument(), bytesArray(""), null);
final Term uidTerm = newUid(doc);
engine.index(indexForDoc(doc));
final BiFunction<String, Engine.SearcherScope, Searcher> searcherFactory = engine::acquireSearcher;
for (int i = 0; i < thread.length; i++) {
thread[i] = new Thread(() -> {
startGun.countDown();
try {
startGun.await();
} catch (InterruptedException e) {
throw new AssertionError(e);
}
for (int op = 0; op < opsPerThread; op++) {
try (Engine.GetResult get = engine.get(new Engine.Get(doc.id(), uidTerm), searcherFactory)) {
FieldsVisitor visitor = new FieldsVisitor(true);
get.docIdAndVersion().reader.document(get.docIdAndVersion().docId, visitor);
List<String> values = new ArrayList<>(Strings.commaDelimitedListToSet(visitor.source().utf8ToString()));
String removed = op % 3 == 0 && values.size() > 0 ? values.remove(0) : null;
String added = "v_" + idGenerator.incrementAndGet();
values.add(added);
Engine.Index index = new Engine.Index(uidTerm, testParsedDocument("1", null, testDocument(), bytesArray(Strings.collectionToCommaDelimitedString(values)), null), UNASSIGNED_SEQ_NO, 2, get.docIdAndVersion().version, VersionType.INTERNAL, PRIMARY, System.currentTimeMillis(), -1, false, UNASSIGNED_SEQ_NO, 0);
Engine.IndexResult indexResult = engine.index(index);
if (indexResult.getResultType() == Engine.Result.Type.SUCCESS) {
history.add(new OpAndVersion(indexResult.getVersion(), removed, added));
}
} catch (IOException e) {
throw new AssertionError(e);
}
}
});
thread[i].start();
}
for (int i = 0; i < thread.length; i++) {
thread[i].join();
}
List<OpAndVersion> sortedHistory = new ArrayList<>(history);
sortedHistory.sort(Comparator.comparing(o -> o.version));
Set<String> currentValues = new HashSet<>();
for (int i = 0; i < sortedHistory.size(); i++) {
OpAndVersion op = sortedHistory.get(i);
if (i > 0) {
assertThat("duplicate version", op.version, not(equalTo(sortedHistory.get(i - 1).version)));
}
boolean exists = op.removed == null ? true : currentValues.remove(op.removed);
assertTrue(op.removed + " should exist", exists);
exists = currentValues.add(op.added);
assertTrue(op.added + " should not exist", exists);
}
try (Engine.GetResult get = engine.get(new Engine.Get(doc.id(), uidTerm), searcherFactory)) {
FieldsVisitor visitor = new FieldsVisitor(true);
get.docIdAndVersion().reader.document(get.docIdAndVersion().docId, visitor);
List<String> values = Arrays.asList(Strings.commaDelimitedListToStringArray(visitor.source().utf8ToString()));
assertThat(currentValues, equalTo(new HashSet<>(values)));
}
}
Aggregations