Examples with Location org.elasticsearch.index.translog.Translog.Location used on opensource projects

Example 1 with Location

use of org.elasticsearch.index.translog.Translog.Location in project elasticsearch by elastic.

the class TranslogTests method testTranslogChecksums.

public void testTranslogChecksums() throws Exception {
    List<Translog.Location> locations = new ArrayList<>();
    int translogOperations = randomIntBetween(10, 100);
    for (int op = 0; op < translogOperations; op++) {
        String ascii = randomAsciiOfLengthBetween(1, 50);
        locations.add(translog.add(new Translog.Index("test", "" + op, ascii.getBytes("UTF-8"))));
    }
    translog.sync();
    corruptTranslogs(translogDir);
    AtomicInteger corruptionsCaught = new AtomicInteger(0);
    Translog.Snapshot snapshot = translog.newSnapshot();
    for (Translog.Location location : locations) {
        try {
            Translog.Operation next = snapshot.next();
            assertNotNull(next);
        } catch (TranslogCorruptedException e) {
            corruptionsCaught.incrementAndGet();
        }
    }
    expectThrows(TranslogCorruptedException.class, () -> snapshot.next());
    assertThat("at least one corruption was caused and caught", corruptionsCaught.get(), greaterThanOrEqualTo(1));
}

Example 2 with Location

use of org.elasticsearch.index.translog.Translog.Location in project elasticsearch by elastic.

the class TranslogTests method testTruncatedTranslogs.

public void testTruncatedTranslogs() throws Exception {
    List<Translog.Location> locations = new ArrayList<>();
    int translogOperations = randomIntBetween(10, 100);
    for (int op = 0; op < translogOperations; op++) {
        String ascii = randomAsciiOfLengthBetween(1, 50);
        locations.add(translog.add(new Translog.Index("test", "" + op, ascii.getBytes("UTF-8"))));
    }
    translog.sync();
    truncateTranslogs(translogDir);
    AtomicInteger truncations = new AtomicInteger(0);
    Translog.Snapshot snap = translog.newSnapshot();
    for (Translog.Location location : locations) {
        try {
            assertNotNull(snap.next());
        } catch (EOFException e) {
            truncations.incrementAndGet();
        }
    }
    assertThat("at least one truncation was caused and caught", truncations.get(), greaterThanOrEqualTo(1));
}

Example 3 with Location

use of org.elasticsearch.index.translog.Translog.Location in project elasticsearch by elastic.

the class TranslogTests method testSyncUpToStream.

public void testSyncUpToStream() throws IOException {
    int iters = randomIntBetween(5, 10);
    for (int i = 0; i < iters; i++) {
        int translogOperations = randomIntBetween(10, 100);
        int count = 0;
        ArrayList<Location> locations = new ArrayList<>();
        for (int op = 0; op < translogOperations; op++) {
            if (rarely()) {
                // do this first so that there is at least one pending tlog entry
                translog.commit();
            }
            final Translog.Location location = translog.add(new Translog.Index("test", "" + op, Integer.toString(++count).getBytes(Charset.forName("UTF-8"))));
            locations.add(location);
        }
        Collections.shuffle(locations, random());
        if (randomBoolean()) {
            assertTrue("at least one operation pending", translog.syncNeeded());
            assertTrue("this operation has not been synced", translog.ensureSynced(locations.stream()));
            // we are the last location so everything should be synced
            assertFalse("the last call to ensureSycned synced all previous ops", translog.syncNeeded());
        } else if (rarely()) {
            translog.commit();
            // not syncing now
            assertFalse("location is from a previous translog - already synced", translog.ensureSynced(locations.stream()));
            assertFalse("no sync needed since no operations in current translog", translog.syncNeeded());
        } else {
            translog.sync();
            assertFalse("translog has been synced already", translog.ensureSynced(locations.stream()));
        }
        for (Location location : locations) {
            assertFalse("all of the locations should be synced: " + location, translog.ensureSynced(location));
        }
    }
}

Example 4 with Location

use of org.elasticsearch.index.translog.Translog.Location in project elasticsearch by elastic.

the class TranslogTests method testLocationComparison.

public void testLocationComparison() throws IOException {
    List<Translog.Location> locations = new ArrayList<>();
    int translogOperations = randomIntBetween(10, 100);
    int count = 0;
    for (int op = 0; op < translogOperations; op++) {
        locations.add(translog.add(new Translog.Index("test", "" + op, Integer.toString(++count).getBytes(Charset.forName("UTF-8")))));
        if (rarely() && translogOperations > op + 1) {
            translog.commit();
        }
    }
    Collections.shuffle(locations, random());
    Translog.Location max = locations.get(0);
    for (Translog.Location location : locations) {
        max = max(max, location);
    }
    assertEquals(max.generation, translog.currentFileGeneration());
    Translog.Snapshot snap = translog.newSnapshot();
    Translog.Operation next;
    Translog.Operation maxOp = null;
    while ((next = snap.next()) != null) {
        maxOp = next;
    }
    assertNotNull(maxOp);
    assertEquals(maxOp.getSource().source.utf8ToString(), Integer.toString(count));
}

Example 5 with Location

use of org.elasticsearch.index.translog.Translog.Location in project elasticsearch by elastic.

the class TranslogTests method testConcurrentWriteViewsAndSnapshot.

/**
     * Tests that concurrent readers and writes maintain view and snapshot semantics
     */
public void testConcurrentWriteViewsAndSnapshot() throws Throwable {
    final Thread[] writers = new Thread[randomIntBetween(1, 10)];
    final Thread[] readers = new Thread[randomIntBetween(1, 10)];
    final int flushEveryOps = randomIntBetween(5, 100);
    // used to notify main thread that so many operations have been written so it can simulate a flush
    final AtomicReference<CountDownLatch> writtenOpsLatch = new AtomicReference<>(new CountDownLatch(0));
    final AtomicLong idGenerator = new AtomicLong();
    final CyclicBarrier barrier = new CyclicBarrier(writers.length + readers.length + 1);
    // a map of all written ops and their returned location.
    final Map<Translog.Operation, Translog.Location> writtenOps = ConcurrentCollections.newConcurrentMap();
    // a signal for all threads to stop
    final AtomicBoolean run = new AtomicBoolean(true);
    // any errors on threads
    final List<Exception> errors = new CopyOnWriteArrayList<>();
    logger.debug("using [{}] readers. [{}] writers. flushing every ~[{}] ops.", readers.length, writers.length, flushEveryOps);
    for (int i = 0; i < writers.length; i++) {
        final String threadName = "writer_" + i;
        final int threadId = i;
        writers[i] = new Thread(new AbstractRunnable() {

            @Override
            public void doRun() throws BrokenBarrierException, InterruptedException, IOException {
                barrier.await();
                int counter = 0;
                while (run.get()) {
                    long id = idGenerator.incrementAndGet();
                    final Translog.Operation op;
                    final Translog.Operation.Type type = Translog.Operation.Type.values()[((int) (id % Translog.Operation.Type.values().length))];
                    switch(type) {
                        case CREATE:
                        case INDEX:
                            op = new Translog.Index("type", "" + id, new byte[] { (byte) id });
                            break;
                        case DELETE:
                            op = new Translog.Delete(newUid("" + id));
                            break;
                        case NO_OP:
                            op = new Translog.NoOp(id, id, Long.toString(id));
                            break;
                        default:
                            throw new AssertionError("unsupported operation type [" + type + "]");
                    }
                    Translog.Location location = translog.add(op);
                    Translog.Location existing = writtenOps.put(op, location);
                    if (existing != null) {
                        fail("duplicate op [" + op + "], old entry at " + location);
                    }
                    if (id % writers.length == threadId) {
                        translog.ensureSynced(location);
                    }
                    writtenOpsLatch.get().countDown();
                    counter++;
                }
                logger.debug("--> [{}] done. wrote [{}] ops.", threadName, counter);
            }

            @Override
            public void onFailure(Exception e) {
                logger.error((Supplier<?>) () -> new ParameterizedMessage("--> writer [{}] had an error", threadName), e);
                errors.add(e);
            }
        }, threadName);
        writers[i].start();
    }
    for (int i = 0; i < readers.length; i++) {
        final String threadId = "reader_" + i;
        readers[i] = new Thread(new AbstractRunnable() {

            Translog.View view = null;

            Set<Translog.Operation> writtenOpsAtView;

            @Override
            public void onFailure(Exception e) {
                logger.error((Supplier<?>) () -> new ParameterizedMessage("--> reader [{}] had an error", threadId), e);
                errors.add(e);
                try {
                    closeView();
                } catch (IOException inner) {
                    inner.addSuppressed(e);
                    logger.error("unexpected error while closing view, after failure", inner);
                }
            }

            void closeView() throws IOException {
                if (view != null) {
                    view.close();
                }
            }

            void newView() throws IOException {
                closeView();
                view = translog.newView();
                // captures the currently written ops so we know what to expect from the view
                writtenOpsAtView = new HashSet<>(writtenOps.keySet());
                logger.debug("--> [{}] opened view from [{}]", threadId, view.minTranslogGeneration());
            }

            @Override
            protected void doRun() throws Exception {
                barrier.await();
                int iter = 0;
                while (run.get()) {
                    if (iter++ % 10 == 0) {
                        newView();
                    }
                    // captures al views that are written since the view was created (with a small caveat see bellow)
                    // these are what we expect the snapshot to return (and potentially some more).
                    Set<Translog.Operation> expectedOps = new HashSet<>(writtenOps.keySet());
                    expectedOps.removeAll(writtenOpsAtView);
                    Translog.Snapshot snapshot = view.snapshot();
                    Translog.Operation op;
                    while ((op = snapshot.next()) != null) {
                        expectedOps.remove(op);
                    }
                    if (expectedOps.isEmpty() == false) {
                        StringBuilder missed = new StringBuilder("missed ").append(expectedOps.size()).append(" operations");
                        boolean failed = false;
                        for (Translog.Operation expectedOp : expectedOps) {
                            final Translog.Location loc = writtenOps.get(expectedOp);
                            if (loc.generation < view.minTranslogGeneration()) {
                                // may yet be available in writtenOpsAtView, meaning we will erroneously expect them
                                continue;
                            }
                            failed = true;
                            missed.append("\n --> [").append(expectedOp).append("] written at ").append(loc);
                        }
                        if (failed) {
                            fail(missed.toString());
                        }
                    }
                    // slow down things a bit and spread out testing..
                    writtenOpsLatch.get().await(200, TimeUnit.MILLISECONDS);
                }
                closeView();
                logger.debug("--> [{}] done. tested [{}] snapshots", threadId, iter);
            }
        }, threadId);
        readers[i].start();
    }
    barrier.await();
    try {
        for (int iterations = scaledRandomIntBetween(10, 200); iterations > 0 && errors.isEmpty(); iterations--) {
            writtenOpsLatch.set(new CountDownLatch(flushEveryOps));
            while (writtenOpsLatch.get().await(200, TimeUnit.MILLISECONDS) == false) {
                if (errors.size() > 0) {
                    break;
                }
            }
            translog.commit();
        }
    } finally {
        run.set(false);
        logger.debug("--> waiting for threads to stop");
        for (Thread thread : writers) {
            thread.join();
        }
        for (Thread thread : readers) {
            thread.join();
        }
        if (errors.size() > 0) {
            Throwable e = errors.get(0);
            for (Throwable suppress : errors.subList(1, errors.size())) {
                e.addSuppressed(suppress);
            }
            throw e;
        }
        logger.info("--> test done. total ops written [{}]", writtenOps.size());
    }
}