Examples with IOContext org.apache.lucene.store.IOContext used on opensource projects

Example 26 with IOContext

use of org.apache.lucene.store.IOContext in project jackrabbit-oak by apache.

the class IndexCopierTest method deleteCorruptedFile.

@Test
public void deleteCorruptedFile() throws Exception {
    Directory baseDir = new RAMDirectory();
    IndexDefinition defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
    RAMIndexCopier c1 = new RAMIndexCopier(baseDir, sameThreadExecutor(), getWorkDir());
    Directory remote = new RAMDirectory() {

        @Override
        public IndexInput openInput(String name, IOContext context) throws IOException {
            throw new IllegalStateException("boom");
        }
    };
    String fileName = "failed.txt";
    Directory wrapped = c1.wrapForRead("/foo", defn, remote, INDEX_DATA_CHILD_NAME);
    byte[] t1 = writeFile(remote, fileName);
    try {
        readAndAssert(wrapped, fileName, t1);
        fail("Read of file should have failed");
    } catch (IllegalStateException ignore) {
    }
    assertFalse(c1.baseDir.fileExists(fileName));
}

Example 27 with IOContext

use of org.apache.lucene.store.IOContext in project jackrabbit-oak by apache.

the class IndexCopierTest method copyInProgressStats.

@Test
public void copyInProgressStats() throws Exception {
    Directory baseDir = new RAMDirectory();
    IndexDefinition defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
    final List<ListenableFuture<?>> submittedTasks = Lists.newArrayList();
    ExecutorService executor = new ForwardingListeningExecutorService() {

        @Override
        protected ListeningExecutorService delegate() {
            return MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor());
        }

        @Override
        public void execute(Runnable command) {
            submittedTasks.add(super.submit(command));
        }
    };
    IndexCopier c1 = new RAMIndexCopier(baseDir, executor, getWorkDir());
    final CountDownLatch copyProceed = new CountDownLatch(1);
    final CountDownLatch copyRequestArrived = new CountDownLatch(1);
    TestRAMDirectory remote = new TestRAMDirectory() {

        @Override
        public void copy(Directory to, String src, String dest, IOContext context) throws IOException {
            copyRequestArrived.countDown();
            try {
                copyProceed.await();
            } catch (InterruptedException e) {
            }
            super.copy(to, src, dest, context);
        }
    };
    Directory wrapped = c1.wrapForRead("/foo", defn, remote, INDEX_DATA_CHILD_NAME);
    byte[] t1 = writeFile(remote, "t1");
    //1. Trigger a read which should go to remote
    readAndAssert(wrapped, "t1", t1);
    copyRequestArrived.await();
    assertEquals(1, c1.getCopyInProgressCount());
    assertEquals(1, remote.openedFiles.size());
    //2. Trigger another read and this should also be
    //served from remote
    readAndAssert(wrapped, "t1", t1);
    assertEquals(1, c1.getCopyInProgressCount());
    assertEquals(IOUtils.humanReadableByteCount(t1.length), c1.getCopyInProgressSize());
    assertEquals(1, c1.getCopyInProgressDetails().length);
    System.out.println(Arrays.toString(c1.getCopyInProgressDetails()));
    assertEquals(2, remote.openedFiles.size());
    //3. Perform copy
    copyProceed.countDown();
    Futures.allAsList(submittedTasks).get();
    remote.reset();
    //4. Now read again after copy is done
    readAndAssert(wrapped, "t1", t1);
    // Now read should be served from local and not from remote
    assertEquals(0, remote.openedFiles.size());
    assertEquals(0, c1.getCopyInProgressCount());
    executor.shutdown();
}

Example 28 with IOContext

use of org.apache.lucene.store.IOContext in project jackrabbit-oak by apache.

the class IndexCopierTest method cowFailureInCopy.

@Test
public void cowFailureInCopy() throws Exception {
    ExecutorService executorService = Executors.newFixedThreadPool(2);
    Directory baseDir = new CloseSafeDir();
    IndexDefinition defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
    IndexCopier copier = new RAMIndexCopier(baseDir, executorService, getWorkDir());
    final Set<String> toFail = Sets.newHashSet();
    Directory remote = new CloseSafeDir() {

        @Override
        public IndexOutput createOutput(String name, IOContext context) throws IOException {
            if (toFail.contains(name)) {
                throw new RuntimeException("Failing copy for " + name);
            }
            return super.createOutput(name, context);
        }
    };
    final Directory local = copier.wrapForWrite(defn, remote, false, INDEX_DATA_CHILD_NAME);
    toFail.add("t2");
    byte[] t1 = writeFile(local, "t1");
    byte[] t2 = writeFile(local, "t2");
    try {
        local.close();
        fail();
    } catch (IOException ignore) {
    }
    executorService.shutdown();
}

Example 29 with IOContext

use of org.apache.lucene.store.IOContext in project jackrabbit-oak by apache.

the class IndexCopierTest method cowConcurrentAccess.

/**
     * Test the interaction between COR and COW using same underlying directory
     */
@Test
public void cowConcurrentAccess() throws Exception {
    CollectingExecutor executor = new CollectingExecutor();
    ExecutorService executorService = Executors.newFixedThreadPool(2);
    executor.setForwardingExecutor(executorService);
    Directory baseDir = new CloseSafeDir();
    IndexDefinition defn = new IndexDefinition(root, builder.getNodeState(), indexPath);
    IndexCopier copier = new RAMIndexCopier(baseDir, executor, getWorkDir(), true);
    Directory remote = new CloseSafeDir();
    byte[] f1 = writeFile(remote, "f1");
    Directory cor1 = copier.wrapForRead(indexPath, defn, remote, INDEX_DATA_CHILD_NAME);
    readAndAssert(cor1, "f1", f1);
    cor1.close();
    final CountDownLatch pauseCopyLatch = new CountDownLatch(1);
    Directory remote2 = new FilterDirectory(remote) {

        @Override
        public IndexOutput createOutput(String name, IOContext context) throws IOException {
            try {
                pauseCopyLatch.await();
            } catch (InterruptedException ignore) {
            }
            return super.createOutput(name, context);
        }
    };
    //Start copying a file to remote via COW
    Directory cow1 = copier.wrapForWrite(defn, remote2, false, INDEX_DATA_CHILD_NAME);
    byte[] f2 = writeFile(cow1, "f2");
    //Before copy is done to remote lets delete f1 from remote and
    //open a COR and close it such that it triggers delete of f1
    remote.deleteFile("f1");
    Directory cor2 = copier.wrapForRead(indexPath, defn, remote, INDEX_DATA_CHILD_NAME);
    //Ensure that deletion task submitted to executor get processed immediately
    executor.enableImmediateExecution();
    cor2.close();
    executor.enableDelayedExecution();
    assertFalse(baseDir.fileExists("f1"));
    assertFalse("f2 should not have been copied to remote so far", remote.fileExists("f2"));
    assertTrue("f2 should exist", baseDir.fileExists("f2"));
    pauseCopyLatch.countDown();
    cow1.close();
    assertTrue("f2 should exist", remote.fileExists("f2"));
    executorService.shutdown();
}

Example 30 with IOContext

use of org.apache.lucene.store.IOContext in project jackrabbit-oak by apache.

the class IndexCopierTest method cowPoolClosedWithTaskInQueue.

@Test
public void cowPoolClosedWithTaskInQueue() throws Exception {
    ExecutorService executorService = Executors.newFixedThreadPool(2);
    Directory baseDir = new CloseSafeDir();
    IndexDefinition defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
    IndexCopier copier = new RAMIndexCopier(baseDir, executorService, getWorkDir());
    final Set<String> toPause = Sets.newHashSet();
    final CountDownLatch pauseCopyLatch = new CountDownLatch(1);
    Directory remote = new CloseSafeDir() {

        @Override
        public IndexOutput createOutput(String name, IOContext context) throws IOException {
            if (toPause.contains(name)) {
                try {
                    pauseCopyLatch.await();
                } catch (InterruptedException ignore) {
                }
            }
            return super.createOutput(name, context);
        }
    };
    final Directory local = copier.wrapForWrite(defn, remote, false, INDEX_DATA_CHILD_NAME);
    toPause.add("t2");
    byte[] t1 = writeFile(local, "t1");
    byte[] t2 = writeFile(local, "t2");
    byte[] t3 = writeFile(local, "t3");
    byte[] t4 = writeFile(local, "t4");
    final AtomicReference<Throwable> error = new AtomicReference<Throwable>();
    Thread closer = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                local.close();
            } catch (Throwable e) {
                e.printStackTrace();
                error.set(e);
            }
        }
    });
    closer.start();
    copier.close();
    executorService.shutdown();
    executorService.awaitTermination(100, TimeUnit.MILLISECONDS);
    pauseCopyLatch.countDown();
    closer.join();
    assertNotNull("Close should have thrown an exception", error.get());
}