Examples with HTableInterfaceReference org.apache.phoenix.hbase.index.table.HTableInterfaceReference used on opensource projects

Example 1 with HTableInterfaceReference

use of org.apache.phoenix.hbase.index.table.HTableInterfaceReference in project phoenix by apache.

the class TrackingParallelWriterIndexCommitter method write.

@Override
public void write(Multimap<HTableInterfaceReference, Mutation> toWrite, final boolean allowLocalUpdates) throws MultiIndexWriteFailureException {
    Set<Entry<HTableInterfaceReference, Collection<Mutation>>> entries = toWrite.asMap().entrySet();
    TaskBatch<Boolean> tasks = new TaskBatch<Boolean>(entries.size());
    List<HTableInterfaceReference> tables = new ArrayList<HTableInterfaceReference>(entries.size());
    for (Entry<HTableInterfaceReference, Collection<Mutation>> entry : entries) {
        // get the mutations for each table. We leak the implementation here a little bit to save
        // doing a complete copy over of all the index update for each table.
        final List<Mutation> mutations = (List<Mutation>) entry.getValue();
        // track each reference so we can get at it easily later, when determing failures
        final HTableInterfaceReference tableReference = entry.getKey();
        final RegionCoprocessorEnvironment env = this.env;
        if (env != null && !allowLocalUpdates && tableReference.getTableName().equals(env.getRegion().getTableDesc().getNameAsString())) {
            continue;
        }
        tables.add(tableReference);
        /*
             * Write a batch of index updates to an index table. This operation stops (is cancelable) via two
             * mechanisms: (1) setting aborted or stopped on the IndexWriter or, (2) interrupting the running thread.
             * The former will only work if we are not in the midst of writing the current batch to the table, though we
             * do check these status variables before starting and before writing the batch. The latter usage,
             * interrupting the thread, will work in the previous situations as was at some points while writing the
             * batch, depending on the underlying writer implementation (HTableInterface#batch is blocking, but doesn't
             * elaborate when is supports an interrupt).
             */
        tasks.add(new Task<Boolean>() {

            /**
                 * Do the actual write to the primary table. We don't need to worry about closing the table because that
                 * is handled the {@link CachingHTableFactory}.
                 */
            @SuppressWarnings("deprecation")
            @Override
            public Boolean call() throws Exception {
                HTableInterface table = null;
                try {
                    // this may have been queued, but there was an abort/stop so we try to early exit
                    throwFailureIfDone();
                    if (allowLocalUpdates && env != null && tableReference.getTableName().equals(env.getRegion().getTableDesc().getNameAsString())) {
                        try {
                            throwFailureIfDone();
                            IndexUtil.writeLocalUpdates(env.getRegion(), mutations, true);
                            return Boolean.TRUE;
                        } catch (IOException ignord) {
                            // when it's failed we fall back to the standard & slow way
                            if (LOG.isTraceEnabled()) {
                                LOG.trace("indexRegion.batchMutate failed and fall back to HTable.batch(). Got error=" + ignord);
                            }
                        }
                    }
                    if (LOG.isTraceEnabled()) {
                        LOG.trace("Writing index update:" + mutations + " to table: " + tableReference);
                    }
                    table = factory.getTable(tableReference.get());
                    throwFailureIfDone();
                    table.batch(mutations);
                } catch (InterruptedException e) {
                    // reset the interrupt status on the thread
                    Thread.currentThread().interrupt();
                    throw e;
                } catch (Exception e) {
                    throw e;
                } finally {
                    if (table != null) {
                        table.close();
                    }
                }
                return Boolean.TRUE;
            }

            private void throwFailureIfDone() throws SingleIndexWriteFailureException {
                if (stopped.isStopped() || abortable.isAborted() || Thread.currentThread().isInterrupted()) {
                    throw new SingleIndexWriteFailureException("Pool closed, not attempting to write to the index!", null);
                }
            }
        });
    }
    List<Boolean> results = null;
    try {
        LOG.debug("Waiting on index update tasks to complete...");
        results = this.pool.submitUninterruptible(tasks);
    } catch (ExecutionException e) {
        throw new RuntimeException("Should not fail on the results while using a WaitForCompletionTaskRunner", e);
    } catch (EarlyExitFailure e) {
        throw new RuntimeException("Stopped while waiting for batch, quiting!", e);
    }
    // track the failures. We only ever access this on return from our calls, so no extra
    // synchronization is needed. We could update all the failures as we find them, but that add a
    // lot of locking overhead, and just doing the copy later is about as efficient.
    List<HTableInterfaceReference> failures = new ArrayList<HTableInterfaceReference>();
    int index = 0;
    for (Boolean result : results) {
        // there was a failure
        if (result == null) {
            // we know which table failed by the index of the result
            failures.add(tables.get(index));
        }
        index++;
    }
    // if any of the tasks failed, then we need to propagate the failure
    if (failures.size() > 0) {
        // make the list unmodifiable to avoid any more synchronization concerns
        throw new MultiIndexWriteFailureException(Collections.unmodifiableList(failures));
    }
    return;
}

Example 2 with HTableInterfaceReference

use of org.apache.phoenix.hbase.index.table.HTableInterfaceReference in project phoenix by apache.

the class RecoveryIndexWriter method resolveTableReferences.

/**
 * Convert the passed index updates to {@link HTableInterfaceReference}s.
 *
 * @param indexUpdates
 *            from the index builder
 * @return pairs that can then be written by an {@link RecoveryIndexWriter}.
 */
@Override
protected Multimap<HTableInterfaceReference, Mutation> resolveTableReferences(Collection<Pair<Mutation, byte[]>> indexUpdates) {
    Multimap<HTableInterfaceReference, Mutation> updates = ArrayListMultimap.<HTableInterfaceReference, Mutation>create();
    // simple map to make lookups easy while we build the map of tables to create
    Map<ImmutableBytesPtr, HTableInterfaceReference> tables = new HashMap<ImmutableBytesPtr, HTableInterfaceReference>(updates.size());
    for (Pair<Mutation, byte[]> entry : indexUpdates) {
        byte[] tableName = entry.getSecond();
        ImmutableBytesPtr ptr = new ImmutableBytesPtr(tableName);
        HTableInterfaceReference table = tables.get(ptr);
        if (nonExistingTablesList.contains(table)) {
            LOG.debug("Edits found for non existing table: " + table.getTableName() + " so skipping it!!");
            continue;
        }
        if (table == null) {
            table = new HTableInterfaceReference(ptr);
            tables.put(ptr, table);
        }
        updates.put(table, entry.getFirst());
    }
    return updates;
}

Example 3 with HTableInterfaceReference

use of org.apache.phoenix.hbase.index.table.HTableInterfaceReference in project phoenix by apache.

the class PhoenixIndexFailurePolicy method handleExceptionFromClient.

private static void handleExceptionFromClient(IndexWriteException indexWriteException, PhoenixConnection conn, PIndexState indexState) {
    try {
        Set<String> indexesToUpdate = new HashSet<>();
        if (indexWriteException instanceof MultiIndexWriteFailureException) {
            MultiIndexWriteFailureException indexException = (MultiIndexWriteFailureException) indexWriteException;
            List<HTableInterfaceReference> failedIndexes = indexException.getFailedTables();
            if (indexException.isDisableIndexOnFailure() && failedIndexes != null) {
                for (HTableInterfaceReference failedIndex : failedIndexes) {
                    String failedIndexTable = failedIndex.getTableName();
                    if (!indexesToUpdate.contains(failedIndexTable)) {
                        updateIndex(failedIndexTable, conn, indexState);
                        indexesToUpdate.add(failedIndexTable);
                    }
                }
            }
        } else if (indexWriteException instanceof SingleIndexWriteFailureException) {
            SingleIndexWriteFailureException indexException = (SingleIndexWriteFailureException) indexWriteException;
            String failedIndex = indexException.getTableName();
            if (indexException.isDisableIndexOnFailure() && failedIndex != null) {
                updateIndex(failedIndex, conn, indexState);
            }
        }
    } catch (Exception handleE) {
        LOG.warn("Error while trying to handle index write exception", indexWriteException);
    }
}

Example 4 with HTableInterfaceReference

use of org.apache.phoenix.hbase.index.table.HTableInterfaceReference in project phoenix by apache.

the class TestParalleIndexWriter method testSynchronouslyCompletesAllWrites.

@SuppressWarnings({ "unchecked", "deprecation" })
@Test
public void testSynchronouslyCompletesAllWrites() throws Exception {
    LOG.info("Starting " + test.getTableNameString());
    LOG.info("Current thread is interrupted: " + Thread.interrupted());
    Abortable abort = new StubAbortable();
    Stoppable stop = Mockito.mock(Stoppable.class);
    ExecutorService exec = Executors.newFixedThreadPool(1);
    Map<ImmutableBytesPtr, HTableInterface> tables = new LinkedHashMap<ImmutableBytesPtr, HTableInterface>();
    FakeTableFactory factory = new FakeTableFactory(tables);
    RegionCoprocessorEnvironment e = Mockito.mock(RegionCoprocessorEnvironment.class);
    Configuration conf = new Configuration();
    Mockito.when(e.getConfiguration()).thenReturn(conf);
    Mockito.when(e.getSharedData()).thenReturn(new ConcurrentHashMap<String, Object>());
    Region mockRegion = Mockito.mock(Region.class);
    Mockito.when(e.getRegion()).thenReturn(mockRegion);
    HTableDescriptor mockTableDesc = Mockito.mock(HTableDescriptor.class);
    Mockito.when(mockRegion.getTableDesc()).thenReturn(mockTableDesc);
    ImmutableBytesPtr tableName = new ImmutableBytesPtr(this.test.getTableName());
    Put m = new Put(row);
    m.add(Bytes.toBytes("family"), Bytes.toBytes("qual"), null);
    Multimap<HTableInterfaceReference, Mutation> indexUpdates = ArrayListMultimap.<HTableInterfaceReference, Mutation>create();
    indexUpdates.put(new HTableInterfaceReference(tableName), m);
    HTableInterface table = Mockito.mock(HTableInterface.class);
    final boolean[] completed = new boolean[] { false };
    Mockito.when(table.batch(Mockito.anyList())).thenAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            // just keep track that it was called
            completed[0] = true;
            return null;
        }
    });
    Mockito.when(table.getTableName()).thenReturn(test.getTableName());
    // add the table to the set of tables, so its returned to the writer
    tables.put(tableName, table);
    // setup the writer and failure policy
    TrackingParallelWriterIndexCommitter writer = new TrackingParallelWriterIndexCommitter(VersionInfo.getVersion());
    writer.setup(factory, exec, abort, stop, e);
    writer.write(indexUpdates, true);
    assertTrue("Writer returned before the table batch completed! Likely a race condition tripped", completed[0]);
    writer.stop(this.test.getTableNameString() + " finished");
    assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown);
    assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown());
}

Example 5 with HTableInterfaceReference

use of org.apache.phoenix.hbase.index.table.HTableInterfaceReference in project phoenix by apache.

the class TestPerRegionIndexWriteCache method testMultipleRegions.

@Test
public void testMultipleRegions() {
    PerRegionIndexWriteCache cache = new PerRegionIndexWriteCache();
    HTableInterfaceReference t1 = new HTableInterfaceReference(new ImmutableBytesPtr(Bytes.toBytes("t1")));
    List<Mutation> mutations = Lists.<Mutation>newArrayList(p);
    List<Mutation> m2 = Lists.<Mutation>newArrayList(p2);
    // add each region
    cache.addEdits(r1, t1, mutations);
    cache.addEdits(r2, t1, m2);
    // check region1
    Multimap<HTableInterfaceReference, Mutation> edits = cache.getEdits(r1);
    Set<Entry<HTableInterfaceReference, Collection<Mutation>>> entries = edits.asMap().entrySet();
    assertEquals("Got more than one table in the the edit map!", 1, entries.size());
    for (Entry<HTableInterfaceReference, Collection<Mutation>> entry : entries) {
        // ensure that we are still storing a list here - otherwise it breaks the parallel writer
        // implementation
        final List<Mutation> stored = (List<Mutation>) entry.getValue();
        assertEquals("Got an unexpected amount of mutations in the entry for region1", 1, stored.size());
        assertEquals("Got an unexpected mutation in the entry for region2", p, stored.get(0));
    }
    // check region2
    edits = cache.getEdits(r2);
    entries = edits.asMap().entrySet();
    assertEquals("Got more than one table in the the edit map!", 1, entries.size());
    for (Entry<HTableInterfaceReference, Collection<Mutation>> entry : entries) {
        // ensure that we are still storing a list here - otherwise it breaks the parallel writer
        // implementation
        final List<Mutation> stored = (List<Mutation>) entry.getValue();
        assertEquals("Got an unexpected amount of mutations in the entry for region2", 1, stored.size());
        assertEquals("Got an unexpected mutation in the entry for region2", p2, stored.get(0));
    }
    // ensure that a second get doesn't have any more edits. This ensures that we don't keep
    // references around to these edits and have a memory leak
    assertNull("Got an entry for a region we removed", cache.getEdits(r1));
}