Examples with HTableInterface org.apache.hadoop.hbase.client.HTableInterface used on opensource projects

Example 61 with HTableInterface

use of org.apache.hadoop.hbase.client.HTableInterface in project metron by apache.

the class LocalImporter method createState.

@Override
protected ThreadLocal<HBaseExtractorState> createState(EnumMap<LoadOptions, Optional<Object>> config, Configuration hadoopConfig, final ExtractorHandler handler) {
    ThreadLocal<HBaseExtractorState> state = new ThreadLocal<HBaseExtractorState>() {

        @Override
        protected HBaseExtractorState initialValue() {
            try {
                String cf = (String) config.get(LoadOptions.HBASE_CF).get();
                HTableInterface table = provider.retrieve().getTable(hadoopConfig, (String) config.get(LoadOptions.HBASE_TABLE).get());
                return new HBaseExtractorState(table, cf, handler.getExtractor(), new EnrichmentConverter(), hadoopConfig);
            } catch (IOException e1) {
                throw new IllegalStateException("Unable to get table: " + e1);
            }
        }
    };
    return state;
}

Example 62 with HTableInterface

use of org.apache.hadoop.hbase.client.HTableInterface in project metron by apache.

the class TaxiiHandler method run.

/**
 * The action to be performed by this timer task.
 */
@Override
public void run() {
    if (inProgress) {
        return;
    }
    Date ts = new Date();
    LOG.info("Polling...{}", new SimpleDateFormat().format(ts));
    try {
        inProgress = true;
        // Prepare the message to send.
        String sessionID = MessageHelper.generateMessageId();
        PollRequest request = messageFactory.get().createPollRequest().withMessageId(sessionID).withCollectionName(collection);
        if (subscriptionId != null) {
            request = request.withSubscriptionID(subscriptionId);
        } else {
            request = request.withPollParameters(messageFactory.get().createPollParametersType());
        }
        if (beginTime != null) {
            Calendar gc = GregorianCalendar.getInstance();
            gc.setTime(beginTime);
            XMLGregorianCalendar gTime = null;
            try {
                gTime = DatatypeFactory.newInstance().newXMLGregorianCalendar((GregorianCalendar) gc).normalize();
            } catch (DatatypeConfigurationException e) {
                ErrorUtils.RuntimeErrors.ILLEGAL_STATE.throwRuntime("Unable to set the begin time due to", e);
            }
            gTime.setFractionalSecond(null);
            LOG.info("Begin Time: {}", gTime);
            request.setExclusiveBeginTimestamp(gTime);
        }
        try {
            PollResponse response = call(request, PollResponse.class);
            LOG.info("Got Poll Response with {} blocks", response.getContentBlocks().size());
            int numProcessed = 0;
            long avgTimeMS = 0;
            long timeStartedBlock = System.currentTimeMillis();
            for (ContentBlock block : response.getContentBlocks()) {
                AnyMixedContentType content = block.getContent();
                for (Object o : content.getContent()) {
                    numProcessed++;
                    long timeS = System.currentTimeMillis();
                    String xml = null;
                    if (o instanceof Element) {
                        Element element = (Element) o;
                        xml = getStringFromDocument(element.getOwnerDocument());
                        if (LOG.isDebugEnabled() && Math.random() < 0.01) {
                            LOG.debug("Random Stix doc: {}", xml);
                        }
                        for (LookupKV<EnrichmentKey, EnrichmentValue> kv : extractor.extract(xml)) {
                            if (allowedIndicatorTypes.isEmpty() || allowedIndicatorTypes.contains(kv.getKey().type)) {
                                kv.getValue().getMetadata().put("source_type", "taxii");
                                kv.getValue().getMetadata().put("taxii_url", endpoint.toString());
                                kv.getValue().getMetadata().put("taxii_collection", collection);
                                Put p = converter.toPut(columnFamily, kv.getKey(), kv.getValue());
                                HTableInterface table = getTable(hbaseTable);
                                table.put(p);
                                LOG.info("Found Threat Intel: {} => ", kv.getKey(), kv.getValue());
                            }
                        }
                    }
                    avgTimeMS += System.currentTimeMillis() - timeS;
                }
                if ((numProcessed + 1) % 100 == 0) {
                    LOG.info("Processed {}  in {} ms, avg time: {}", numProcessed, System.currentTimeMillis() - timeStartedBlock, avgTimeMS / content.getContent().size());
                    timeStartedBlock = System.currentTimeMillis();
                    avgTimeMS = 0;
                    numProcessed = 0;
                }
            }
        } catch (Exception e) {
            LOG.error(e.getMessage(), e);
            throw new RuntimeException("Unable to make request", e);
        }
    } finally {
        inProgress = false;
        beginTime = ts;
    }
}

Example 63 with HTableInterface

use of org.apache.hadoop.hbase.client.HTableInterface in project metron by apache.

the class SimpleHbaseEnrichmentWriter method write.

@Override
public BulkWriterResponse write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<JSONObject> messages) throws Exception {
    Map<String, Object> sensorConfig = configurations.getSensorConfig(sensorType);
    HTableInterface table = getTable(sensorConfig);
    KeyTransformer transformer = getTransformer(sensorConfig);
    Object enrichmentTypeObj = Configurations.ENRICHMENT_TYPE.get(sensorConfig);
    String enrichmentType = enrichmentTypeObj == null ? null : enrichmentTypeObj.toString();
    Set<String> valueColumns = new HashSet<>(getColumns(Configurations.VALUE_COLUMNS.get(sensorConfig), true));
    List<Put> puts = new ArrayList<>();
    for (JSONObject message : messages) {
        EnrichmentKey key = getKey(message, transformer, enrichmentType);
        EnrichmentValue value = getValue(message, transformer.keySet, valueColumns);
        if (key == null || value == null) {
            continue;
        }
        Put put = converter.toPut(this.cf, key, value);
        if (put != null) {
            LOG.debug("Put: {Column Family: '{}', Key: '{}', Value: '{}'}", this.cf, key, value);
            puts.add(put);
        }
    }
    BulkWriterResponse response = new BulkWriterResponse();
    try {
        table.put(puts);
    } catch (Exception e) {
        response.addAllErrors(e, tuples);
        return response;
    }
    // Can return no errors, because put will throw Exception on error.
    response.addAllSuccesses(tuples);
    return response;
}

Example 64 with HTableInterface

use of org.apache.hadoop.hbase.client.HTableInterface in project cdap by caskdata.

the class ConsumerConfigCache method updateCache.

/**
   * This forces an immediate update of the config cache. It should only be called from the refresh thread or from
   * tests, to avoid having to add a sleep for the duration of the refresh interval.
   *
   * This method is synchronized to protect from race conditions if called directly from a test. Otherwise this is
   * only called from the refresh thread, and there will not be concurrent invocations.
   *
   * @throws IOException if failed to update config cache
   */
@VisibleForTesting
public synchronized void updateCache() throws IOException {
    Map<byte[], QueueConsumerConfig> newCache = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
    long now = System.currentTimeMillis();
    TransactionVisibilityState txSnapshot = transactionSnapshotSupplier.get();
    if (txSnapshot == null) {
        LOG.debug("No transaction snapshot is available. Not updating the consumer config cache.");
        return;
    }
    HTableInterface table = hTableSupplier.getInput();
    try {
        // Scan the table with the transaction snapshot
        Scan scan = new Scan();
        scan.addFamily(QueueEntryRow.COLUMN_FAMILY);
        Transaction tx = TxUtils.createDummyTransaction(txSnapshot);
        setScanAttribute(scan, TxConstants.TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
        ResultScanner scanner = table.getScanner(scan);
        int configCnt = 0;
        for (Result result : scanner) {
            if (!result.isEmpty()) {
                NavigableMap<byte[], byte[]> familyMap = result.getFamilyMap(QueueEntryRow.COLUMN_FAMILY);
                if (familyMap != null) {
                    configCnt++;
                    Map<ConsumerInstance, byte[]> consumerInstances = new HashMap<>();
                    // Gather the startRow of all instances across all consumer groups.
                    int numGroups = 0;
                    Long groupId = null;
                    for (Map.Entry<byte[], byte[]> entry : familyMap.entrySet()) {
                        if (entry.getKey().length != STATE_COLUMN_SIZE) {
                            continue;
                        }
                        long gid = Bytes.toLong(entry.getKey());
                        int instanceId = Bytes.toInt(entry.getKey(), Bytes.SIZEOF_LONG);
                        consumerInstances.put(new ConsumerInstance(gid, instanceId), entry.getValue());
                        // Columns are sorted by groupId, hence if it change, then numGroups would get +1
                        if (groupId == null || groupId != gid) {
                            numGroups++;
                            groupId = gid;
                        }
                    }
                    byte[] queueName = result.getRow();
                    newCache.put(queueName, new QueueConsumerConfig(consumerInstances, numGroups));
                }
            }
        }
        long elapsed = System.currentTimeMillis() - now;
        this.configCache = newCache;
        this.lastUpdated = now;
        if (LOG.isDebugEnabled()) {
            LOG.debug("Updated consumer config cache with {} entries, took {} msec", configCnt, elapsed);
        }
    } finally {
        try {
            table.close();
        } catch (IOException ioe) {
            LOG.error("Error closing table {}", queueConfigTableName, ioe);
        }
    }
}

Example 65 with HTableInterface

use of org.apache.hadoop.hbase.client.HTableInterface in project hive by apache.

the class VanillaHBaseConnection method getHBaseTable.

@Override
public HTableInterface getHBaseTable(String tableName, boolean force) throws IOException {
    HTableInterface htab = tables.get(tableName);
    if (htab == null) {
        LOG.debug("Trying to connect to table " + tableName);
        try {
            htab = conn.getTable(tableName);
            // and see if the table is there.
            if (force)
                htab.get(new Get("nosuchkey".getBytes(HBaseUtils.ENCODING)));
        } catch (IOException e) {
            LOG.info("Caught exception when table was missing");
            return null;
        }
        htab.setAutoFlushTo(false);
        tables.put(tableName, htab);
    }
    return htab;
}