Examples with ZNRecordSerializer org.apache.helix.manager.zk.ZNRecordSerializer used on opensource projects

Example 11 with ZNRecordSerializer

use of org.apache.helix.manager.zk.ZNRecordSerializer in project pinot by linkedin.

the class LLCRealtimeClusterIntegrationTest method testSegmentFlushSize.

@Test
public void testSegmentFlushSize() {
    ZkClient zkClient = new ZkClient(ZkStarter.DEFAULT_ZK_STR, 10000);
    zkClient.setZkSerializer(new ZNRecordSerializer());
    String zkPath = "/LLCRealtimeClusterIntegrationTest/PROPERTYSTORE/SEGMENTS/mytable_REALTIME";
    List<String> segmentNames = zkClient.getChildren(zkPath);
    for (String segmentName : segmentNames) {
        ZNRecord znRecord = zkClient.<ZNRecord>readData(zkPath + "/" + segmentName);
        Assert.assertEquals(znRecord.getSimpleField(CommonConstants.Segment.FLUSH_THRESHOLD_SIZE), Integer.toString(ROW_COUNT_FOR_REALTIME_SEGMENT_FLUSH / KAFKA_PARTITION_COUNT), "Segment " + segmentName + " does not have the expected flush size");
    }
    zkClient.close();
}

Example 12 with ZNRecordSerializer

use of org.apache.helix.manager.zk.ZNRecordSerializer in project pinot by linkedin.

the class RandomRoutingTableTest method testHelixExternalViewBasedRoutingTable.

@Test
public void testHelixExternalViewBasedRoutingTable() throws Exception {
    URL resourceUrl = getClass().getClassLoader().getResource("SampleExternalView.json");
    Assert.assertNotNull(resourceUrl);
    String fileName = resourceUrl.getFile();
    String tableName = "testTable_OFFLINE";
    InputStream evInputStream = new FileInputStream(fileName);
    ZNRecordSerializer znRecordSerializer = new ZNRecordSerializer();
    ZNRecord externalViewRecord = (ZNRecord) znRecordSerializer.deserialize(IOUtils.toByteArray(evInputStream));
    int totalRuns = 10000;
    RoutingTableBuilder routingStrategy = new BalancedRandomRoutingTableBuilder(10);
    HelixExternalViewBasedRouting routingTable = new HelixExternalViewBasedRouting(null, new PercentageBasedRoutingTableSelector(), null, new BaseConfiguration());
    routingTable.setSmallClusterRoutingTableBuilder(routingStrategy);
    ExternalView externalView = new ExternalView(externalViewRecord);
    routingTable.markDataResourceOnline(tableName, externalView, getInstanceConfigs(externalView));
    double[] globalArrays = new double[9];
    for (int numRun = 0; numRun < totalRuns; ++numRun) {
        RoutingTableLookupRequest request = new RoutingTableLookupRequest(tableName, Collections.<String>emptyList());
        Map<ServerInstance, SegmentIdSet> serversMap = routingTable.findServers(request);
        TreeSet<ServerInstance> serverInstances = new TreeSet<ServerInstance>(serversMap.keySet());
        int i = 0;
        double[] arrays = new double[9];
        for (ServerInstance serverInstance : serverInstances) {
            globalArrays[i] += serversMap.get(serverInstance).getSegments().size();
            arrays[i++] = serversMap.get(serverInstance).getSegments().size();
        }
        for (int j = 0; i < arrays.length; ++j) {
            Assert.assertTrue(arrays[j] / totalRuns <= 31);
            Assert.assertTrue(arrays[j] / totalRuns >= 28);
        }
    //      System.out.println(Arrays.toString(arrays) + " : " + new StandardDeviation().evaluate(arrays) + " : " + new Mean().evaluate(arrays));
    }
    for (int i = 0; i < globalArrays.length; ++i) {
        Assert.assertTrue(globalArrays[i] / totalRuns <= 31);
        Assert.assertTrue(globalArrays[i] / totalRuns >= 28);
    }
//    System.out.println(Arrays.toString(globalArrays) + " : " + new StandardDeviation().evaluate(globalArrays) + " : "
//        + new Mean().evaluate(globalArrays));
}

Example 13 with ZNRecordSerializer

use of org.apache.helix.manager.zk.ZNRecordSerializer in project databus by linkedin.

the class ClusterCheckpointPersistenceProvider method createCluster.

/**
     * Create a cluster if it doesn't exist Note: This method is not thread-safe
     * as HelixAdmin.addCluster appears to fail on concurrent execution within
     * threads
     * 
     * @return true if cluster was created false otherwise
     */
public static boolean createCluster(String zkAddr, String clusterName) {
    boolean created = false;
    ZkClient zkClient = null;
    try {
        zkClient = new ZkClient(zkAddr, ZkClient.DEFAULT_SESSION_TIMEOUT, ZkClient.DEFAULT_CONNECTION_TIMEOUT, new ZNRecordSerializer());
        ZKHelixAdmin admin = new ZKHelixAdmin(zkClient);
        admin.addCluster(clusterName, false);
        created = true;
    } catch (HelixException e) {
        LOG.warn("Warn! Cluster might already exist! " + clusterName);
        created = false;
    } finally {
        // close this connection
        if (zkClient != null) {
            zkClient.close();
        }
    }
    return created;
}

Example 14 with ZNRecordSerializer

use of org.apache.helix.manager.zk.ZNRecordSerializer in project pinot by linkedin.

the class OfflineClusterIntegrationTest method setUp.

@BeforeClass
public void setUp() throws Exception {
    //Clean up
    ensureDirectoryExistsAndIsEmpty(_tmpDir);
    ensureDirectoryExistsAndIsEmpty(_segmentDir);
    ensureDirectoryExistsAndIsEmpty(_tarDir);
    // Start the cluster
    startCluster();
    // Unpack the Avro files
    final List<File> avroFiles = unpackAvroData(_tmpDir, SEGMENT_COUNT);
    createTable();
    // Get the list of instances through the REST API
    URL url = new URL("http://" + ControllerTestUtils.DEFAULT_CONTROLLER_HOST + ":" + ControllerTestUtils.DEFAULT_CONTROLLER_API_PORT + "/instances");
    InputStream inputStream = url.openConnection().getInputStream();
    String instanceApiResponseString = IOUtils.toString(inputStream);
    IOUtils.closeQuietly(inputStream);
    JSONObject instanceApiResponse = new JSONObject(instanceApiResponseString);
    JSONArray instanceArray = instanceApiResponse.getJSONArray("instances");
    HelixAdmin helixAdmin = new ZKHelixAdmin(new ZkClient(ZkStarter.DEFAULT_ZK_STR, 10000, 10000, new ZNRecordSerializer()));
    for (int i = 0; i < instanceArray.length(); i++) {
        String instance = instanceArray.getString(i);
        if (instance.startsWith("Server_")) {
            _serverServiceStatusCallback = new ServiceStatus.IdealStateAndExternalViewMatchServiceStatusCallback(helixAdmin, getHelixClusterName(), instance);
        }
        if (instance.startsWith("Broker_")) {
            _brokerServiceStatusCallback = new ServiceStatus.IdealStateAndExternalViewMatchServiceStatusCallback(helixAdmin, getHelixClusterName(), instance, Collections.singletonList("brokerResource"));
        }
    }
    // Load data into H2
    ExecutorService executor = Executors.newCachedThreadPool();
    setupH2AndInsertAvro(avroFiles, executor);
    // Create segments from Avro data
    buildSegmentsFromAvro(avroFiles, executor, 0, _segmentDir, _tarDir, "mytable", false, null);
    // Initialize query generator
    setupQueryGenerator(avroFiles, executor);
    executor.shutdown();
    executor.awaitTermination(10, TimeUnit.MINUTES);
    // Set up a Helix spectator to count the number of segments that are uploaded and unlock the latch once 12 segments are online
    final CountDownLatch latch = setupSegmentCountCountDownLatch("mytable", SEGMENT_COUNT);
    // Upload the segments
    int i = 0;
    for (String segmentName : _tarDir.list()) {
        //      System.out.println("Uploading segment " + (i++) + " : " + segmentName);
        File file = new File(_tarDir, segmentName);
        FileUploadUtils.sendSegmentFile("localhost", "8998", segmentName, file, file.length());
    }
    // Wait for all segments to be online
    latch.await();
    TOTAL_DOCS = 115545;
    long timeInTwoMinutes = System.currentTimeMillis() + 2 * 60 * 1000L;
    long numDocs;
    while ((numDocs = getCurrentServingNumDocs("mytable")) < TOTAL_DOCS) {
        //      System.out.println("Current number of documents: " + numDocs);
        if (System.currentTimeMillis() < timeInTwoMinutes) {
            Thread.sleep(1000);
        } else {
            Assert.fail("Segments were not completely loaded within two minutes");
        }
    }
}

Example 15 with ZNRecordSerializer

use of org.apache.helix.manager.zk.ZNRecordSerializer in project pinot by linkedin.

the class PinotRealtimeSegmentManager method start.

public void start(ControllerMetrics controllerMetrics) {
    _controllerMetrics = controllerMetrics;
    LOGGER.info("Starting realtime segments manager, adding a listener on the property store table configs path.");
    String zkUrl = _pinotHelixResourceManager.getHelixZkURL();
    _zkClient = new ZkClient(zkUrl, ZkClient.DEFAULT_SESSION_TIMEOUT, ZkClient.DEFAULT_CONNECTION_TIMEOUT);
    _zkClient.setZkSerializer(new ZNRecordSerializer());
    _zkClient.waitUntilConnected();
    // Subscribe to any data/child changes to property
    _zkClient.subscribeChildChanges(_tableConfigPath, this);
    _zkClient.subscribeDataChanges(_tableConfigPath, this);
    // Subscribe to leadership changes
    _pinotHelixResourceManager.getHelixZkManager().addControllerListener(new ControllerChangeListener() {

        @Override
        public void onControllerChange(NotificationContext changeContext) {
            processPropertyStoreChange(CONTROLLER_LEADER_CHANGE);
        }
    });
    // Setup change listeners for already existing tables, if any.
    processPropertyStoreChange(_tableConfigPath);
}