From 0a2a91357b6c2afabd9bfb480e270ee10f863354 Mon Sep 17 00:00:00 2001 From: Junegunn Choi Date: Fri, 16 Oct 2026 08:58:30 +0900 Subject: [PATCH] HBASE-30473 Expose Master switch states as numeric metrics (#8758) Signed-off-by: Nihal Jain Signed-off-by: Xiao Liu --- .../hbase/master/MetricsMasterSource.java | 8 ++ .../hbase/master/MetricsMasterWrapper.java | 7 ++ .../hbase/master/MetricsMasterSourceImpl.java | 3 + .../master/MetricsMasterWrapperImpl.java | 22 ++++ .../hbase/master/TestMasterMetrics.java | 36 ++++++ .../hbase/master/TestMasterRepairMode.java | 30 +++++ .../hbase/master/TestMasterSwitchMetrics.java | 111 ++++++++++++++++++ 7 files changed, 217 insertions(+) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java index 5952bdc4d8e8..6981efae09db 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java @@ -66,6 +66,14 @@ public interface MetricsMasterSource extends BaseSource { String MERGE_PLAN_COUNT_NAME = "mergePlanCount"; String CLUSTER_REQUESTS_NAME = "clusterRequests"; + String BALANCER_ENABLED_NAME = "balancerEnabled"; + String NORMALIZER_ENABLED_NAME = "normalizerEnabled"; + String SPLIT_ENABLED_NAME = "splitEnabled"; + String MERGE_ENABLED_NAME = "mergeEnabled"; + String CLEANER_CHORE_ENABLED_NAME = "cleanerChoreEnabled"; + String CATALOG_JANITOR_ENABLED_NAME = "catalogJanitorEnabled"; + String SNAPSHOT_CLEANUP_ENABLED_NAME = "snapshotCleanupEnabled"; + String SWITCH_STATE_DESC = "Switch state: 1 enabled, 0 disabled"; String MASTER_ACTIVE_TIME_DESC = "Master Active Time"; String MASTER_START_TIME_DESC = "Master Start Time"; String MASTER_FINISHED_INITIALIZATION_TIME_DESC = diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java index 9f0f4db63658..7cf609600511 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java @@ -152,4 +152,11 @@ public interface MetricsMasterWrapper { * Get the time in Millis when the master finished initializing/becoming the active master */ long getMasterInitializationTime(); + + /** + * Returns known switch states (1 enabled, 0 disabled), or an empty map unless this Master is + * active and initialized. Unavailable states are omitted. Implementations must not perform remote + * reads. + */ + Map getSwitchStates(); } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java index 2dbdeff0dd49..d875f893c5d6 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java @@ -107,6 +107,9 @@ public void getMetrics(MetricsCollector metricsCollector, boolean all) { .tag(Interns.info(CLUSTER_ID_NAME, CLUSTER_ID_DESC), masterWrapper.getClusterId()) .tag(Interns.info(IS_ACTIVE_MASTER_NAME, IS_ACTIVE_MASTER_DESC), String.valueOf(masterWrapper.getIsActiveMaster())); + + masterWrapper.getSwitchStates().forEach((name, state) -> metricsRecordBuilder + .addGauge(Interns.info(name, SWITCH_STATE_DESC), state)); } metricsRegistry.snapshot(metricsRecordBuilder, all); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java index 4fa0848615f2..4ec9966402b5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java @@ -25,6 +25,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.MasterSwitchType; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; @@ -230,4 +231,25 @@ public Map> getNamespaceSpaceUtilization() { Entry convertSnapshot(SpaceQuotaSnapshot snapshot) { return new SimpleImmutableEntry(snapshot.getUsage(), snapshot.getLimit()); } + + @Override + public Map getSwitchStates() { + if (!master.isActiveMaster() || !master.isInitialized()) { + return Collections.emptyMap(); + } + Map states = new HashMap<>(); + states.put(MetricsMasterSource.BALANCER_ENABLED_NAME, master.isBalancerOn() ? 1 : 0); + states.put(MetricsMasterSource.NORMALIZER_ENABLED_NAME, master.isNormalizerOn() ? 1 : 0); + states.put(MetricsMasterSource.SPLIT_ENABLED_NAME, + master.isSplitOrMergeEnabled(MasterSwitchType.SPLIT) ? 1 : 0); + states.put(MetricsMasterSource.MERGE_ENABLED_NAME, + master.isSplitOrMergeEnabled(MasterSwitchType.MERGE) ? 1 : 0); + states.put(MetricsMasterSource.CLEANER_CHORE_ENABLED_NAME, + master.isCleanerChoreEnabled() ? 1 : 0); + states.put(MetricsMasterSource.CATALOG_JANITOR_ENABLED_NAME, + master.isCatalogJanitorEnabled() ? 1 : 0); + states.put(MetricsMasterSource.SNAPSHOT_CLEANUP_ENABLED_NAME, + master.snapshotCleanupTracker.isSnapshotCleanupEnabled() ? 1 : 0); + return states; + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java index ad62db33a3d4..8601e0e88211 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.HashMap; +import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CompatibilityFactory; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -28,6 +29,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.StartMiniClusterOption; import org.apache.hadoop.hbase.YouAreDeadException; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.regionserver.RSRpcServices; import org.apache.hadoop.hbase.test.MetricsAssertHelper; import org.apache.hadoop.hbase.testclassification.MasterTests; @@ -121,6 +123,40 @@ public static void after() throws Exception { } } + @FunctionalInterface + private interface SwitchSetter { + void accept(boolean enabled) throws IOException; + } + + @Test + public void testSwitchMetricsFollowAdminChanges() throws IOException { + Admin admin = TEST_UTIL.getAdmin(); + MetricsMasterSource source = master.getMasterMetrics().getMetricsSource(); + Map switches = new HashMap<>(); + switches.put(MetricsMasterSource.BALANCER_ENABLED_NAME, + enabled -> admin.balancerSwitch(enabled, false)); + switches.put(MetricsMasterSource.NORMALIZER_ENABLED_NAME, admin::normalizerSwitch); + switches.put(MetricsMasterSource.SPLIT_ENABLED_NAME, + enabled -> admin.splitSwitch(enabled, false)); + switches.put(MetricsMasterSource.MERGE_ENABLED_NAME, + enabled -> admin.mergeSwitch(enabled, false)); + switches.put(MetricsMasterSource.CLEANER_CHORE_ENABLED_NAME, admin::cleanerChoreSwitch); + switches.put(MetricsMasterSource.CATALOG_JANITOR_ENABLED_NAME, admin::catalogJanitorSwitch); + switches.put(MetricsMasterSource.SNAPSHOT_CLEANUP_ENABLED_NAME, + enabled -> admin.snapshotCleanupSwitch(enabled, false)); + Map initialStates = new MetricsMasterWrapperImpl(master).getSwitchStates(); + for (Map.Entry entry : switches.entrySet()) { + try { + for (boolean enabled : new boolean[] { false, true }) { + entry.getValue().accept(enabled); + metricsHelper.assertGauge(entry.getKey(), enabled ? 1L : 0L, source); + } + } finally { + entry.getValue().accept(initialStates.get(entry.getKey()) == 1); + } + } + } + @Test public void testClusterRequests() throws Exception { // sending fake request to master to see how metric value has changed diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRepairMode.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRepairMode.java index 1a60c3337dc6..8c0e1bb5b024 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRepairMode.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRepairMode.java @@ -17,15 +17,19 @@ */ package org.apache.hadoop.hbase.master; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Arrays; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CompatibilityFactory; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.StartMiniClusterOption; @@ -39,6 +43,8 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.quotas.QuotaUtil; +import org.apache.hadoop.hbase.test.MetricsAssertHelper; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.util.Bytes; @@ -98,6 +104,30 @@ public void testNewCluster() throws Exception { } } + @Test + public void testSwitchMetricsWithQuotasEnabled() throws Exception { + enableMaintenanceMode(); + TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); + TEST_UTIL.startMiniCluster( + StartMiniClusterOption.builder().numRegionServers(0).numDataNodes(1).build()); + + HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); + assertTrue(master.isInMaintenanceMode()); + assertNull(master.getMasterQuotaManager()); + MetricsMasterSource source = master.getMasterMetrics().getMetricsSource(); + MetricsAssertHelper metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class); + Map states = new MetricsMasterWrapperImpl(master).getSwitchStates(); + assertEquals(7, states.size()); + states.forEach((name, state) -> { + assertTrue(state == 0 || state == 1); + metricsHelper.assertGauge(name, state.longValue(), source); + }); + metricsHelper.assertGauge(MetricsMasterSource.BALANCER_ENABLED_NAME, 0L, source); + metricsHelper.assertGauge(MetricsMasterSource.NORMALIZER_ENABLED_NAME, 0L, source); + metricsHelper.assertGauge(MetricsMasterSource.SPLIT_ENABLED_NAME, 0L, source); + metricsHelper.assertGauge(MetricsMasterSource.MERGE_ENABLED_NAME, 0L, source); + } + @Test public void testExistingCluster() throws Exception { TableName testRepairMode = TableName.valueOf(testMethodName); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java new file mode 100644 index 000000000000..c88110bc58f5 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import org.apache.hadoop.hbase.CompatibilityFactory; +import org.apache.hadoop.hbase.client.MasterSwitchType; +import org.apache.hadoop.hbase.metrics.MetricRegistries; +import org.apache.hadoop.hbase.test.MetricsAssertHelper; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.zookeeper.SnapshotCleanupTracker; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) +public class TestMasterSwitchMetrics { + private static final MetricsAssertHelper METRICS_HELPER = + CompatibilityFactory.getInstance(MetricsAssertHelper.class); + + private HMaster master; + private MetricsMasterWrapperImpl wrapper; + private MetricsMasterSourceImpl source; + + @BeforeEach + public void setUp() { + master = mock(HMaster.class); + when(master.isActiveMaster()).thenReturn(true); + when(master.isInitialized()).thenReturn(true); + master.snapshotCleanupTracker = mock(SnapshotCleanupTracker.class); + wrapper = new MetricsMasterWrapperImpl(master); + source = new MetricsMasterSourceImpl(wrapper); + } + + @AfterEach + public void tearDown() { + DefaultMetricsSystem.instance().unregisterSource(MetricsMasterSource.METRICS_JMX_CONTEXT); + MetricRegistries.global().remove(source.getMetricRegistryInfo()); + } + + @Test + public void testEachSwitchChangesItsGauge() { + Map> switches = new HashMap<>(); + switches.put(MetricsMasterSource.BALANCER_ENABLED_NAME, + enabled -> when(master.isBalancerOn()).thenReturn(enabled)); + switches.put(MetricsMasterSource.NORMALIZER_ENABLED_NAME, + enabled -> when(master.isNormalizerOn()).thenReturn(enabled)); + switches.put(MetricsMasterSource.SPLIT_ENABLED_NAME, + enabled -> when(master.isSplitOrMergeEnabled(MasterSwitchType.SPLIT)).thenReturn(enabled)); + switches.put(MetricsMasterSource.MERGE_ENABLED_NAME, + enabled -> when(master.isSplitOrMergeEnabled(MasterSwitchType.MERGE)).thenReturn(enabled)); + switches.put(MetricsMasterSource.CLEANER_CHORE_ENABLED_NAME, + enabled -> when(master.isCleanerChoreEnabled()).thenReturn(enabled)); + switches.put(MetricsMasterSource.CATALOG_JANITOR_ENABLED_NAME, + enabled -> when(master.isCatalogJanitorEnabled()).thenReturn(enabled)); + switches.put(MetricsMasterSource.SNAPSHOT_CLEANUP_ENABLED_NAME, + enabled -> when(master.snapshotCleanupTracker.isSnapshotCleanupEnabled()) + .thenReturn(enabled)); + Map expected = new HashMap<>(); + switches.keySet().forEach(name -> expected.put(name, 0)); + for (Map.Entry> entry : switches.entrySet()) { + for (boolean enabled : new boolean[] { true, false }) { + entry.getValue().accept(enabled); + expected.put(entry.getKey(), enabled ? 1 : 0); + assertEquals(expected, wrapper.getSwitchStates()); + expected + .forEach((name, state) -> METRICS_HELPER.assertGauge(name, state.longValue(), source)); + } + } + } + + @Test + public void testOnlyActiveInitializedMasterExposesSwitches() { + Set metricNames = wrapper.getSwitchStates().keySet(); + when(master.isInitialized()).thenReturn(false); + assertTrue(wrapper.getSwitchStates().isEmpty()); + metricNames.forEach(name -> assertFalse(METRICS_HELPER.checkGaugeExists(name, source))); + when(master.isInitialized()).thenReturn(true); + when(master.isActiveMaster()).thenReturn(false); + assertTrue(wrapper.getSwitchStates().isEmpty()); + metricNames.forEach(name -> assertFalse(METRICS_HELPER.checkGaugeExists(name, source))); + } +}