diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java index 7bbd57e98b47..6382b0e9fe22 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java @@ -67,6 +67,16 @@ public interface MetricsMasterSource extends BaseSource { String CLUSTER_REQUESTS_NAME = "clusterRequests"; String OLD_WAL_DIR_SIZE_NAME = "oldWALsDirSize"; + String BALANCER_ENABLED_NAME = "balancerEnabled"; + String NORMALIZER_ENABLED_NAME = "normalizerEnabled"; + String SPLIT_ENABLED_NAME = "splitEnabled"; + String MERGE_ENABLED_NAME = "mergeEnabled"; + String CLEANER_CHORE_ENABLED_NAME = "cleanerChoreEnabled"; + String CATALOG_JANITOR_ENABLED_NAME = "catalogJanitorEnabled"; + String SNAPSHOT_CLEANUP_ENABLED_NAME = "snapshotCleanupEnabled"; + String REPLICATION_PEER_MODIFICATION_ENABLED_NAME = "replicationPeerModificationEnabled"; + String RPC_THROTTLE_ENABLED_NAME = "rpcThrottleEnabled"; + String SWITCH_STATE_DESC = "Switch state: 1 enabled, 0 disabled"; String MASTER_ACTIVE_TIME_DESC = "Master Active Time"; String MASTER_START_TIME_DESC = "Master Start Time"; String MASTER_FINISHED_INITIALIZATION_TIME_DESC = diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java index 51311124eed2..c6db85ab0126 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java @@ -157,4 +157,11 @@ public interface MetricsMasterWrapper { * Get the size of old WALs directory in bytes. */ long getOldWALsDirSize(); + + /** + * Returns known switch states (1 enabled, 0 disabled), or an empty map unless this Master is + * active and initialized. Unavailable states are omitted. Implementations must not perform remote + * reads. + */ + Map getSwitchStates(); } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java index f34b79fb3048..4b470b28eeed 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java @@ -109,6 +109,9 @@ public void getMetrics(MetricsCollector metricsCollector, boolean all) { String.valueOf(masterWrapper.getIsActiveMaster())) .addGauge(Interns.info(OLD_WAL_DIR_SIZE_NAME, OLD_WAL_DIR_SIZE_DESC), masterWrapper.getOldWALsDirSize()); + + masterWrapper.getSwitchStates().forEach((name, state) -> metricsRecordBuilder + .addGauge(Interns.info(name, SWITCH_STATE_DESC), state)); } metricsRegistry.snapshot(metricsRecordBuilder, all); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java index 0057f2d9d4fc..ffdf010f4e34 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java @@ -25,10 +25,13 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.MasterSwitchType; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.quotas.MasterQuotaManager; import org.apache.hadoop.hbase.quotas.QuotaObserverChore; +import org.apache.hadoop.hbase.quotas.QuotaUtil; import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; @@ -238,4 +241,34 @@ public long getOldWALsDirSize() { } return master.getMasterWalManager().getOldWALsDirSize(); } + + @Override + public Map getSwitchStates() { + if (!master.isActiveMaster() || !master.isInitialized()) { + return Collections.emptyMap(); + } + Map states = new HashMap<>(); + states.put(MetricsMasterSource.BALANCER_ENABLED_NAME, master.isBalancerOn() ? 1 : 0); + states.put(MetricsMasterSource.NORMALIZER_ENABLED_NAME, master.isNormalizerOn() ? 1 : 0); + states.put(MetricsMasterSource.SPLIT_ENABLED_NAME, + master.isSplitOrMergeEnabled(MasterSwitchType.SPLIT) ? 1 : 0); + states.put(MetricsMasterSource.MERGE_ENABLED_NAME, + master.isSplitOrMergeEnabled(MasterSwitchType.MERGE) ? 1 : 0); + states.put(MetricsMasterSource.CLEANER_CHORE_ENABLED_NAME, + master.isCleanerChoreEnabled() ? 1 : 0); + states.put(MetricsMasterSource.CATALOG_JANITOR_ENABLED_NAME, + master.isCatalogJanitorEnabled() ? 1 : 0); + states.put(MetricsMasterSource.SNAPSHOT_CLEANUP_ENABLED_NAME, + master.snapshotCleanupStateStore.get() ? 1 : 0); + states.put(MetricsMasterSource.REPLICATION_PEER_MODIFICATION_ENABLED_NAME, + master.isReplicationPeerModificationEnabled() ? 1 : 0); + MasterQuotaManager quotaManager = master.getMasterQuotaManager(); + if (quotaManager != null) { + states.put(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, + quotaManager.isRpcThrottleEnabled() ? 1 : 0); + } else if (!QuotaUtil.isQuotaEnabled(master.getConfiguration())) { + states.put(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, 0); + } + return states; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java index a424da355ef8..71e291b92aaa 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java @@ -427,7 +427,7 @@ public IsRpcThrottleEnabledResponse isRpcThrottleEnabled(IsRpcThrottleEnabledReq } } - public boolean isRpcThrottleEnabled() throws IOException { + public boolean isRpcThrottleEnabled() { return initialized && masterServices.getRpcThrottleStateStore().get(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java index 6bdf31ac22ea..eaa3d1131c1b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.HashMap; +import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CompatibilityFactory; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -28,6 +29,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.StartMiniClusterOption; import org.apache.hadoop.hbase.YouAreDeadException; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.regionserver.RSRpcServices; import org.apache.hadoop.hbase.test.MetricsAssertHelper; import org.apache.hadoop.hbase.testclassification.MasterTests; @@ -121,6 +123,44 @@ public static void after() throws Exception { } } + @FunctionalInterface + private interface SwitchSetter { + void accept(boolean enabled) throws IOException; + } + + @Test + public void testSwitchMetricsFollowAdminChanges() throws IOException { + Admin admin = TEST_UTIL.getAdmin(); + MetricsMasterSource source = master.getMasterMetrics().getMetricsSource(); + Map switches = new HashMap<>(); + switches.put(MetricsMasterSource.BALANCER_ENABLED_NAME, + enabled -> admin.balancerSwitch(enabled, false)); + switches.put(MetricsMasterSource.NORMALIZER_ENABLED_NAME, admin::normalizerSwitch); + switches.put(MetricsMasterSource.SPLIT_ENABLED_NAME, + enabled -> admin.splitSwitch(enabled, false)); + switches.put(MetricsMasterSource.MERGE_ENABLED_NAME, + enabled -> admin.mergeSwitch(enabled, false)); + switches.put(MetricsMasterSource.CLEANER_CHORE_ENABLED_NAME, admin::cleanerChoreSwitch); + switches.put(MetricsMasterSource.CATALOG_JANITOR_ENABLED_NAME, admin::catalogJanitorSwitch); + switches.put(MetricsMasterSource.SNAPSHOT_CLEANUP_ENABLED_NAME, + enabled -> admin.snapshotCleanupSwitch(enabled, false)); + switches.put(MetricsMasterSource.REPLICATION_PEER_MODIFICATION_ENABLED_NAME, + admin::replicationPeerModificationSwitch); + Map initialStates = new MetricsMasterWrapperImpl(master).getSwitchStates(); + for (Map.Entry entry : switches.entrySet()) { + try { + for (boolean enabled : new boolean[] { false, true }) { + entry.getValue().accept(enabled); + metricsHelper.assertGauge(entry.getKey(), enabled ? 1L : 0L, source); + } + } finally { + entry.getValue().accept(initialStates.get(entry.getKey()) == 1); + } + } + // Quota support is disabled by default. + metricsHelper.assertGauge(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, 0L, source); + } + @Test public void testClusterRequests() throws Exception { // sending fake request to master to see how metric value has changed diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRepairMode.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRepairMode.java index 1a60c3337dc6..ee834265fdb1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRepairMode.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRepairMode.java @@ -17,15 +17,20 @@ */ package org.apache.hadoop.hbase.master; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Arrays; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CompatibilityFactory; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.StartMiniClusterOption; @@ -39,6 +44,8 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.quotas.QuotaUtil; +import org.apache.hadoop.hbase.test.MetricsAssertHelper; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.util.Bytes; @@ -98,6 +105,33 @@ public void testNewCluster() throws Exception { } } + @Test + public void testSwitchMetricsWithQuotasEnabled() throws Exception { + enableMaintenanceMode(); + TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); + TEST_UTIL.startMiniCluster( + StartMiniClusterOption.builder().numRegionServers(0).numDataNodes(1).build()); + + HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); + assertTrue(master.isInMaintenanceMode()); + assertNull(master.getMasterQuotaManager()); + MetricsMasterSource source = master.getMasterMetrics().getMetricsSource(); + MetricsAssertHelper metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class); + Map states = new MetricsMasterWrapperImpl(master).getSwitchStates(); + assertEquals(8, states.size()); + assertFalse(states.containsKey(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME)); + assertFalse( + metricsHelper.checkGaugeExists(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, source)); + states.forEach((name, state) -> { + assertTrue(state == 0 || state == 1); + metricsHelper.assertGauge(name, state.longValue(), source); + }); + metricsHelper.assertGauge(MetricsMasterSource.BALANCER_ENABLED_NAME, 0L, source); + metricsHelper.assertGauge(MetricsMasterSource.NORMALIZER_ENABLED_NAME, 0L, source); + metricsHelper.assertGauge(MetricsMasterSource.SPLIT_ENABLED_NAME, 0L, source); + metricsHelper.assertGauge(MetricsMasterSource.MERGE_ENABLED_NAME, 0L, source); + } + @Test public void testExistingCluster() throws Exception { TableName testRepairMode = TableName.valueOf(testMethodName); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java new file mode 100644 index 000000000000..8dc87e82a142 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CompatibilityFactory; +import org.apache.hadoop.hbase.TableDescriptors; +import org.apache.hadoop.hbase.client.MasterSwitchType; +import org.apache.hadoop.hbase.master.snapshot.SnapshotCleanupStateStore; +import org.apache.hadoop.hbase.metrics.MetricRegistries; +import org.apache.hadoop.hbase.quotas.MasterQuotaManager; +import org.apache.hadoop.hbase.quotas.QuotaUtil; +import org.apache.hadoop.hbase.quotas.RpcThrottleStateStore; +import org.apache.hadoop.hbase.test.MetricsAssertHelper; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) +public class TestMasterSwitchMetrics { + private static final MetricsAssertHelper METRICS_HELPER = + CompatibilityFactory.getInstance(MetricsAssertHelper.class); + + private HMaster master; + private Configuration conf; + private RpcThrottleStateStore rpcThrottleStateStore; + private MetricsMasterWrapperImpl wrapper; + private MetricsMasterSourceImpl source; + + @BeforeEach + public void setUp() throws IOException { + master = mock(HMaster.class); + conf = new Configuration(false); + conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); + when(master.getConfiguration()).thenReturn(conf); + when(master.isActiveMaster()).thenReturn(true); + when(master.isInitialized()).thenReturn(true); + when(master.getMasterWalManager()).thenReturn(mock(MasterWalManager.class)); + TableDescriptors tableDescriptors = mock(TableDescriptors.class); + when(tableDescriptors.exists(QuotaUtil.QUOTA_TABLE_NAME)).thenReturn(true); + when(master.getTableDescriptors()).thenReturn(tableDescriptors); + when(master.getClusterSchema()).thenReturn(mock(ClusterSchema.class)); + MasterQuotaManager quotaManager = new MasterQuotaManager(master); + quotaManager.start(); + when(master.getMasterQuotaManager()).thenReturn(quotaManager); + master.snapshotCleanupStateStore = mock(SnapshotCleanupStateStore.class); + rpcThrottleStateStore = mock(RpcThrottleStateStore.class); + when(master.getRpcThrottleStateStore()).thenReturn(rpcThrottleStateStore); + wrapper = new MetricsMasterWrapperImpl(master); + source = new MetricsMasterSourceImpl(wrapper); + } + + @AfterEach + public void tearDown() { + DefaultMetricsSystem.instance().unregisterSource(MetricsMasterSource.METRICS_JMX_CONTEXT); + MetricRegistries.global().remove(source.getMetricRegistryInfo()); + } + + @Test + public void testEachSwitchChangesItsGauge() { + Map> switches = new HashMap<>(); + switches.put(MetricsMasterSource.BALANCER_ENABLED_NAME, + enabled -> when(master.isBalancerOn()).thenReturn(enabled)); + switches.put(MetricsMasterSource.NORMALIZER_ENABLED_NAME, + enabled -> when(master.isNormalizerOn()).thenReturn(enabled)); + switches.put(MetricsMasterSource.SPLIT_ENABLED_NAME, + enabled -> when(master.isSplitOrMergeEnabled(MasterSwitchType.SPLIT)).thenReturn(enabled)); + switches.put(MetricsMasterSource.MERGE_ENABLED_NAME, + enabled -> when(master.isSplitOrMergeEnabled(MasterSwitchType.MERGE)).thenReturn(enabled)); + switches.put(MetricsMasterSource.CLEANER_CHORE_ENABLED_NAME, + enabled -> when(master.isCleanerChoreEnabled()).thenReturn(enabled)); + switches.put(MetricsMasterSource.CATALOG_JANITOR_ENABLED_NAME, + enabled -> when(master.isCatalogJanitorEnabled()).thenReturn(enabled)); + switches.put(MetricsMasterSource.SNAPSHOT_CLEANUP_ENABLED_NAME, + enabled -> when(master.snapshotCleanupStateStore.get()).thenReturn(enabled)); + switches.put(MetricsMasterSource.REPLICATION_PEER_MODIFICATION_ENABLED_NAME, + enabled -> when(master.isReplicationPeerModificationEnabled()).thenReturn(enabled)); + switches.put(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, + enabled -> when(rpcThrottleStateStore.get()).thenReturn(enabled)); + Map expected = new HashMap<>(); + switches.keySet().forEach(name -> expected.put(name, 0)); + for (Map.Entry> entry : switches.entrySet()) { + for (boolean enabled : new boolean[] { true, false }) { + entry.getValue().accept(enabled); + expected.put(entry.getKey(), enabled ? 1 : 0); + assertEquals(expected, wrapper.getSwitchStates()); + expected + .forEach((name, state) -> METRICS_HELPER.assertGauge(name, state.longValue(), source)); + } + } + } + + @Test + public void testOnlyActiveInitializedMasterExposesSwitches() { + Set metricNames = wrapper.getSwitchStates().keySet(); + when(master.isInitialized()).thenReturn(false); + assertTrue(wrapper.getSwitchStates().isEmpty()); + metricNames.forEach(name -> assertFalse(METRICS_HELPER.checkGaugeExists(name, source))); + when(master.isInitialized()).thenReturn(true); + when(master.isActiveMaster()).thenReturn(false); + assertTrue(wrapper.getSwitchStates().isEmpty()); + metricNames.forEach(name -> assertFalse(METRICS_HELPER.checkGaugeExists(name, source))); + } + + @Test + public void testRpcThrottleOmittedUntilQuotaManagerReady() { + MasterQuotaManager quotaManager = master.getMasterQuotaManager(); + Map expected = new HashMap<>(wrapper.getSwitchStates()); + expected.remove(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME); + when(master.getMasterQuotaManager()).thenReturn(null); + assertEquals(expected, wrapper.getSwitchStates()); + assertFalse( + METRICS_HELPER.checkGaugeExists(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, source)); + expected.forEach((name, state) -> METRICS_HELPER.assertGauge(name, state.longValue(), source)); + + when(master.getMasterQuotaManager()).thenReturn(quotaManager); + when(rpcThrottleStateStore.get()).thenReturn(true); + expected.put(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, 1); + assertEquals(expected, wrapper.getSwitchStates()); + METRICS_HELPER.assertGauge(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, 1L, source); + } + + @Test + public void testRpcThrottleDisabledWithoutQuotaSupport() { + conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, false); + when(master.getMasterQuotaManager()).thenReturn(null); + when(rpcThrottleStateStore.get()).thenReturn(true); + assertEquals(0, wrapper.getSwitchStates().get(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME)); + METRICS_HELPER.assertGauge(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, 0L, source); + verifyNoInteractions(rpcThrottleStateStore); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void testRpcThrottleUnaffectedByConfigurationReload(boolean quotaEnabled, + @TempDir Path tempDir) throws IOException { + Path site = tempDir.resolve("hbase-site.xml"); + writeQuotaConfiguration(site, quotaEnabled); + conf = new Configuration(false); + conf.addResource(new org.apache.hadoop.fs.Path(site.toUri())); + when(master.getConfiguration()).thenReturn(conf); + MasterQuotaManager quotaManager = new MasterQuotaManager(master); + quotaManager.start(); + when(master.getMasterQuotaManager()).thenReturn(quotaManager); + when(rpcThrottleStateStore.get()).thenReturn(true); + long expected = quotaEnabled ? 1L : 0L; + assertEquals(quotaEnabled, quotaManager.isRpcThrottleEnabled()); + METRICS_HELPER.assertGauge(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, expected, source); + + writeQuotaConfiguration(site, !quotaEnabled); + conf.reloadConfiguration(); + assertEquals(!quotaEnabled, QuotaUtil.isQuotaEnabled(conf)); + assertEquals(quotaEnabled, quotaManager.isRpcThrottleEnabled()); + METRICS_HELPER.assertGauge(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, expected, source); + } + + private static void writeQuotaConfiguration(Path path, boolean quotaEnabled) throws IOException { + Configuration config = new Configuration(false); + config.setBoolean(QuotaUtil.QUOTA_CONF_KEY, quotaEnabled); + try (OutputStream out = Files.newOutputStream(path)) { + config.writeXml(out); + } + } +}