Backport "HBASE-30473 Expose Master switch states as numeric metrics (#8758)" to branch-2.5

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java

@@ -66,6 +66,14 @@ public interface MetricsMasterSource extends BaseSource {
   String MERGE_PLAN_COUNT_NAME = "mergePlanCount";
 
   String CLUSTER_REQUESTS_NAME = "clusterRequests";
+  String BALANCER_ENABLED_NAME = "balancerEnabled";
+  String NORMALIZER_ENABLED_NAME = "normalizerEnabled";
+  String SPLIT_ENABLED_NAME = "splitEnabled";
+  String MERGE_ENABLED_NAME = "mergeEnabled";
+  String CLEANER_CHORE_ENABLED_NAME = "cleanerChoreEnabled";
+  String CATALOG_JANITOR_ENABLED_NAME = "catalogJanitorEnabled";
+  String SNAPSHOT_CLEANUP_ENABLED_NAME = "snapshotCleanupEnabled";
+  String SWITCH_STATE_DESC = "Switch state: 1 enabled, 0 disabled";
   String MASTER_ACTIVE_TIME_DESC = "Master Active Time";
   String MASTER_START_TIME_DESC = "Master Start Time";
   String MASTER_FINISHED_INITIALIZATION_TIME_DESC =

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java

@@ -152,4 +152,11 @@ public interface MetricsMasterWrapper {
    * Get the time in Millis when the master finished initializing/becoming the active master
    */
   long getMasterInitializationTime();
+
+  /**
+   * Returns known switch states (1 enabled, 0 disabled), or an empty map unless this Master is
+   * active and initialized. Unavailable states are omitted. Implementations must not perform remote
+   * reads.
+   */
+  Map<String, Integer> getSwitchStates();
 }

hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java

@@ -107,6 +107,9 @@ public void getMetrics(MetricsCollector metricsCollector, boolean all) {
         .tag(Interns.info(CLUSTER_ID_NAME, CLUSTER_ID_DESC), masterWrapper.getClusterId())
         .tag(Interns.info(IS_ACTIVE_MASTER_NAME, IS_ACTIVE_MASTER_DESC),
           String.valueOf(masterWrapper.getIsActiveMaster()));
+
+      masterWrapper.getSwitchStates().forEach((name, state) -> metricsRecordBuilder
+        .addGauge(Interns.info(name, SWITCH_STATE_DESC), state));
     }
 
     metricsRegistry.snapshot(metricsRecordBuilder, all);

hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java

@@ -25,6 +25,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.MasterSwitchType;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
 import org.apache.hadoop.hbase.procedure2.Procedure;
 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
@@ -230,4 +231,25 @@ public Map<String, Entry<Long, Long>> getNamespaceSpaceUtilization() {
   Entry<Long, Long> convertSnapshot(SpaceQuotaSnapshot snapshot) {
     return new SimpleImmutableEntry<Long, Long>(snapshot.getUsage(), snapshot.getLimit());
   }
+
+  @Override
+  public Map<String, Integer> getSwitchStates() {
+    if (!master.isActiveMaster() || !master.isInitialized()) {
+      return Collections.emptyMap();
+    }
+    Map<String, Integer> states = new HashMap<>();
+    states.put(MetricsMasterSource.BALANCER_ENABLED_NAME, master.isBalancerOn() ? 1 : 0);
+    states.put(MetricsMasterSource.NORMALIZER_ENABLED_NAME, master.isNormalizerOn() ? 1 : 0);
+    states.put(MetricsMasterSource.SPLIT_ENABLED_NAME,
+      master.isSplitOrMergeEnabled(MasterSwitchType.SPLIT) ? 1 : 0);
+    states.put(MetricsMasterSource.MERGE_ENABLED_NAME,
+      master.isSplitOrMergeEnabled(MasterSwitchType.MERGE) ? 1 : 0);
+    states.put(MetricsMasterSource.CLEANER_CHORE_ENABLED_NAME,
+      master.isCleanerChoreEnabled() ? 1 : 0);
+    states.put(MetricsMasterSource.CATALOG_JANITOR_ENABLED_NAME,
+      master.isCatalogJanitorEnabled() ? 1 : 0);
+    states.put(MetricsMasterSource.SNAPSHOT_CLEANUP_ENABLED_NAME,
+      master.snapshotCleanupTracker.isSnapshotCleanupEnabled() ? 1 : 0);
+    return states;
+  }
 }

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java

@@ -20,6 +20,7 @@
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.HashMap;
+import java.util.Map;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.CompatibilityFactory;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -28,6 +29,7 @@
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.StartMiniClusterOption;
 import org.apache.hadoop.hbase.YouAreDeadException;
+import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.regionserver.RSRpcServices;
 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -121,6 +123,40 @@ public static void after() throws Exception {
     }
   }
 
+  @FunctionalInterface
+  private interface SwitchSetter {
+    void accept(boolean enabled) throws IOException;
+  }
+
+  @Test
+  public void testSwitchMetricsFollowAdminChanges() throws IOException {
+    Admin admin = TEST_UTIL.getAdmin();
+    MetricsMasterSource source = master.getMasterMetrics().getMetricsSource();
+    Map<String, SwitchSetter> switches = new HashMap<>();
+    switches.put(MetricsMasterSource.BALANCER_ENABLED_NAME,
+      enabled -> admin.balancerSwitch(enabled, false));
+    switches.put(MetricsMasterSource.NORMALIZER_ENABLED_NAME, admin::normalizerSwitch);
+    switches.put(MetricsMasterSource.SPLIT_ENABLED_NAME,
+      enabled -> admin.splitSwitch(enabled, false));
+    switches.put(MetricsMasterSource.MERGE_ENABLED_NAME,
+      enabled -> admin.mergeSwitch(enabled, false));
+    switches.put(MetricsMasterSource.CLEANER_CHORE_ENABLED_NAME, admin::cleanerChoreSwitch);
+    switches.put(MetricsMasterSource.CATALOG_JANITOR_ENABLED_NAME, admin::catalogJanitorSwitch);
+    switches.put(MetricsMasterSource.SNAPSHOT_CLEANUP_ENABLED_NAME,
+      enabled -> admin.snapshotCleanupSwitch(enabled, false));
+    Map<String, Integer> initialStates = new MetricsMasterWrapperImpl(master).getSwitchStates();
+    for (Map.Entry<String, SwitchSetter> entry : switches.entrySet()) {
+      try {
+        for (boolean enabled : new boolean[] { false, true }) {
+          entry.getValue().accept(enabled);
+          metricsHelper.assertGauge(entry.getKey(), enabled ? 1L : 0L, source);
+        }
+      } finally {
+        entry.getValue().accept(initialStates.get(entry.getKey()) == 1);
+      }
+    }
+  }
+
   @Test
   public void testClusterRequests() throws Exception {
     // sending fake request to master to see how metric value has changed

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRepairMode.java

@@ -17,15 +17,19 @@
  */
 package org.apache.hadoop.hbase.master;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.util.Arrays;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CompatibilityFactory;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.StartMiniClusterOption;
@@ -39,6 +43,8 @@
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.quotas.QuotaUtil;
+import org.apache.hadoop.hbase.test.MetricsAssertHelper;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -98,6 +104,30 @@ public void testNewCluster() throws Exception {
     }
   }
 
+  @Test
+  public void testSwitchMetricsWithQuotasEnabled() throws Exception {
+    enableMaintenanceMode();
+    TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
+    TEST_UTIL.startMiniCluster(
+      StartMiniClusterOption.builder().numRegionServers(0).numDataNodes(1).build());
+
+    HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
+    assertTrue(master.isInMaintenanceMode());
+    assertNull(master.getMasterQuotaManager());
+    MetricsMasterSource source = master.getMasterMetrics().getMetricsSource();
+    MetricsAssertHelper metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class);
+    Map<String, Integer> states = new MetricsMasterWrapperImpl(master).getSwitchStates();
+    assertEquals(7, states.size());
+    states.forEach((name, state) -> {
+      assertTrue(state == 0 || state == 1);
+      metricsHelper.assertGauge(name, state.longValue(), source);
+    });
+    metricsHelper.assertGauge(MetricsMasterSource.BALANCER_ENABLED_NAME, 0L, source);
+    metricsHelper.assertGauge(MetricsMasterSource.NORMALIZER_ENABLED_NAME, 0L, source);
+    metricsHelper.assertGauge(MetricsMasterSource.SPLIT_ENABLED_NAME, 0L, source);
+    metricsHelper.assertGauge(MetricsMasterSource.MERGE_ENABLED_NAME, 0L, source);
+  }
+
   @Test
   public void testExistingCluster() throws Exception {
     TableName testRepairMode = TableName.valueOf(testMethodName);

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java

@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import org.apache.hadoop.hbase.CompatibilityFactory;
+import org.apache.hadoop.hbase.client.MasterSwitchType;
+import org.apache.hadoop.hbase.metrics.MetricRegistries;
+import org.apache.hadoop.hbase.test.MetricsAssertHelper;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.zookeeper.SnapshotCleanupTracker;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+@Tag(MasterTests.TAG)
+@Tag(SmallTests.TAG)
+public class TestMasterSwitchMetrics {
+  private static final MetricsAssertHelper METRICS_HELPER =
+    CompatibilityFactory.getInstance(MetricsAssertHelper.class);
+
+  private HMaster master;
+  private MetricsMasterWrapperImpl wrapper;
+  private MetricsMasterSourceImpl source;
+
+  @BeforeEach
+  public void setUp() {
+    master = mock(HMaster.class);
+    when(master.isActiveMaster()).thenReturn(true);
+    when(master.isInitialized()).thenReturn(true);
+    master.snapshotCleanupTracker = mock(SnapshotCleanupTracker.class);
+    wrapper = new MetricsMasterWrapperImpl(master);
+    source = new MetricsMasterSourceImpl(wrapper);
+  }
+
+  @AfterEach
+  public void tearDown() {
+    DefaultMetricsSystem.instance().unregisterSource(MetricsMasterSource.METRICS_JMX_CONTEXT);
+    MetricRegistries.global().remove(source.getMetricRegistryInfo());
+  }
+
+  @Test
+  public void testEachSwitchChangesItsGauge() {
+    Map<String, Consumer<Boolean>> switches = new HashMap<>();
+    switches.put(MetricsMasterSource.BALANCER_ENABLED_NAME,
+      enabled -> when(master.isBalancerOn()).thenReturn(enabled));
+    switches.put(MetricsMasterSource.NORMALIZER_ENABLED_NAME,
+      enabled -> when(master.isNormalizerOn()).thenReturn(enabled));
+    switches.put(MetricsMasterSource.SPLIT_ENABLED_NAME,
+      enabled -> when(master.isSplitOrMergeEnabled(MasterSwitchType.SPLIT)).thenReturn(enabled));
+    switches.put(MetricsMasterSource.MERGE_ENABLED_NAME,
+      enabled -> when(master.isSplitOrMergeEnabled(MasterSwitchType.MERGE)).thenReturn(enabled));
+    switches.put(MetricsMasterSource.CLEANER_CHORE_ENABLED_NAME,
+      enabled -> when(master.isCleanerChoreEnabled()).thenReturn(enabled));
+    switches.put(MetricsMasterSource.CATALOG_JANITOR_ENABLED_NAME,
+      enabled -> when(master.isCatalogJanitorEnabled()).thenReturn(enabled));
+    switches.put(MetricsMasterSource.SNAPSHOT_CLEANUP_ENABLED_NAME,
+      enabled -> when(master.snapshotCleanupTracker.isSnapshotCleanupEnabled())
+        .thenReturn(enabled));
+    Map<String, Integer> expected = new HashMap<>();
+    switches.keySet().forEach(name -> expected.put(name, 0));
+    for (Map.Entry<String, Consumer<Boolean>> entry : switches.entrySet()) {
+      for (boolean enabled : new boolean[] { true, false }) {
+        entry.getValue().accept(enabled);
+        expected.put(entry.getKey(), enabled ? 1 : 0);
+        assertEquals(expected, wrapper.getSwitchStates());
+        expected
+          .forEach((name, state) -> METRICS_HELPER.assertGauge(name, state.longValue(), source));
+      }
+    }
+  }
+
+  @Test
+  public void testOnlyActiveInitializedMasterExposesSwitches() {
+    Set<String> metricNames = wrapper.getSwitchStates().keySet();
+    when(master.isInitialized()).thenReturn(false);
+    assertTrue(wrapper.getSwitchStates().isEmpty());
+    metricNames.forEach(name -> assertFalse(METRICS_HELPER.checkGaugeExists(name, source)));
+    when(master.isInitialized()).thenReturn(true);
+    when(master.isActiveMaster()).thenReturn(false);
+    assertTrue(wrapper.getSwitchStates().isEmpty());
+    metricNames.forEach(name -> assertFalse(METRICS_HELPER.checkGaugeExists(name, source)));
+  }
+}