From df62f0525a59f570c3cf73fbdab7b26dc4b65f52 Mon Sep 17 00:00:00 2001 From: skumar34 Date: Fri, 28 Feb 2025 20:17:55 +0530 Subject: [PATCH] AMBARI-26313: Change metrics names in grafana dashboards as per the GC used in JDK17 --- .github/workflows/ambari.yml | 2 +- .../conf/unix/amshbase_metrics_whitelist | 8 ++--- .../conf/unix/metrics_whitelist | 24 +++++++------- .../resources/metrics_def/AMBARI_SERVER.dat | 8 ++--- .../main/resources/metrics_def/AMS-HBASE.dat | 8 ++--- .../main/resources/metrics_def/DATANODE.dat | 8 ++--- .../resources/metrics_def/HBASE_MASTER.dat | 8 ++--- .../metrics_def/HBASE_REGIONSERVER.dat | 8 ++--- .../main/resources/metrics_def/NAMENODE.dat | 8 ++--- .../resources/test_data/full_whitelist.dat | 32 +++++++++---------- .../resources/ui_metrics_def/NAMENODE.dat | 4 +-- pom.xml | 1 + 12 files changed, 60 insertions(+), 59 deletions(-) diff --git a/.github/workflows/ambari.yml b/.github/workflows/ambari.yml index efb55370..f155466e 100644 --- a/.github/workflows/ambari.yml +++ b/.github/workflows/ambari.yml @@ -34,7 +34,7 @@ jobs: distribution: temurin cache: maven - name: Install Kerberos development libraries - run: sudo apt-get install -y libkrb5-dev + run: sudo apt-get update && sudo apt-get install -y libkrb5-dev - name: Install Kerberos run: pip3 install kerberos - name: Build with Maven diff --git a/ambari-metrics-timelineservice/conf/unix/amshbase_metrics_whitelist b/ambari-metrics-timelineservice/conf/unix/amshbase_metrics_whitelist index 3617f55e..204a9c3a 100644 --- a/ambari-metrics-timelineservice/conf/unix/amshbase_metrics_whitelist +++ b/ambari-metrics-timelineservice/conf/unix/amshbase_metrics_whitelist @@ -5,11 +5,11 @@ jvm.Master.JvmMetrics.ThreadsTerminated jvm.Master.JvmMetrics.ThreadsTimedWaiting jvm.Master.JvmMetrics.ThreadsWaiting jvm.RegionServer.JvmMetrics.GcCount -jvm.RegionServer.JvmMetrics.GcCountConcurrentMarkSweep -jvm.RegionServer.JvmMetrics.GcCountParNew +jvm.RegionServer.JvmMetrics.GcCountG1 Old Generation +jvm.RegionServer.JvmMetrics.GcCountG1 Young Generation jvm.RegionServer.JvmMetrics.GcTimeMillis -jvm.RegionServer.JvmMetrics.GcTimeMillisConcurrentMarkSweep -jvm.RegionServer.JvmMetrics.GcTimeMillisParNew +jvm.RegionServer.JvmMetrics.GcTimeMillisG1 Old Generation +jvm.RegionServer.JvmMetrics.GcTimeMillisG1 Young Generation jvm.RegionServer.JvmMetrics.MemHeapCommittedM jvm.RegionServer.JvmMetrics.MemHeapMaxM jvm.RegionServer.JvmMetrics.MemHeapUsedM diff --git a/ambari-metrics-timelineservice/conf/unix/metrics_whitelist b/ambari-metrics-timelineservice/conf/unix/metrics_whitelist index 2774c611..aef17f3d 100644 --- a/ambari-metrics-timelineservice/conf/unix/metrics_whitelist +++ b/ambari-metrics-timelineservice/conf/unix/metrics_whitelist @@ -202,12 +202,12 @@ io.IOMetrics.PercentileDecodingTime_30s99thPercentileLatency ipc.client.org.apache.hadoop.ipc.DecayRpcScheduler.Caller(*).Priority ipc.client.org.apache.hadoop.ipc.DecayRpcScheduler.Caller(*).Volume jvm.JvmMetrics.GcCount -jvm.JvmMetrics.GcCountConcurrentMarkSweep -jvm.JvmMetrics.GcCountParNew +jvm.JvmMetrics.GcCountG1 Old Generation +jvm.JvmMetrics.GcCountG1 Young Generation jvm.JvmMetrics.GcNumWarnThresholdExceeded jvm.JvmMetrics.GcTimeMillis -jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep -jvm.JvmMetrics.GcTimeMillisParNew +jvm.JvmMetrics.GcTimeMillisG1 Old Generation +jvm.JvmMetrics.GcTimeMillisG1 Young Generation jvm.JvmMetrics.GcTotalExtraSleepTime jvm.JvmMetrics.LogError jvm.JvmMetrics.LogFatal @@ -234,11 +234,11 @@ jvm.Master.JvmMetrics.ThreadsTerminated jvm.Master.JvmMetrics.ThreadsTimedWaiting jvm.Master.JvmMetrics.ThreadsWaiting jvm.RegionServer.JvmMetrics.GcCount -jvm.RegionServer.JvmMetrics.GcCountConcurrentMarkSweep -jvm.RegionServer.JvmMetrics.GcCountParNew +jvm.RegionServer.JvmMetrics.GcCountG1 Old Generation +jvm.RegionServer.JvmMetrics.GcCountG1 Young Generation jvm.RegionServer.JvmMetrics.GcTimeMillis -jvm.RegionServer.JvmMetrics.GcTimeMillisConcurrentMarkSweep -jvm.RegionServer.JvmMetrics.GcTimeMillisParNew +jvm.RegionServer.JvmMetrics.GcTimeMillisG1 Old Generation +jvm.RegionServer.JvmMetrics.GcTimeMillisG1 Young Generation jvm.RegionServer.JvmMetrics.MemHeapCommittedM jvm.RegionServer.JvmMetrics.MemHeapMaxM jvm.RegionServer.JvmMetrics.MemHeapUsedM @@ -253,10 +253,10 @@ jvm.RegionServer.JvmMetrics.ThreadsTimedWaiting jvm.RegionServer.JvmMetrics.ThreadsWaiting jvm.daemon_thread_count jvm.file_descriptor_usage -jvm.gc.ConcurrentMarkSweep.count -jvm.gc.ConcurrentMarkSweep.time -jvm.gc.ParNew.count -jvm.gc.ParNew.time +jvm.gc.G1 Old Generation.count +jvm.gc.G1 Old Generation.time +jvm.gc.G1 Young Generation.count +jvm.gc.G1 Young Generation.time jvm.heap_usage jvm.memory.heap.committed jvm.memory.heap.max diff --git a/ambari-metrics-timelineservice/src/main/resources/metrics_def/AMBARI_SERVER.dat b/ambari-metrics-timelineservice/src/main/resources/metrics_def/AMBARI_SERVER.dat index 407b0f85..65204cdd 100644 --- a/ambari-metrics-timelineservice/src/main/resources/metrics_def/AMBARI_SERVER.dat +++ b/ambari-metrics-timelineservice/src/main/resources/metrics_def/AMBARI_SERVER.dat @@ -5,10 +5,10 @@ jvm.buffers.mapped.capacity jvm.buffers.mapped.count jvm.buffers.mapped.used jvm.file.open.descriptor.ratio -jvm.gc.ConcurrentMarkSweep.count -jvm.gc.ConcurrentMarkSweep.time -jvm.gc.ParNew.count -jvm.gc.ParNew.time +jvm.gc.G1 Old Generation.count +jvm.gc.G1 Old Generation.time +jvm.gc.G1 Young Generation.count +jvm.gc.G1 Young Generation.time jvm.memory.heap.committed jvm.memory.heap.init jvm.memory.heap.max diff --git a/ambari-metrics-timelineservice/src/main/resources/metrics_def/AMS-HBASE.dat b/ambari-metrics-timelineservice/src/main/resources/metrics_def/AMS-HBASE.dat index 172d15bd..200a9578 100644 --- a/ambari-metrics-timelineservice/src/main/resources/metrics_def/AMS-HBASE.dat +++ b/ambari-metrics-timelineservice/src/main/resources/metrics_def/AMS-HBASE.dat @@ -2,10 +2,10 @@ regionserver.WAL.SyncTime_min regionserver.WAL.SyncTime_num_ops regionserver.WAL.appendCount regionserver.WAL.slowAppendCount -jvm.JvmMetrics.GcCountParNew +jvm.JvmMetrics.GcCountG1 Young Generation jvm.JvmMetrics.GcTimeMillis -jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep -jvm.JvmMetrics.GcTimeMillisParNew +jvm.JvmMetrics.GcTimeMillisG1 Old Generation +jvm.JvmMetrics.GcTimeMillisG1 Young Generation ugi.UgiMetrics.GetGroupsAvgTime ugi.UgiMetrics.GetGroupsNumOps ugi.UgiMetrics.LoginFailureNumOps @@ -197,7 +197,7 @@ regionserver.Server.checkMutateFailedCount regionserver.Server.checkMutatePassedCount regionserver.Server.compactionQueueLength regionserver.Server.flushQueueLength -jvm.JvmMetrics.GcCountConcurrentMarkSweep +jvm.JvmMetrics.GcCountG1 Old Generation regionserver.Server.hlogFileCount regionserver.Server.hlogFileSize regionserver.Server.memStoreSize diff --git a/ambari-metrics-timelineservice/src/main/resources/metrics_def/DATANODE.dat b/ambari-metrics-timelineservice/src/main/resources/metrics_def/DATANODE.dat index c0158654..d0ee7ee3 100644 --- a/ambari-metrics-timelineservice/src/main/resources/metrics_def/DATANODE.dat +++ b/ambari-metrics-timelineservice/src/main/resources/metrics_def/DATANODE.dat @@ -95,13 +95,13 @@ dfs.datanode.WriteBlockOpNumOps dfs.datanode.WritesFromLocalClient dfs.datanode.WritesFromRemoteClient jvm.JvmMetrics.GcCount -jvm.JvmMetrics.GcCountConcurrentMarkSweep -jvm.JvmMetrics.GcCountParNew +jvm.JvmMetrics.GcCountG1 Old Generation +jvm.JvmMetrics.GcCountG1 Young Generation jvm.JvmMetrics.GcNumInfoThresholdExceeded jvm.JvmMetrics.GcNumWarnThresholdExceeded jvm.JvmMetrics.GcTimeMillis -jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep -jvm.JvmMetrics.GcTimeMillisParNew +jvm.JvmMetrics.GcTimeMillisG1 Old Generation +jvm.JvmMetrics.GcTimeMillisG1 Young Generation jvm.JvmMetrics.GcTotalExtraSleepTime jvm.JvmMetrics.LogError jvm.JvmMetrics.LogFatal diff --git a/ambari-metrics-timelineservice/src/main/resources/metrics_def/HBASE_MASTER.dat b/ambari-metrics-timelineservice/src/main/resources/metrics_def/HBASE_MASTER.dat index c4191648..cb9529aa 100644 --- a/ambari-metrics-timelineservice/src/main/resources/metrics_def/HBASE_MASTER.dat +++ b/ambari-metrics-timelineservice/src/main/resources/metrics_def/HBASE_MASTER.dat @@ -26,11 +26,11 @@ ipc.IPC.queueSize ipc.IPC.receivedBytes ipc.IPC.sentBytes jvm.Master.JvmMetrics.GcCount -jvm.Master.JvmMetrics.GcCountConcurrentMarkSweep -jvm.Master.JvmMetrics.GcCountParNew +jvm.Master.JvmMetrics.GcCountG1 Old Generation +jvm.Master.JvmMetrics.GcCountG1 Young Generation jvm.Master.JvmMetrics.GcTimeMillis -jvm.Master.JvmMetrics.GcTimeMillisConcurrentMarkSweep -jvm.Master.JvmMetrics.GcTimeMillisParNew +jvm.Master.JvmMetrics.GcTimeMillisG1 Old Generation +jvm.Master.JvmMetrics.GcTimeMillisG1 Young Generation jvm.Master.JvmMetrics.LogError jvm.Master.JvmMetrics.LogFatal jvm.Master.JvmMetrics.LogInfo diff --git a/ambari-metrics-timelineservice/src/main/resources/metrics_def/HBASE_REGIONSERVER.dat b/ambari-metrics-timelineservice/src/main/resources/metrics_def/HBASE_REGIONSERVER.dat index 3b8e5862..e2581b81 100644 --- a/ambari-metrics-timelineservice/src/main/resources/metrics_def/HBASE_REGIONSERVER.dat +++ b/ambari-metrics-timelineservice/src/main/resources/metrics_def/HBASE_REGIONSERVER.dat @@ -26,11 +26,11 @@ ipc.IPC.queueSize ipc.IPC.receivedBytes ipc.IPC.sentBytes jvm.RegionServer.JvmMetrics.GcCount -jvm.RegionServer.JvmMetrics.GcCountConcurrentMarkSweep -jvm.RegionServer.JvmMetrics.GcCountParNew +jvm.RegionServer.JvmMetrics.GcCountG1 Old Generation +jvm.RegionServer.JvmMetrics.GcCountG1 Young Generation jvm.RegionServer.JvmMetrics.GcTimeMillis -jvm.RegionServer.JvmMetrics.GcTimeMillisConcurrentMarkSweep -jvm.RegionServer.JvmMetrics.GcTimeMillisParNew +jvm.RegionServer.JvmMetrics.GcTimeMillisG1 Old Generation +jvm.RegionServer.JvmMetrics.GcTimeMillisG1 Young Generation jvm.RegionServer.JvmMetrics.LogError jvm.RegionServer.JvmMetrics.LogFatal jvm.RegionServer.JvmMetrics.LogInfo diff --git a/ambari-metrics-timelineservice/src/main/resources/metrics_def/NAMENODE.dat b/ambari-metrics-timelineservice/src/main/resources/metrics_def/NAMENODE.dat index f9d2f27d..c3e280e6 100644 --- a/ambari-metrics-timelineservice/src/main/resources/metrics_def/NAMENODE.dat +++ b/ambari-metrics-timelineservice/src/main/resources/metrics_def/NAMENODE.dat @@ -267,13 +267,13 @@ dfs.namenode.TransactionsAvgTime dfs.namenode.TransactionsBatchedInSync dfs.namenode.TransactionsNumOps jvm.JvmMetrics.GcCount -jvm.JvmMetrics.GcCountConcurrentMarkSweep -jvm.JvmMetrics.GcCountParNew +jvm.JvmMetrics.GcCountG1 Old Generation +jvm.JvmMetrics.GcCountG1 Young Generation jvm.JvmMetrics.GcNumInfoThresholdExceeded jvm.JvmMetrics.GcNumWarnThresholdExceeded jvm.JvmMetrics.GcTimeMillis -jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep -jvm.JvmMetrics.GcTimeMillisParNew +jvm.JvmMetrics.GcTimeMillisG1 Old Generation +jvm.JvmMetrics.GcTimeMillisG1 Young Generation jvm.JvmMetrics.GcTotalExtraSleepTime jvm.JvmMetrics.LogError jvm.JvmMetrics.LogFatal diff --git a/ambari-metrics-timelineservice/src/test/resources/test_data/full_whitelist.dat b/ambari-metrics-timelineservice/src/test/resources/test_data/full_whitelist.dat index df6c9356..6dd8b037 100644 --- a/ambari-metrics-timelineservice/src/test/resources/test_data/full_whitelist.dat +++ b/ambari-metrics-timelineservice/src/test/resources/test_data/full_whitelist.dat @@ -60,8 +60,8 @@ regionserver.WAL.SyncTime_num_ops regionserver.WAL.appendCount regionserver.WAL.slowAppendCount jvm.JvmMetrics.GcTimeMillis -jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep -jvm.JvmMetrics.GcTimeMillisParNew +jvm.JvmMetrics.GcTimeMillisG1 Old Generation +jvm.JvmMetrics.GcTimeMillisG1 Young Generation ugi.UgiMetrics.GetGroupsAvgTime ugi.UgiMetrics.GetGroupsNumOps ugi.UgiMetrics.LoginFailureNumOps @@ -253,7 +253,7 @@ regionserver.Server.checkMutateFailedCount regionserver.Server.checkMutatePassedCount regionserver.Server.compactionQueueLength regionserver.Server.flushQueueLength -jvm.JvmMetrics.GcCountConcurrentMarkSweep +jvm.JvmMetrics.GcCountG1 Old Generation regionserver.Server.hlogFileCount regionserver.Server.hlogFileSize regionserver.Server.memStoreSize @@ -284,7 +284,7 @@ regionserver.WAL.AppendSize_max regionserver.WAL.AppendSize_mean regionserver.WAL.AppendSize_median regionserver.WAL.SyncTime_median -jvm.JvmMetrics.GcCountParNew +jvm.JvmMetrics.GcCountG1 Young Generation regionserver.WAL.AppendSize_min regionserver.WAL.AppendSize_num_ops regionserver.WAL.SyncTime_max @@ -322,8 +322,8 @@ regionserver.WAL.SyncTime_num_ops regionserver.WAL.appendCount regionserver.WAL.slowAppendCount jvm.JvmMetrics.GcTimeMillis -jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep -jvm.JvmMetrics.GcTimeMillisParNew +jvm.JvmMetrics.GcTimeMillisG1 Old Generation +jvm.JvmMetrics.GcTimeMillisG1 Young Generation ugi.UgiMetrics.GetGroupsAvgTime ugi.UgiMetrics.GetGroupsNumOps ugi.UgiMetrics.LoginFailureNumOps @@ -515,7 +515,7 @@ regionserver.Server.checkMutateFailedCount regionserver.Server.checkMutatePassedCount regionserver.Server.compactionQueueLength regionserver.Server.flushQueueLength -jvm.JvmMetrics.GcCountConcurrentMarkSweep +jvm.JvmMetrics.GcCountG1 Old Generation regionserver.Server.hlogFileCount regionserver.Server.hlogFileSize regionserver.Server.memStoreSize @@ -546,7 +546,7 @@ regionserver.WAL.AppendSize_max regionserver.WAL.AppendSize_mean regionserver.WAL.AppendSize_median regionserver.WAL.SyncTime_median -jvm.JvmMetrics.GcCountParNew +jvm.JvmMetrics.GcCountG1 Young Generation regionserver.WAL.AppendSize_min regionserver.WAL.AppendSize_num_ops regionserver.WAL.SyncTime_max @@ -585,7 +585,7 @@ regionserver.WAL.SyncTime_mean regionserver.WAL.AppendSize_99th_percentile jvm.JvmMetrics.GcTimeMillis regionserver.WAL.AppendSize_75th_percentile -jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep +jvm.JvmMetrics.GcTimeMillisG1 Old Generation regionserver.WAL.SyncTime_max regionserver.Server.Increment_median regionserver.Server.updatesBlockedTime @@ -595,7 +595,7 @@ regionserver.WAL.lowReplicaRollRequest ugi.UgiMetrics.GetGroupsNumOps regionserver.Server.storeFileSize regionserver.Server.Increment_95th_percentile -jvm.JvmMetrics.GcTimeMillisParNew +jvm.JvmMetrics.GcTimeMillisG1 Young Generation ugi.UgiMetrics.LoginFailureAvgTime ugi.UgiMetrics.LoginFailureNumOps regionserver.Server.storeFileCount @@ -885,7 +885,7 @@ regionserver.Server.compactionQueueLength regionserver.Server.flushedCellsSize regionserver.Server.memStoreSize regionserver.Server.mutationsWithoutWALSize -jvm.JvmMetrics.GcCountConcurrentMarkSweep +jvm.JvmMetrics.GcCountG1 Old Generation regionserver.Server.regionServerStartTime regionserver.Server.slowDeleteCount regionserver.Server.slowIncrementCount @@ -903,7 +903,7 @@ regionserver.Server.Replay_95th_percentile regionserver.WAL.AppendTime_95th_percentile regionserver.WAL.AppendTime_median regionserver.WAL.AppendTime_max -jvm.JvmMetrics.GcCountParNew +jvm.JvmMetrics.GcCountG1 Young Generation regionserver.WAL.AppendTime_mean FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.CacheCapacity FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.CacheUsed @@ -1100,13 +1100,13 @@ dfs.namenode.TransactionsAvgTime dfs.namenode.TransactionsBatchedInSync dfs.namenode.TransactionsNumOps jvm.JvmMetrics.GcCount -jvm.JvmMetrics.GcCountConcurrentMarkSweep -jvm.JvmMetrics.GcCountParNew +jvm.JvmMetrics.GcCountG1 Old Generation +jvm.JvmMetrics.GcCountG1 Young Generation jvm.JvmMetrics.GcNumInfoThresholdExceeded jvm.JvmMetrics.GcNumWarnThresholdExceeded jvm.JvmMetrics.GcTimeMillis -jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep -jvm.JvmMetrics.GcTimeMillisParNew +jvm.JvmMetrics.GcTimeMillisG1 Old Generation +jvm.JvmMetrics.GcTimeMillisG1 Young Generation jvm.JvmMetrics.GcTotalExtraSleepTime jvm.JvmMetrics.LogError jvm.JvmMetrics.LogFatal diff --git a/ambari-metrics-timelineservice/src/test/resources/ui_metrics_def/NAMENODE.dat b/ambari-metrics-timelineservice/src/test/resources/ui_metrics_def/NAMENODE.dat index b41c516b..0f65e7b3 100644 --- a/ambari-metrics-timelineservice/src/test/resources/ui_metrics_def/NAMENODE.dat +++ b/ambari-metrics-timelineservice/src/test/resources/ui_metrics_def/NAMENODE.dat @@ -21,8 +21,8 @@ dfs.FSNamesystem.UnderReplicatedBlocks || rpc.rpc.NumOpenConnections rpc.rpc.RpcQueueTimeAvgTime -jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep -jvm.JvmMetrics.GcCountConcurrentMarkSweep +jvm.JvmMetrics.GcTimeMillisG1 Old Generation +jvm.JvmMetrics.GcCountG1 Old Generation jvm.JvmMetrics.MemHeapCommittedM rpc.rpc.RpcProcessingTimeAvgTime jvm.JvmMetrics.MemHeapUsedM diff --git a/pom.xml b/pom.xml index e600d417..6f926efd 100644 --- a/pom.xml +++ b/pom.xml @@ -183,6 +183,7 @@ --add-opens java.base/java.util.regex=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/java.net=ALL-UNNAMED + --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED