Skip to content

Commit 5dbc2ef

Browse files
abh1sarabh1sar
committed
Backup: publish the backup usage metric only when it changes
The backup sync published a BACKUP.USAGE.METRIC event for every VM on every run (every 5 minutes by default) even when nothing changed, which made up most of the usage_event rows. Remember the last value published per VM and offering and skip unchanged ones. The usage server ignores unchanged sizes, so republishing after a restart or on another management server is harmless. The VM's current offering is now always reported. It was skipped when the VM still had backups from an earlier offering, so deleting its last backup never brought its usage down to zero.
1 parent cec8bc4 commit 5dbc2ef

2 files changed

Lines changed: 104 additions & 2 deletions

File tree

‎server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import java.util.TimeZone;
3333
import java.util.Timer;
3434
import java.util.TimerTask;
35+
import java.util.concurrent.ConcurrentHashMap;
3536
import java.util.stream.Collectors;
3637
import java.util.stream.Stream;
3738

@@ -249,6 +250,12 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager {
249250
private static Map<String, BackupProvider> backupProvidersMap = new HashMap<>();
250251
private List<BackupProvider> backupProviders;
251252

253+
// Per zone, the backup usage metric last published for each (VM, backup offering) pair, as (size,
254+
// protected size). The backup sync runs every few minutes but the usage server only needs the
255+
// metric when it changes. After a restart or on another management server each value is
256+
// published once more, which the usage server ignores as the size is unchanged.
257+
private final Map<Long, Map<Pair<Long, Long>, Pair<Long, Long>>> lastPublishedBackupUsageMetrics = new ConcurrentHashMap<>();
258+
252259
public AsyncJobDispatcher getAsyncJobDispatcher() {
253260
return asyncJobDispatcher;
254261
}
@@ -2014,7 +2021,7 @@ private void syncOutOfBandBackups(final BackupProvider backupProvider, DataCente
20142021
}
20152022
}
20162023

2017-
private void updateBackupUsageRecords(final BackupProvider backupProvider, DataCenter dataCenter) {
2024+
protected void updateBackupUsageRecords(final BackupProvider backupProvider, DataCenter dataCenter) {
20182025
List<Long> vmIdsWithBackups = backupDao.listVmIdsWithBackupsInZone(dataCenter.getId());
20192026
List<VMInstanceVO> vmsWithBackups;
20202027
if (vmIdsWithBackups.size() == 0) {
@@ -2025,11 +2032,13 @@ private void updateBackupUsageRecords(final BackupProvider backupProvider, DataC
20252032
List<VMInstanceVO> vmsWithBackupOffering = vmInstanceDao.listByZoneAndBackupOffering(dataCenter.getId(), null); //should return including removed
20262033
Set<VMInstanceVO> vms = Stream.concat(vmsWithBackups.stream(), vmsWithBackupOffering.stream()) .collect(Collectors.toSet());
20272034

2035+
Map<Pair<Long, Long>, Pair<Long, Long>> lastPublished = lastPublishedBackupUsageMetrics.getOrDefault(dataCenter.getId(), Collections.emptyMap());
2036+
Map<Pair<Long, Long>, Pair<Long, Long>> published = new HashMap<>();
20282037
for (final VirtualMachine vm : vms) {
20292038

20302039
Map<Long, Pair<Long, Long>> backupOfferingToSizeMap = new HashMap<>();
20312040
List<Backup> backups = backupDao.listByVmId(null, vm.getId());
2032-
if (backups.isEmpty() && vm.getBackupOfferingId() != null) {
2041+
if (vm.getBackupOfferingId() != null) {
20332042
backupOfferingToSizeMap.put(vm.getBackupOfferingId(), new Pair<>(0L, 0L));
20342043
}
20352044
for (final Backup backup: backups) {
@@ -2055,12 +2064,20 @@ private void updateBackupUsageRecords(final BackupProvider backupProvider, DataC
20552064
Pair<Long, Long> sizes = entry.getValue();
20562065
Long backupSize = sizes.first();
20572066
Long protectedSize = sizes.second();
2067+
Pair<Long, Long> key = new Pair<>(vm.getId(), offeringId);
2068+
Pair<Long, Long> metric = new Pair<>(backupSize, protectedSize);
2069+
published.put(key, metric);
2070+
if (metric.equals(lastPublished.get(key))) {
2071+
continue;
2072+
}
20582073
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_USAGE_METRIC, vm.getAccountId(),
20592074
vm.getDataCenterId(), vm.getId(), "Backup-" + vm.getHostName() + "-" + vm.getUuid(),
20602075
offeringId, null, backupSize, protectedSize,
20612076
Backup.class.getSimpleName(), vm.getUuid());
20622077
}
20632078
}
2079+
// Replacing the map also drops pairs that are no longer reported, e.g. expunged VMs.
2080+
lastPublishedBackupUsageMetrics.put(dataCenter.getId(), published);
20642081
}
20652082

20662083
private Backup checkAndUpdateIfBackupEntryExistsForRestorePoint(Backup.RestorePoint restorePoint, List<Backup> backupsInDb, VirtualMachine vm) {

‎server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -898,6 +898,91 @@ public void testBackupSyncTask() {
898898
}
899899
}
900900

901+
private void verifyBackupUsageMetricPublished(MockedStatic<UsageEventUtils> usageEventUtilsMocked, int times, Long size, Long protectedSize) {
902+
usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(Mockito.eq(EventTypes.EVENT_VM_BACKUP_USAGE_METRIC), Mockito.anyLong(),
903+
Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.anyLong(), Mockito.any(), Mockito.eq(size), Mockito.eq(protectedSize),
904+
Mockito.any(), Mockito.any()), times(times));
905+
}
906+
907+
@Test
908+
public void updateBackupUsageRecordsPublishesMetricOnlyWhenItChanges() {
909+
Long dataCenterId = 1L;
910+
Long vmId = 2L;
911+
Long offeringId = 3L;
912+
913+
DataCenterVO dataCenter = mock(DataCenterVO.class);
914+
when(dataCenter.getId()).thenReturn(dataCenterId);
915+
VMInstanceVO vm = mock(VMInstanceVO.class);
916+
when(vm.getId()).thenReturn(vmId);
917+
when(backupDao.listVmIdsWithBackupsInZone(dataCenterId)).thenReturn(List.of(vmId));
918+
when(vmInstanceDao.listByIdsIncludingRemoved(List.of(vmId))).thenReturn(List.of(vm));
919+
when(vmInstanceDao.listByZoneAndBackupOffering(dataCenterId, null)).thenReturn(List.of(vm));
920+
921+
BackupVO backup = new BackupVO();
922+
backup.setBackupOfferingId(offeringId);
923+
backup.setSize(100L);
924+
backup.setProtectedSize(1000L);
925+
when(backupDao.listByVmId(null, vmId)).thenReturn(List.of(backup));
926+
927+
BackupManagerImpl.BackupSyncTask backupSyncTask = backupManager.new BackupSyncTask(backupManager);
928+
BackupProvider backupProvider = mock(BackupProvider.class);
929+
try (MockedStatic<UsageEventUtils> usageEventUtilsMocked = Mockito.mockStatic(UsageEventUtils.class)) {
930+
backupSyncTask.updateBackupUsageRecords(backupProvider, dataCenter);
931+
backupSyncTask.updateBackupUsageRecords(backupProvider, dataCenter);
932+
verifyBackupUsageMetricPublished(usageEventUtilsMocked, 1, 100L, 1000L);
933+
934+
backup.setSize(200L);
935+
backupSyncTask.updateBackupUsageRecords(backupProvider, dataCenter);
936+
backupSyncTask.updateBackupUsageRecords(backupProvider, dataCenter);
937+
verifyBackupUsageMetricPublished(usageEventUtilsMocked, 1, 200L, 1000L);
938+
939+
// A VM that is no longer reported is forgotten, so it is published again if it comes back.
940+
when(backupDao.listVmIdsWithBackupsInZone(dataCenterId)).thenReturn(List.of());
941+
when(vmInstanceDao.listByZoneAndBackupOffering(dataCenterId, null)).thenReturn(List.of());
942+
backupSyncTask.updateBackupUsageRecords(backupProvider, dataCenter);
943+
when(backupDao.listVmIdsWithBackupsInZone(dataCenterId)).thenReturn(List.of(vmId));
944+
when(vmInstanceDao.listByZoneAndBackupOffering(dataCenterId, null)).thenReturn(List.of(vm));
945+
backupSyncTask.updateBackupUsageRecords(backupProvider, dataCenter);
946+
verifyBackupUsageMetricPublished(usageEventUtilsMocked, 2, 200L, 1000L);
947+
}
948+
}
949+
950+
@Test
951+
public void updateBackupUsageRecordsReportsCurrentOfferingWithoutBackups() {
952+
Long dataCenterId = 1L;
953+
Long vmId = 2L;
954+
Long oldOfferingId = 3L;
955+
Long currentOfferingId = 4L;
956+
957+
DataCenterVO dataCenter = mock(DataCenterVO.class);
958+
when(dataCenter.getId()).thenReturn(dataCenterId);
959+
VMInstanceVO vm = mock(VMInstanceVO.class);
960+
when(vm.getId()).thenReturn(vmId);
961+
when(vm.getBackupOfferingId()).thenReturn(currentOfferingId);
962+
when(backupDao.listVmIdsWithBackupsInZone(dataCenterId)).thenReturn(List.of(vmId));
963+
when(vmInstanceDao.listByIdsIncludingRemoved(List.of(vmId))).thenReturn(List.of(vm));
964+
when(vmInstanceDao.listByZoneAndBackupOffering(dataCenterId, null)).thenReturn(List.of(vm));
965+
966+
// Backups kept from an offering the VM had before; none for its current offering.
967+
BackupVO oldBackup = new BackupVO();
968+
oldBackup.setBackupOfferingId(oldOfferingId);
969+
oldBackup.setSize(100L);
970+
oldBackup.setProtectedSize(1000L);
971+
when(backupDao.listByVmId(null, vmId)).thenReturn(List.of(oldBackup));
972+
973+
BackupManagerImpl.BackupSyncTask backupSyncTask = backupManager.new BackupSyncTask(backupManager);
974+
try (MockedStatic<UsageEventUtils> usageEventUtilsMocked = Mockito.mockStatic(UsageEventUtils.class)) {
975+
backupSyncTask.updateBackupUsageRecords(mock(BackupProvider.class), dataCenter);
976+
977+
usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(Mockito.eq(EventTypes.EVENT_VM_BACKUP_USAGE_METRIC), Mockito.anyLong(),
978+
Mockito.anyLong(), Mockito.eq(vmId), Mockito.any(), Mockito.eq(currentOfferingId), Mockito.any(), Mockito.eq(0L), Mockito.eq(0L),
979+
Mockito.any(), Mockito.any()));
980+
usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(Mockito.eq(EventTypes.EVENT_VM_BACKUP_USAGE_METRIC), Mockito.anyLong(),
981+
Mockito.anyLong(), Mockito.eq(vmId), Mockito.any(), Mockito.eq(oldOfferingId), Mockito.any(), Mockito.eq(100L), Mockito.eq(1000L),
982+
Mockito.any(), Mockito.any()));
983+
}
984+
}
985+
901986
@Test
902987
public void checkCallerAccessToBackupScheduleVmTestExecuteAccessCheckMethods() {
903988
long vmId = 1L;

0 commit comments

Comments
 (0)