Skip to content

Commit 48169b4

Browse files
abh1sarabh1sar
committed
Backup: publish the backup usage metric only when it changes
The backup sync published a BACKUP.USAGE.METRIC event for every VM on every run (every 5 minutes by default) even when nothing changed, which made up most of the usage_event rows. Keep the last value published per VM and offering in the new backup_usage_metric table and skip unchanged ones. The event and the stored value are committed together. Every management server runs the backup sync, so each zone is now synced under a per-zone lock. The usage metrics are then always compared against the last published values, and two servers no longer add or remove the same out-of-band backups at the same time. The VM's current offering is now always reported. It was skipped when the VM still had backups from an earlier offering, so deleting its last backup never brought its usage down to zero.
1 parent 5a64aad commit 48169b4

7 files changed

Lines changed: 391 additions & 11 deletions

File tree

‎engine/schema/src/main/java/org/apache/cloudstack/backup/BackupUsageMetricVO.java‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
package org.apache.cloudstack.backup;
18+
19+
import java.util.Date;
20+
21+
import javax.persistence.Column;
22+
import javax.persistence.Entity;
23+
import javax.persistence.GeneratedValue;
24+
import javax.persistence.GenerationType;
25+
import javax.persistence.Id;
26+
import javax.persistence.Table;
27+
import javax.persistence.Temporal;
28+
import javax.persistence.TemporalType;
29+
30+
import org.apache.cloudstack.api.InternalIdentity;
31+
32+
/**
33+
* The backup usage metric last published for a VM and backup offering.
34+
*/
35+
@Entity
36+
@Table(name = "backup_usage_metric")
37+
public class BackupUsageMetricVO implements InternalIdentity {
38+
39+
@Id
40+
@GeneratedValue(strategy = GenerationType.IDENTITY)
41+
@Column(name = "id")
42+
private long id;
43+
44+
@Column(name = "vm_id")
45+
private long vmId;
46+
47+
@Column(name = "backup_offering_id")
48+
private long backupOfferingId;
49+
50+
@Column(name = "size")
51+
private long size;
52+
53+
@Column(name = "protected_size")
54+
private long protectedSize;
55+
56+
@Column(name = "updated")
57+
@Temporal(value = TemporalType.TIMESTAMP)
58+
private Date updated;
59+
60+
protected BackupUsageMetricVO() {
61+
}
62+
63+
public BackupUsageMetricVO(long vmId, long backupOfferingId, long size, long protectedSize, Date updated) {
64+
this.vmId = vmId;
65+
this.backupOfferingId = backupOfferingId;
66+
this.size = size;
67+
this.protectedSize = protectedSize;
68+
this.updated = updated;
69+
}
70+
71+
@Override
72+
public long getId() {
73+
return id;
74+
}
75+
76+
public long getVmId() {
77+
return vmId;
78+
}
79+
80+
public long getBackupOfferingId() {
81+
return backupOfferingId;
82+
}
83+
84+
public long getSize() {
85+
return size;
86+
}
87+
88+
public void setSize(long size) {
89+
this.size = size;
90+
}
91+
92+
public long getProtectedSize() {
93+
return protectedSize;
94+
}
95+
96+
public void setProtectedSize(long protectedSize) {
97+
this.protectedSize = protectedSize;
98+
}
99+
100+
public Date getUpdated() {
101+
return updated;
102+
}
103+
104+
public void setUpdated(Date updated) {
105+
this.updated = updated;
106+
}
107+
}

‎engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupUsageMetricDao.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
package org.apache.cloudstack.backup.dao;
18+
19+
import org.apache.cloudstack.backup.BackupUsageMetricVO;
20+
21+
import com.cloud.utils.db.GenericDao;
22+
23+
public interface BackupUsageMetricDao extends GenericDao<BackupUsageMetricVO, Long> {
24+
BackupUsageMetricVO findByVmAndOffering(long vmId, long backupOfferingId);
25+
26+
int removeByVmAndOffering(long vmId, long backupOfferingId);
27+
}

‎engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupUsageMetricDaoImpl.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
package org.apache.cloudstack.backup.dao;
18+
19+
import javax.annotation.PostConstruct;
20+
21+
import org.apache.cloudstack.backup.BackupUsageMetricVO;
22+
23+
import com.cloud.utils.db.GenericDaoBase;
24+
import com.cloud.utils.db.SearchBuilder;
25+
import com.cloud.utils.db.SearchCriteria;
26+
27+
public class BackupUsageMetricDaoImpl extends GenericDaoBase<BackupUsageMetricVO, Long> implements BackupUsageMetricDao {
28+
private SearchBuilder<BackupUsageMetricVO> vmAndOfferingSearch;
29+
30+
@PostConstruct
31+
protected void init() {
32+
vmAndOfferingSearch = createSearchBuilder();
33+
vmAndOfferingSearch.and("vmId", vmAndOfferingSearch.entity().getVmId(), SearchCriteria.Op.EQ);
34+
vmAndOfferingSearch.and("backupOfferingId", vmAndOfferingSearch.entity().getBackupOfferingId(), SearchCriteria.Op.EQ);
35+
vmAndOfferingSearch.done();
36+
}
37+
38+
private SearchCriteria<BackupUsageMetricVO> createVmAndOfferingCriteria(long vmId, long backupOfferingId) {
39+
SearchCriteria<BackupUsageMetricVO> sc = vmAndOfferingSearch.create();
40+
sc.setParameters("vmId", vmId);
41+
sc.setParameters("backupOfferingId", backupOfferingId);
42+
return sc;
43+
}
44+
45+
@Override
46+
public BackupUsageMetricVO findByVmAndOffering(long vmId, long backupOfferingId) {
47+
return findOneBy(createVmAndOfferingCriteria(vmId, backupOfferingId));
48+
}
49+
50+
@Override
51+
public int removeByVmAndOffering(long vmId, long backupOfferingId) {
52+
return remove(createVmAndOfferingCriteria(vmId, backupOfferingId));
53+
}
54+
}

‎engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,7 @@
273273
<bean id="backupDaoImpl" class="org.apache.cloudstack.backup.dao.BackupDaoImpl" />
274274
<bean id="backupDetailsDaoImpl" class="org.apache.cloudstack.backup.dao.BackupDetailsDaoImpl" />
275275
<bean id="backupRepositoryDaoImpl" class="org.apache.cloudstack.backup.dao.BackupRepositoryDaoImpl" />
276+
<bean id="backupUsageMetricDaoImpl" class="org.apache.cloudstack.backup.dao.BackupUsageMetricDaoImpl" />
276277
<bean id="directDownloadCertificateDaoImpl" class="org.apache.cloudstack.direct.download.DirectDownloadCertificateDaoImpl" />
277278
<bean id="directDownloadCertificateHostMapDaoImpl" class="org.apache.cloudstack.direct.download.DirectDownloadCertificateHostMapDaoImpl" />
278279
<bean id="routerHealthCheckResultsDaoImpl" class="com.cloud.network.dao.RouterHealthCheckResultDaoImpl" />

‎engine/schema/src/main/resources/META-INF/db/schema-42210to42220.sql‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,15 @@
1818
--;
1919
-- Schema upgrade from 4.22.1.0 to 4.22.2.0
2020
--;
21+
22+
-- Last backup usage metric published per VM and backup offering
23+
CREATE TABLE IF NOT EXISTS `cloud`.`backup_usage_metric` (
24+
`id` bigint unsigned NOT NULL auto_increment COMMENT 'id',
25+
`vm_id` bigint unsigned NOT NULL COMMENT 'VM ID',
26+
`backup_offering_id` bigint unsigned NOT NULL COMMENT 'Backup offering ID',
27+
`size` bigint unsigned NOT NULL COMMENT 'Backup size last published',
28+
`protected_size` bigint unsigned NOT NULL COMMENT 'Protected size last published',
29+
`updated` datetime NOT NULL COMMENT 'Date the metric was last published',
30+
PRIMARY KEY (`id`),
31+
UNIQUE KEY `uk_backup_usage_metric__vm_id__backup_offering_id` (`vm_id`, `backup_offering_id`)
32+
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

‎server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java‎

Lines changed: 55 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
import org.apache.cloudstack.backup.dao.BackupDetailsDao;
7070
import org.apache.cloudstack.backup.dao.BackupOfferingDao;
7171
import org.apache.cloudstack.backup.dao.BackupScheduleDao;
72+
import org.apache.cloudstack.backup.dao.BackupUsageMetricDao;
7273
import org.apache.cloudstack.context.CallContext;
7374
import org.apache.cloudstack.framework.config.ConfigKey;
7475
import org.apache.cloudstack.framework.jobs.AsyncJobDispatcher;
@@ -182,6 +183,8 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager {
182183
@Inject
183184
private BackupDetailsDao backupDetailsDao;
184185
@Inject
186+
private BackupUsageMetricDao backupUsageMetricDao;
187+
@Inject
185188
private BackupScheduleDao backupScheduleDao;
186189
@Inject
187190
private BackupOfferingDao backupOfferingDao;
@@ -546,6 +549,7 @@ public boolean removeVMFromBackupOffering(final Long vmId, final boolean forced)
546549
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVED_AND_BACKUPS_DELETED, vm.getAccountId(), vm.getDataCenterId(), vm.getId(),
547550
"Backup-" + vm.getHostName() + "-" + vm.getUuid(), backupOfferingId, null, null,
548551
Backup.class.getSimpleName(), vm.getUuid());
552+
backupUsageMetricDao.removeByVmAndOffering(vm.getId(), backupOfferingId);
549553
}
550554
final List<BackupScheduleVO> backupSchedules = backupScheduleDao.listByVM(vm.getId());
551555
for(BackupSchedule backupSchedule: backupSchedules) {
@@ -1656,6 +1660,7 @@ private void checkAndGenerateUsageForLastBackupDeletedAfterOfferingRemove(Virtua
16561660
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVED_AND_BACKUPS_DELETED, vm.getAccountId(),
16571661
vm.getDataCenterId(), vm.getId(), "Backup-" + vm.getHostName() + "-" + vm.getUuid(),
16581662
backup.getBackupOfferingId(), null, null, Backup.class.getSimpleName(), vm.getUuid());
1663+
backupUsageMetricDao.removeByVmAndOffering(vm.getId(), backup.getBackupOfferingId());
16591664
}
16601665
}
16611666
}
@@ -1985,11 +1990,7 @@ protected void runInContext() {
19851990
continue;
19861991
}
19871992

1988-
backupProvider.syncBackupStorageStats(dataCenter.getId());
1989-
1990-
syncOutOfBandBackups(backupProvider, dataCenter);
1991-
1992-
updateBackupUsageRecords(backupProvider, dataCenter);
1993+
syncZone(backupProvider, dataCenter);
19931994
}
19941995
} catch (final Throwable t) {
19951996
logger.error(String.format("Error trying to run backup-sync background task due to: [%s].", t.getMessage()), t);
@@ -2014,7 +2015,30 @@ private void syncOutOfBandBackups(final BackupProvider backupProvider, DataCente
20142015
}
20152016
}
20162017

2017-
private void updateBackupUsageRecords(final BackupProvider backupProvider, DataCenter dataCenter) {
2018+
private void syncZone(final BackupProvider backupProvider, final DataCenter dataCenter) {
2019+
// Every management server runs this task. The lock lets one of them at a time sync a zone, so out-of-band
2020+
// backups are not added or removed twice and the usage metrics are compared against the last published ones.
2021+
GlobalLock lock = GlobalLock.getInternLock("backup.sync." + dataCenter.getId());
2022+
try {
2023+
if (!lock.lock(5)) {
2024+
logger.debug("Backups of zone {} are being synced by another management server, skipping.", dataCenter);
2025+
return;
2026+
}
2027+
try {
2028+
backupProvider.syncBackupStorageStats(dataCenter.getId());
2029+
2030+
syncOutOfBandBackups(backupProvider, dataCenter);
2031+
2032+
updateBackupUsageRecords(backupProvider, dataCenter);
2033+
} finally {
2034+
lock.unlock();
2035+
}
2036+
} finally {
2037+
lock.releaseRef();
2038+
}
2039+
}
2040+
2041+
protected void updateBackupUsageRecords(final BackupProvider backupProvider, DataCenter dataCenter) {
20182042
List<Long> vmIdsWithBackups = backupDao.listVmIdsWithBackupsInZone(dataCenter.getId());
20192043
List<VMInstanceVO> vmsWithBackups;
20202044
if (vmIdsWithBackups.size() == 0) {
@@ -2029,7 +2053,7 @@ private void updateBackupUsageRecords(final BackupProvider backupProvider, DataC
20292053

20302054
Map<Long, Pair<Long, Long>> backupOfferingToSizeMap = new HashMap<>();
20312055
List<Backup> backups = backupDao.listByVmId(null, vm.getId());
2032-
if (backups.isEmpty() && vm.getBackupOfferingId() != null) {
2056+
if (vm.getBackupOfferingId() != null) {
20332057
backupOfferingToSizeMap.put(vm.getBackupOfferingId(), new Pair<>(0L, 0L));
20342058
}
20352059
for (final Backup backup: backups) {
@@ -2053,14 +2077,34 @@ private void updateBackupUsageRecords(final BackupProvider backupProvider, DataC
20532077
for (final Map.Entry<Long, Pair<Long, Long>> entry : backupOfferingToSizeMap.entrySet()) {
20542078
Long offeringId = entry.getKey();
20552079
Pair<Long, Long> sizes = entry.getValue();
2056-
Long backupSize = sizes.first();
2057-
Long protectedSize = sizes.second();
2080+
publishBackupUsageMetricIfChanged(vm, offeringId, sizes.first(), sizes.second());
2081+
}
2082+
}
2083+
}
2084+
2085+
protected void publishBackupUsageMetricIfChanged(final VirtualMachine vm, final long offeringId, final long size, final long protectedSize) {
2086+
final BackupUsageMetricVO lastPublished = backupUsageMetricDao.findByVmAndOffering(vm.getId(), offeringId);
2087+
if (lastPublished != null && lastPublished.getSize() == size && lastPublished.getProtectedSize() == protectedSize) {
2088+
return;
2089+
}
2090+
// The event and the value the next sync compares against are committed together.
2091+
Transaction.execute(new TransactionCallbackNoReturn() {
2092+
@Override
2093+
public void doInTransactionWithoutResult(TransactionStatus status) {
20582094
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_USAGE_METRIC, vm.getAccountId(),
20592095
vm.getDataCenterId(), vm.getId(), "Backup-" + vm.getHostName() + "-" + vm.getUuid(),
2060-
offeringId, null, backupSize, protectedSize,
2096+
offeringId, null, size, protectedSize,
20612097
Backup.class.getSimpleName(), vm.getUuid());
2098+
if (lastPublished == null) {
2099+
backupUsageMetricDao.persist(new BackupUsageMetricVO(vm.getId(), offeringId, size, protectedSize, new Date()));
2100+
} else {
2101+
lastPublished.setSize(size);
2102+
lastPublished.setProtectedSize(protectedSize);
2103+
lastPublished.setUpdated(new Date());
2104+
backupUsageMetricDao.update(lastPublished.getId(), lastPublished);
2105+
}
20622106
}
2063-
}
2107+
});
20642108
}
20652109

20662110
private Backup checkAndUpdateIfBackupEntryExistsForRestorePoint(Backup.RestorePoint restorePoint, List<Backup> backupsInDb, VirtualMachine vm) {

0 commit comments

Comments
 (0)