Events listing, deletion and archival optimizations

api/src/main/java/org/apache/cloudstack/api/command/user/event/ArchiveEventsCmd.java

@@ -31,7 +31,6 @@
 import org.apache.cloudstack.context.CallContext;
 
 import com.cloud.event.Event;
-import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.user.Account;
 
 @APICommand(name = "archiveEvents", description = "Archive one or more events.", responseObject = SuccessResponse.class, entityType = {Event.class},
@@ -82,6 +81,22 @@ public String getType() {
         return type;
     }
 
+    public void setIds(List<Long> ids) {
+        this.ids = ids;
+    }
+
+    public void setEndDate(Date endDate) {
+        this.endDate = endDate;
+    }
+
+    public void setStartDate(Date startDate) {
+        this.startDate = startDate;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
@@ -97,17 +112,12 @@ public long getEntityOwnerId() {
 
     @Override
     public void execute() {
-        if (ids == null && type == null && endDate == null) {
-            throw new InvalidParameterValueException("either ids, type or enddate must be specified");
-        } else if (startDate != null && endDate == null) {
-            throw new InvalidParameterValueException("enddate must be specified with startdate parameter");
-        }
         boolean result = _mgr.archiveEvents(this);
         if (result) {
             SuccessResponse response = new SuccessResponse(getCommandName());
             setResponseObject(response);
-        } else {
-            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Unable to archive Events, one or more parameters has invalid values");
+            return;
         }
+        throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Unable to archive Events. One or more parameters have invalid values.");
     }
 }

api/src/main/java/org/apache/cloudstack/api/command/user/event/DeleteEventsCmd.java

@@ -31,7 +31,6 @@
 import org.apache.cloudstack.context.CallContext;
 
 import com.cloud.event.Event;
-import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.user.Account;
 
 @APICommand(name = "deleteEvents", description = "Delete one or more events.", responseObject = SuccessResponse.class, entityType = {Event.class},
@@ -82,6 +81,22 @@ public String getType() {
         return type;
     }
 
+    public void setIds(List<Long> ids) {
+        this.ids = ids;
+    }
+
+    public void setEndDate(Date endDate) {
+        this.endDate = endDate;
+    }
+
+    public void setStartDate(Date startDate) {
+        this.startDate = startDate;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+
     // ///////////////////////////////////////////////////
     // ///////////// API Implementation///////////////////
     // ///////////////////////////////////////////////////
@@ -97,17 +112,12 @@ public long getEntityOwnerId() {
 
     @Override
     public void execute() {
-        if (ids == null && type == null && endDate == null) {
-            throw new InvalidParameterValueException("either ids, type or enddate must be specified");
-        } else if (startDate != null && endDate == null) {
-            throw new InvalidParameterValueException("enddate must be specified with startdate parameter");
-        }
         boolean result = _mgr.deleteEvents(this);
         if (result) {
             SuccessResponse response = new SuccessResponse(getCommandName());
             setResponseObject(response);
         } else {
-            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Unable to delete Events, one or more parameters has invalid values");
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Unable to delete any events. One or more parameters have invalid values.");
         }
     }
 }

api/src/main/java/org/apache/cloudstack/api/command/user/event/ListEventsCmd.java

@@ -134,7 +134,6 @@ public String getState() {
 
     @Override
     public void execute() {
-
         ListResponse<EventResponse> response = _queryService.searchForEvents(this);
         response.setResponseName(getCommandName());
         setResponseObject(response);

engine/schema/src/main/java/com/cloud/event/dao/EventDao.java

@@ -21,17 +21,9 @@
 
 import com.cloud.event.Event;
 import com.cloud.event.EventVO;
-import com.cloud.utils.db.Filter;
 import com.cloud.utils.db.GenericDao;
-import com.cloud.utils.db.SearchCriteria;
 
 public interface EventDao extends GenericDao<EventVO, Long> {
-    public List<EventVO> searchAllEvents(SearchCriteria<EventVO> sc, Filter filter);
-
-    public List<EventVO> listOlderEvents(Date oldTime);
-
-    EventVO findCompletedEvent(long startId);
-
     /**
      * Finds the last non-archived start event matching the specified criteria.
      * Events are ordered by ID in descending order, returning the most recent one.
@@ -44,8 +36,10 @@ public interface EventDao extends GenericDao<EventVO, Long> {
      */
     EventVO findLastEvent(String type, Event.State state, Long resourceId, String resourceType);
 
-    public List<EventVO> listToArchiveOrDeleteEvents(List<Long> ids, String type, Date startDate, Date endDate, List<Long> accountIds);
+    long archiveEvents(List<Long> ids, String type, Date startDate, Date endDate, Long accountId, List<Long> domainIds,
+                       long limitPerQuery);
 
-    public void archiveEvents(List<EventVO> events);
+    long purgeAll(List<Long> ids, Date startDate, Date endDate, Date limitDate, String type, Long accountId, List<Long> domainIds,
+                  long limitPerQuery);
 
 }

engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java

@@ -18,15 +18,13 @@
 
 import java.util.Date;
 import java.util.List;
-import java.util.stream.Collectors;
 
 
-import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Component;
 
 import com.cloud.event.Event.State;
 import com.cloud.event.EventVO;
-import com.cloud.utils.db.Filter;
 import com.cloud.utils.db.GenericDaoBase;
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
@@ -35,31 +33,23 @@
 
 @Component
 public class EventDaoImpl extends GenericDaoBase<EventVO, Long> implements EventDao {
-    protected final SearchBuilder<EventVO> CompletedEventSearch;
+
     protected final SearchBuilder<EventVO> ToArchiveOrDeleteEventSearch;
-    protected final SearchBuilder<EventVO> ArchiveByIdsSearch;
     protected final SearchBuilder<EventVO> LastStartEventSearch;
 
     public EventDaoImpl() {
-        CompletedEventSearch = createSearchBuilder();
-        CompletedEventSearch.and("state", CompletedEventSearch.entity().getState(), SearchCriteria.Op.EQ);
-        CompletedEventSearch.and("startId", CompletedEventSearch.entity().getStartId(), SearchCriteria.Op.EQ);
-        CompletedEventSearch.and("archived", CompletedEventSearch.entity().getArchived(), Op.EQ);
-        CompletedEventSearch.done();
-
         ToArchiveOrDeleteEventSearch = createSearchBuilder();
+        ToArchiveOrDeleteEventSearch.select("id", SearchCriteria.Func.NATIVE, ToArchiveOrDeleteEventSearch.entity().getId());
         ToArchiveOrDeleteEventSearch.and("id", ToArchiveOrDeleteEventSearch.entity().getId(), Op.IN);
         ToArchiveOrDeleteEventSearch.and("type", ToArchiveOrDeleteEventSearch.entity().getType(), Op.EQ);
-        ToArchiveOrDeleteEventSearch.and("accountIds", ToArchiveOrDeleteEventSearch.entity().getAccountId(), Op.IN);
+        ToArchiveOrDeleteEventSearch.and("accountId", ToArchiveOrDeleteEventSearch.entity().getAccountId(), Op.EQ);
+        ToArchiveOrDeleteEventSearch.and("domainIds", ToArchiveOrDeleteEventSearch.entity().getDomainId(), Op.IN);
         ToArchiveOrDeleteEventSearch.and("createdDateB", ToArchiveOrDeleteEventSearch.entity().getCreateDate(), Op.BETWEEN);
         ToArchiveOrDeleteEventSearch.and("createdDateL", ToArchiveOrDeleteEventSearch.entity().getCreateDate(), Op.LTEQ);
+        ToArchiveOrDeleteEventSearch.and("createdDateLT", ToArchiveOrDeleteEventSearch.entity().getCreateDate(), Op.LT);
         ToArchiveOrDeleteEventSearch.and("archived", ToArchiveOrDeleteEventSearch.entity().getArchived(), Op.EQ);
         ToArchiveOrDeleteEventSearch.done();
 
-        ArchiveByIdsSearch = createSearchBuilder();
-        ArchiveByIdsSearch.and("id", ArchiveByIdsSearch.entity().getId(), Op.IN);
-        ArchiveByIdsSearch.done();
-
         LastStartEventSearch = createSearchBuilder();
         LastStartEventSearch.and("type", LastStartEventSearch.entity().getType(), Op.EQ);
         LastStartEventSearch.and("state", LastStartEventSearch.entity().getState(), Op.EQ);
@@ -69,30 +59,6 @@ public EventDaoImpl() {
         LastStartEventSearch.done();
     }
 
-    @Override
-    public List<EventVO> searchAllEvents(SearchCriteria<EventVO> sc, Filter filter) {
-        return listIncludingRemovedBy(sc, filter);
-    }
-
-    @Override
-    public List<EventVO> listOlderEvents(Date oldTime) {
-        if (oldTime == null)
-            return null;
-        SearchCriteria<EventVO> sc = createSearchCriteria();
-        sc.addAnd("createDate", SearchCriteria.Op.LT, oldTime);
-        sc.addAnd("archived", SearchCriteria.Op.EQ, false);
-        return listIncludingRemovedBy(sc, null);
-    }
-
-    @Override
-    public EventVO findCompletedEvent(long startId) {
-        SearchCriteria<EventVO> sc = CompletedEventSearch.create();
-        sc.setParameters("state", State.Completed);
-        sc.setParameters("startId", startId);
-        sc.setParameters("archived", false);
-        return findOneIncludingRemovedBy(sc);
-    }
-
     @Override
     public EventVO findLastEvent(String type, State state, Long resourceId, String resourceType) {
         SearchCriteria<EventVO> sc = LastStartEventSearch.create();
@@ -104,39 +70,51 @@ public EventVO findLastEvent(String type, State state, Long resourceId, String r
         return findLastOneBy(sc);
     }
 
-    @Override
-    public List<EventVO> listToArchiveOrDeleteEvents(List<Long> ids, String type, Date startDate, Date endDate, List<Long> accountIds) {
+    private SearchCriteria<EventVO> createEventSearchCriteria(List<Long> ids, String type, Date startDate, Date endDate,
+                                                              Date limitDate, Long accountId, List<Long> domainIds) {
         SearchCriteria<EventVO> sc = ToArchiveOrDeleteEventSearch.create();
-        if (ids != null) {
-            sc.setParameters("id", ids.toArray(new Object[ids.size()]));
+
+        if (CollectionUtils.isNotEmpty(ids)) {
+            sc.setParameters("id", ids.toArray(new Object[0]));
         }
-        if (type != null) {
-            sc.setParameters("type", type);
+        if (CollectionUtils.isNotEmpty(domainIds)) {
+            sc.setParameters("domainIds", domainIds.toArray(new Object[0]));
         }
         if (startDate != null && endDate != null) {
             sc.setParameters("createdDateB", startDate, endDate);
         } else if (endDate != null) {
             sc.setParameters("createdDateL", endDate);
         }
-        if (accountIds != null && !accountIds.isEmpty()) {
-            sc.setParameters("accountIds", accountIds.toArray(new Object[accountIds.size()]));
-        }
+        sc.setParametersIfNotNull("accountId", accountId);
+        sc.setParametersIfNotNull("createdDateLT", limitDate);
+        sc.setParametersIfNotNull("type", type);
         sc.setParameters("archived", false);
-        return search(sc, null);
+
+        return sc;
     }
 
     @Override
-    public void archiveEvents(List<EventVO> events) {
-        if (CollectionUtils.isEmpty(events)) {
-            return;
-        }
+    public long archiveEvents(List<Long> ids, String type, Date startDate, Date endDate, Long accountId, List<Long> domainIds,
+                              long limitPerQuery) {
+        SearchCriteria<EventVO> sc = createEventSearchCriteria(ids, type, startDate, endDate, null, accountId, domainIds);
+
+        long totalArchived = 0L;
+        int archived;
+        do {
+            EventVO eventForUpdate = createForUpdate();
+            eventForUpdate.setArchived(true);
+            UpdateBuilder ub = getUpdateBuilder(eventForUpdate);
+            archived = update(ub, sc, limitPerQuery > 0 ? (int) Math.min(limitPerQuery, Integer.MAX_VALUE) : null);
+            totalArchived += archived;
+        } while (limitPerQuery > 0 && archived >= limitPerQuery);
+
+        return totalArchived;
+    }
 
-        List<Long> ids = events.stream().map(EventVO::getId).collect(Collectors.toList());
-        SearchCriteria<EventVO> sc = ArchiveByIdsSearch.create();
-        sc.setParameters("id", ids.toArray(new Object[ids.size()]));
-        EventVO eventForUpdate = createForUpdate();
-        eventForUpdate.setArchived(true);
-        UpdateBuilder ub = getUpdateBuilder(eventForUpdate);
-        update(ub, sc, null);
+    @Override
+    public long purgeAll(List<Long> ids, Date startDate, Date endDate, Date limitDate, String type, Long accountId,
+                         List<Long> domainIds, long limitPerQuery) {
+        SearchCriteria<EventVO> sc = createEventSearchCriteria(ids, type, startDate, endDate, limitDate, accountId, domainIds);
+        return batchExpunge(sc, limitPerQuery);
     }
 }

engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java

@@ -144,6 +144,20 @@ public void createIndex(Connection conn, String tableName, String indexName, Str
         }
     }
 
+    public void dropIndexIfExists(Connection conn, String tableName, String indexName) {
+        if (!indexExists(conn, tableName, indexName)) {
+            return;
+        }
+        String stmt = String.format("DROP INDEX %s ON %s", indexName, tableName);
+        logger.debug("Statement: {}", stmt);
+        try (PreparedStatement pstmt = conn.prepareStatement(stmt)) {
+            pstmt.execute();
+            logger.debug("Dropped index {}", indexName);
+        } catch (SQLException e) {
+            logger.warn("Unable to drop index {}", indexName, e);
+        }
+    }
+
     public void renameIndex(Connection conn, String tableName, String oldName, String newName) {
         String stmt = String.format("ALTER TABLE %s RENAME INDEX %s TO %s", tableName, oldName, newName);
         logger.debug("Statement: {}", stmt);

engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java

@@ -25,12 +25,23 @@ public class DbUpgradeUtils {
 
     public static void addIndexIfNeeded(Connection conn, String tableName, String... columnNames) {
         String indexName = dao.generateIndexName(tableName, columnNames);
+        addIndexWithNameIfNeeded(conn, tableName, indexName, columnNames);
+    }
 
+    public static void addIndexWithNameIfNeeded(Connection conn, String tableName, String indexName, String... columnNames) {
         if (!dao.indexExists(conn, tableName, indexName)) {
             dao.createIndex(conn, tableName, indexName, columnNames);
         }
     }
 
+    public static boolean indexExists(Connection conn, String tableName, String indexName) {
+        return dao.indexExists(conn, tableName, indexName);
+    }
+
+    public static void dropIndexIfExists(Connection conn, String tableName, String indexName) {
+        dao.dropIndexIfExists(conn, tableName, indexName);
+    }
+
     public static void renameIndexIfNeeded(Connection conn, String tableName, String oldName, String newName) {
         if (!dao.indexExists(conn, tableName, oldName)) {
             dao.renameIndex(conn, tableName, oldName, newName);

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42300to2400.java

@@ -16,6 +16,8 @@
 // under the License.
 package com.cloud.upgrade.dao;
 
+import java.sql.Connection;
+
 public class Upgrade42300to2400 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate {
 
     @Override
@@ -27,4 +29,25 @@ public String[] getUpgradableVersionRange() {
     public String getUpgradedVersion() {
         return "24.0.0";
     }
+
+    @Override
+    public void performDataMigration(Connection conn) {
+        addIndexes(conn);
+        dropOldIndexes(conn);
+    }
+
+    private void addIndexes(Connection conn) {
+        DbUpgradeUtils.addIndexWithNameIfNeeded(conn, "event", "i_event__archived__created__id",
+                "archived", "created", "id");
+        DbUpgradeUtils.addIndexWithNameIfNeeded(conn, "event", "i_event__account_id__archived__created__id",
+                "account_id", "archived", "created", "id");
+        DbUpgradeUtils.addIndexWithNameIfNeeded(conn, "event", "i_event__domain_id__archived__created__id",
+                "domain_id", "archived", "created", "id");
+        DbUpgradeUtils.addIndexWithNameIfNeeded(conn, "event", "i_event__resource_type__resource_id__archived__created__id",
+                "resource_type", "resource_id", "archived", "created", "id");
+    }
+
+    private void dropOldIndexes(Connection conn) {
+        DbUpgradeUtils.dropIndexIfExists(conn, "event", "i_event__resource_type__resource_id");
+    }
 }

server/src/main/java/com/cloud/api/query/QueryManagerImpl.java

@@ -963,7 +963,7 @@ private Pair<List<Long>, Integer> searchForEventIdsAndCount(ListEventsCmd cmd) {
         searchFilter.addOrderBy(EventVO.class, "id", false);
 
         SearchBuilder<EventVO> eventSearchBuilder = eventDao.createSearchBuilder();
-        eventSearchBuilder.select(null, Func.DISTINCT, eventSearchBuilder.entity().getId());
+        eventSearchBuilder.select(null, Func.NATIVE, eventSearchBuilder.entity().getId());
         accountMgr.buildACLSearchBuilder(eventSearchBuilder, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria);
 
         eventSearchBuilder.and("id", eventSearchBuilder.entity().getId(), SearchCriteria.Op.EQ);

server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -561,7 +561,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
     public static final ConfigKey<Long> DELETE_QUERY_BATCH_SIZE = new ConfigKey<>("Advanced", Long.class, "delete.query.batch.size", "0",
             "Indicates the limit applied while deleting entries in bulk. With this, the delete query will apply the limit as many times as necessary," +
                     " to delete all the entries. This is advised when retaining several days of records, which can lead to slowness. <= 0 means that no limit will " +
-                    "be applied. Default value is 0. For now, this is used for deletion of VM stats, volume stats, and usage records.", true);
+                    "be applied. Default value is 0. For now, this is used for deletion of VM stats, volume stats, events, and usage records.", true);
 
     private static final String IOPS_READ_RATE = "IOPS Read";
     private static final String IOPS_WRITE_RATE = "IOPS Write";