StorageManagerImpl.java

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.storage;

import static com.cloud.configuration.ConfigurationManagerImpl.SystemVMUseLocalStorage;
import static com.cloud.utils.NumbersUtil.toHumanReadableSize;

import java.math.BigDecimal;
import java.math.BigInteger;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import javax.inject.Inject;

import org.apache.cloudstack.annotation.AnnotationService;
import org.apache.cloudstack.annotation.dao.AnnotationDao;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd;
import org.apache.cloudstack.api.command.admin.storage.ChangeStoragePoolScopeCmd;
import org.apache.cloudstack.api.command.admin.storage.ConfigureStorageAccessCmd;
import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.CreateStoragePoolCmd;
import org.apache.cloudstack.api.command.admin.storage.DeleteImageStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.DeleteObjectStoragePoolCmd;
import org.apache.cloudstack.api.command.admin.storage.DeletePoolCmd;
import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.SyncStoragePoolCmd;
import org.apache.cloudstack.api.command.admin.storage.UpdateImageStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.UpdateObjectStoragePoolCmd;
import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd;
import org.apache.cloudstack.api.command.admin.storage.heuristics.CreateSecondaryStorageSelectorCmd;
import org.apache.cloudstack.api.command.admin.storage.heuristics.RemoveSecondaryStorageSelectorCmd;
import org.apache.cloudstack.api.command.admin.storage.heuristics.UpdateSecondaryStorageSelectorCmd;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
import org.apache.cloudstack.engine.subsystem.api.storage.HostScope;
import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener;
import org.apache.cloudstack.engine.subsystem.api.storage.ImageStoreProvider;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreLifeCycle;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreProvider;
import org.apache.cloudstack.engine.subsystem.api.storage.Scope;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService.VolumeApiResult;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.framework.async.AsyncCallFuture;
import org.apache.cloudstack.framework.config.ConfigDepot;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.jsinterpreter.JsInterpreterHelper;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.management.ManagementServerHost;
import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao;
import org.apache.cloudstack.secstorage.HeuristicVO;
import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao;
import org.apache.cloudstack.secstorage.heuristics.Heuristic;
import org.apache.cloudstack.secstorage.heuristics.HeuristicType;
import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplianceCommand;
import org.apache.cloudstack.storage.command.DettachCommand;
import org.apache.cloudstack.storage.command.SyncVolumePathAnswer;
import org.apache.cloudstack.storage.command.SyncVolumePathCommand;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreObjectDownloadDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreObjectDownloadVO;
import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao;
import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao;
import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO;
import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
import org.apache.cloudstack.storage.object.ObjectStore;
import org.apache.cloudstack.storage.object.ObjectStoreEntity;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.EnumUtils;
import org.springframework.stereotype.Component;

import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.Command;
import com.cloud.agent.api.DeleteStoragePoolCommand;
import com.cloud.agent.api.GetStoragePoolCapabilitiesAnswer;
import com.cloud.agent.api.GetStoragePoolCapabilitiesCommand;
import com.cloud.agent.api.GetStorageStatsAnswer;
import com.cloud.agent.api.GetStorageStatsCommand;
import com.cloud.agent.api.GetVolumeStatsAnswer;
import com.cloud.agent.api.GetVolumeStatsCommand;
import com.cloud.agent.api.ModifyStoragePoolAnswer;
import com.cloud.agent.api.ModifyStoragePoolCommand;
import com.cloud.agent.api.StoragePoolInfo;
import com.cloud.agent.api.VolumeStatsEntry;
import com.cloud.agent.api.to.DataTO;
import com.cloud.agent.api.to.DiskTO;
import com.cloud.agent.api.to.StorageFilerTO;
import com.cloud.agent.manager.Commands;
import com.cloud.api.ApiDBUtils;
import com.cloud.api.query.dao.TemplateJoinDao;
import com.cloud.api.query.vo.TemplateJoinVO;
import com.cloud.capacity.Capacity;
import com.cloud.capacity.CapacityManager;
import com.cloud.capacity.CapacityState;
import com.cloud.capacity.CapacityVO;
import com.cloud.capacity.dao.CapacityDao;
import com.cloud.cluster.ClusterManagerListener;
import com.cloud.configuration.Config;
import com.cloud.configuration.ConfigurationManager;
import com.cloud.configuration.Resource.ResourceType;
import com.cloud.cpu.CPU;
import com.cloud.dc.ClusterVO;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.HostPodVO;
import com.cloud.dc.VsphereStoragePolicyVO;
import com.cloud.dc.dao.ClusterDao;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.dc.dao.HostPodDao;
import com.cloud.dc.dao.VsphereStoragePolicyDao;
import com.cloud.event.ActionEvent;
import com.cloud.event.EventTypes;
import com.cloud.exception.AgentUnavailableException;
import com.cloud.exception.ConnectionException;
import com.cloud.exception.DiscoveryException;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.exception.OperationTimedoutException;
import com.cloud.exception.PermissionDeniedException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.exception.StorageConflictException;
import com.cloud.exception.StorageUnavailableException;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.Status;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.hypervisor.HypervisorGuruManager;
import com.cloud.network.router.VirtualNetworkApplianceManager;
import com.cloud.offering.DiskOffering;
import com.cloud.offering.ServiceOffering;
import com.cloud.org.Grouping;
import com.cloud.org.Grouping.AllocationState;
import com.cloud.resource.ResourceManager;
import com.cloud.resource.ResourceState;
import com.cloud.server.ManagementServer;
import com.cloud.server.ManagementService;
import com.cloud.server.StatsCollector;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Storage.StoragePoolType;
import com.cloud.storage.Volume.Type;
import com.cloud.storage.dao.BucketDao;
import com.cloud.storage.dao.DiskOfferingDao;
import com.cloud.storage.dao.SnapshotDao;
import com.cloud.storage.dao.StoragePoolAndAccessGroupMapDao;
import com.cloud.storage.dao.StoragePoolHostDao;
import com.cloud.storage.dao.StoragePoolTagsDao;
import com.cloud.storage.dao.StoragePoolWorkDao;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplatePoolDao;
import com.cloud.storage.dao.VMTemplateZoneDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.storage.listener.StoragePoolMonitor;
import com.cloud.storage.listener.VolumeStateListener;
import com.cloud.template.TemplateManager;
import com.cloud.template.VirtualMachineTemplate;
import com.cloud.upgrade.SystemVmTemplateRegistration;
import com.cloud.user.Account;
import com.cloud.user.AccountManager;
import com.cloud.user.ResourceLimitService;
import com.cloud.user.dao.UserDao;
import com.cloud.utils.DateUtil;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.Pair;
import com.cloud.utils.StringUtils;
import com.cloud.utils.UriUtils;
import com.cloud.utils.component.ComponentContext;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.EntityManager;
import com.cloud.utils.db.Filter;
import com.cloud.utils.db.GenericSearchBuilder;
import com.cloud.utils.db.GlobalLock;
import com.cloud.utils.db.JoinBuilder;
import com.cloud.utils.db.JoinBuilder.JoinType;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.db.SearchCriteria.Op;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallbackNoReturn;
import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn;
import com.cloud.utils.db.TransactionLegacy;
import com.cloud.utils.db.TransactionStatus;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.DiskProfile;
import com.cloud.vm.UserVmManager;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine.State;
import com.cloud.vm.dao.VMInstanceDao;
import com.google.common.collect.Sets;


@Component
public class StorageManagerImpl extends ManagerBase implements StorageManager, ClusterManagerListener, Configurable {

    protected String _name;
    @Inject
    protected AgentManager _agentMgr;
    @Inject
    protected TemplateManager _tmpltMgr;
    @Inject
    protected AccountManager _accountMgr;
    @Inject
    protected ConfigurationManager _configMgr;
    @Inject
    private VolumeDataStoreDao _volumeDataStoreDao;
    @Inject
    protected HostDao _hostDao;
    @Inject
    protected SnapshotDao _snapshotDao;
    @Inject
    protected StoragePoolHostDao _storagePoolHostDao;
    @Inject
    protected VMTemplatePoolDao _vmTemplatePoolDao = null;
    @Inject
    protected VMTemplateZoneDao _vmTemplateZoneDao;
    @Inject
    protected VMTemplateDao _vmTemplateDao = null;
    @Inject
    protected VMInstanceDao _vmInstanceDao;
    @Inject
    protected PrimaryDataStoreDao _storagePoolDao = null;
    @Inject
    protected StoragePoolDetailsDao _storagePoolDetailsDao;
    @Inject
    protected ImageStoreDao _imageStoreDao = null;
    @Inject
    protected ImageStoreDetailsDao _imageStoreDetailsDao = null;
    @Inject
    protected ImageStoreObjectDownloadDao _imageStoreObjectDownloadDao = null;
    @Inject
    protected SnapshotDataStoreDao _snapshotStoreDao = null;
    @Inject
    protected TemplateDataStoreDao _templateStoreDao = null;
    @Inject
    protected TemplateJoinDao _templateViewDao = null;
    @Inject
    protected VolumeDataStoreDao _volumeStoreDao = null;
    @Inject
    protected CapacityDao _capacityDao;
    @Inject
    protected CapacityManager _capacityMgr;
    @Inject
    protected DataCenterDao _dcDao = null;
    @Inject
    protected VMTemplateDao _templateDao;
    @Inject
    protected UserDao _userDao;
    @Inject
    protected ClusterDao _clusterDao;
    @Inject
    protected StoragePoolWorkDao _storagePoolWorkDao;
    @Inject
    protected HypervisorGuruManager _hvGuruMgr;
    @Inject
    protected VolumeDao volumeDao;
    @Inject
    ConfigurationDao _configDao;
    @Inject
    ManagementServer _msServer;
    @Inject
    VolumeService volService;
    @Inject
    VolumeDataFactory volFactory;
    @Inject
    TemplateDataFactory tmplFactory;
    @Inject
    SnapshotDataFactory snapshotFactory;
    @Inject
    DataStoreManager _dataStoreMgr;
    @Inject
    DataStoreProviderManager _dataStoreProviderMgr;
    @Inject
    private TemplateService _imageSrv;
    @Inject
    EndPointSelector _epSelector;
    @Inject
    private DiskOfferingDao _diskOfferingDao;
    @Inject
    ResourceLimitService _resourceLimitMgr;
    @Inject
    EntityManager _entityMgr;
    @Inject
    SnapshotService _snapshotService;
    @Inject
    StoragePoolTagsDao _storagePoolTagsDao;
    @Inject
    StoragePoolAndAccessGroupMapDao _storagePoolAccessGroupMapDao;
    @Inject
    PrimaryDataStoreDao primaryStoreDao;
    @Inject
    DiskOfferingDetailsDao _diskOfferingDetailsDao;
    @Inject
    VsphereStoragePolicyDao _vsphereStoragePolicyDao;
    @Inject
    private AnnotationDao annotationDao;

    @Inject
    private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao;

    @Inject
    protected UserVmManager userVmManager;
    @Inject
    protected ObjectStoreDao _objectStoreDao;

    @Inject
    protected ObjectStoreDetailsDao _objectStoreDetailsDao;

    @Inject
    protected BucketDao _bucketDao;
    @Inject
    ConfigDepot configDepot;
    @Inject
    ConfigurationDao configurationDao;
    @Inject
    private ImageStoreDetailsUtil imageStoreDetailsUtil;
    @Inject
    protected HostPodDao _podDao;
    @Inject
    ResourceManager _resourceMgr;
    @Inject
    ManagementService managementService;
    @Inject
    JsInterpreterHelper jsInterpreterHelper;

    protected GenericSearchBuilder<StoragePoolHostVO, Long> UpHostsInPoolSearch;
    protected SearchBuilder<VMInstanceVO> StoragePoolSearch;
    protected SearchBuilder<StoragePoolVO> LocalStorageSearch;
    protected SearchBuilder<CapacityVO> PoolCapacitySearch;
    protected SearchBuilder<StoragePoolVO> PoolUsedStatsSearch;

    ScheduledExecutorService _executor = null;
    int _storagePoolAcquisitionWaitSeconds = 1800; // 30 minutes
    int _downloadUrlCleanupInterval;
    int _downloadUrlExpirationInterval;
    private long _serverId;

    private final Map<String, HypervisorHostListener> hostListeners = new HashMap<>();

    private final Set<HypervisorType> zoneWidePoolSupportedHypervisorTypes = Sets.newHashSet(HypervisorType.KVM, HypervisorType.VMware,
            HypervisorType.Hyperv, HypervisorType.LXC, HypervisorType.Any, HypervisorType.Simulator);

    private static final String NFS_MOUNT_OPTIONS_INCORRECT = "An incorrect mount option was specified";

    protected void enableDefaultDatastoreDownloadRedirectionForExistingInstallations() {
        if (!configDepot.isNewConfig(DataStoreDownloadFollowRedirects)) {
            logger.trace("{} is not a new configuration, skipping updating its value",
                    DataStoreDownloadFollowRedirects.key());
            return;
        }
        List<DataCenterVO> zones =
                _dcDao.listAll(new Filter(1));
        if (CollectionUtils.isNotEmpty(zones)) {
            logger.debug("Updating value for configuration: {} to true",
                DataStoreDownloadFollowRedirects.key());
            configurationDao.update(DataStoreDownloadFollowRedirects.key(), "true");
        }
    }

    @Override
    public boolean isLocalStorageActiveOnHost(Long hostId) {
        List<StoragePoolHostVO> storagePoolHostRefs = _storagePoolHostDao.listByHostId(hostId);
        for (StoragePoolHostVO storagePoolHostRef : storagePoolHostRefs) {
            StoragePoolVO primaryDataStoreVO = _storagePoolDao.findById(storagePoolHostRef.getPoolId());
            if (primaryDataStoreVO != null && (primaryDataStoreVO.getPoolType() == StoragePoolType.LVM || primaryDataStoreVO.getPoolType() == StoragePoolType.EXT)) {
                SearchBuilder<VolumeVO> volumeSB = volumeDao.createSearchBuilder();
                volumeSB.and("poolId", volumeSB.entity().getPoolId(), SearchCriteria.Op.EQ);
                volumeSB.and("removed", volumeSB.entity().getRemoved(), SearchCriteria.Op.NULL);
                volumeSB.and("state", volumeSB.entity().getState(), SearchCriteria.Op.NIN);

                SearchBuilder<VMInstanceVO> activeVmSB = _vmInstanceDao.createSearchBuilder();
                activeVmSB.and("state", activeVmSB.entity().getState(), SearchCriteria.Op.IN);
                volumeSB.join("activeVmSB", activeVmSB, volumeSB.entity().getInstanceId(), activeVmSB.entity().getId(), JoinBuilder.JoinType.INNER);

                SearchCriteria<VolumeVO> volumeSC = volumeSB.create();
                volumeSC.setParameters("poolId", primaryDataStoreVO.getId());
                volumeSC.setParameters("state", Volume.State.Expunging, Volume.State.Destroy);
                volumeSC.setJoinParameters("activeVmSB", "state", State.Starting, State.Running, State.Stopping, State.Migrating);

                List<VolumeVO> volumes = volumeDao.search(volumeSC, null);
                if (!volumes.isEmpty()) {
                    return true;
                }
            }
        }

        return false;
    }

    @Override
    public Answer[] sendToPool(StoragePool pool, Commands cmds) throws StorageUnavailableException {
        return sendToPool(pool, null, null, cmds).second();
    }

    @Override
    public Answer sendToPool(StoragePool pool, long[] hostIdsToTryFirst, Command cmd) throws StorageUnavailableException {
        Answer[] answers = sendToPool(pool, hostIdsToTryFirst, null, new Commands(cmd)).second();
        if (answers == null) {
            return null;
        }
        return answers[0];
    }

    @Override
    public Answer sendToPool(StoragePool pool, Command cmd) throws StorageUnavailableException {
        if (cmd instanceof GetStorageStatsCommand && canPoolProvideStorageStats(pool)) {
            // Get stats from the pool directly instead of sending cmd to host
            return getStoragePoolStats(pool, (GetStorageStatsCommand) cmd);
        }

        Answer[] answers = sendToPool(pool, new Commands(cmd));
        if (answers == null) {
            return null;
        }
        return answers[0];
    }

    protected Pair<Long, Long> getStoragePoolIopsStats(PrimaryDataStoreDriver primaryStoreDriver, StoragePool pool) {
        Pair<Long, Long> result = primaryStoreDriver.getStorageIopsStats(pool);
        if (result != null) {
            return result;
        }
        Long usedIops = primaryStoreDriver.getUsedIops(pool);
        if (usedIops <= 0) {
            usedIops = null;
        }
        return new Pair<>(pool.getCapacityIops(), usedIops);
    }

    private GetStorageStatsAnswer getStoragePoolStats(StoragePool pool, GetStorageStatsCommand cmd) {
        DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(pool.getStorageProviderName());
        DataStoreDriver storeDriver = storeProvider.getDataStoreDriver();
        PrimaryDataStoreDriver primaryStoreDriver = (PrimaryDataStoreDriver) storeDriver;
        Pair<Long, Long> storageStats = primaryStoreDriver.getStorageStats(pool);
        if (storageStats == null) {
            return new GetStorageStatsAnswer(cmd, "Failed to get storage stats for pool: " + pool.getId());
        }
        Pair<Long, Long> iopsStats = getStoragePoolIopsStats(primaryStoreDriver, pool);
        return new GetStorageStatsAnswer(cmd, storageStats.first(), storageStats.second(),
                iopsStats.first(), iopsStats.second());
    }

    @Override
    public boolean canPoolProvideStorageStats(StoragePool pool) {
        DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(pool.getStorageProviderName());
        DataStoreDriver storeDriver = storeProvider.getDataStoreDriver();
        return storeDriver instanceof PrimaryDataStoreDriver && ((PrimaryDataStoreDriver)storeDriver).canProvideStorageStats();
    }

    @Override
    public Answer getVolumeStats(StoragePool pool, Command cmd) {
        DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(pool.getStorageProviderName());
        DataStoreDriver storeDriver = storeProvider.getDataStoreDriver();
        PrimaryDataStoreDriver primaryStoreDriver = (PrimaryDataStoreDriver) storeDriver;
        HashMap<String, VolumeStatsEntry> statEntry = new HashMap<>();
        GetVolumeStatsCommand getVolumeStatsCommand = (GetVolumeStatsCommand) cmd;
        for (String volumeUuid : getVolumeStatsCommand.getVolumeUuids()) {
            Pair<Long, Long> volumeStats = primaryStoreDriver.getVolumeStats(pool, volumeUuid);
            if (volumeStats == null) {
                return new GetVolumeStatsAnswer(getVolumeStatsCommand, "Failed to get stats for volume: " + volumeUuid,
                        null);
            } else {
                VolumeStatsEntry volumeStatsEntry = new VolumeStatsEntry(volumeUuid, volumeStats.first(),
                        volumeStats.second());
                statEntry.put(volumeUuid, volumeStatsEntry);
            }
        }
        return new GetVolumeStatsAnswer(getVolumeStatsCommand, "", statEntry);
    }

    @Override
    public boolean configure(String name, Map<String, Object> params) {
        Map<String, String> configs = _configDao.getConfiguration("management-server", params);

        _storagePoolAcquisitionWaitSeconds = NumbersUtil.parseInt(configs.get("pool.acquisition.wait.seconds"), 1800);
        logger.info("pool.acquisition.wait.seconds is configured as {} seconds", _storagePoolAcquisitionWaitSeconds);

        _agentMgr.registerForHostEvents(new StoragePoolMonitor(this, _storagePoolDao, _storagePoolHostDao, _dataStoreProviderMgr), true, false, true);

        logger.info("Storage cleanup enabled: {}, interval: {}, delay: {}, template cleanup enabled: {}",
                StorageCleanupEnabled.value(),
                StorageCleanupInterval.value(),
                StorageCleanupDelay.value(),
                TemplateCleanupEnabled.value());

        String cleanupInterval = configs.get("extract.url.cleanup.interval");
        _downloadUrlCleanupInterval = NumbersUtil.parseInt(cleanupInterval, 7200);

        String urlExpirationInterval = configs.get("extract.url.expiration.interval");
        _downloadUrlExpirationInterval = NumbersUtil.parseInt(urlExpirationInterval, 14400);

        String workers = configs.get("expunge.workers");
        int wrks = NumbersUtil.parseInt(workers, 10);
        _executor = Executors.newScheduledThreadPool(wrks, new NamedThreadFactory("StorageManager-Scavenger"));

        _agentMgr.registerForHostEvents(ComponentContext.inject(LocalStoragePoolListener.class), true, false, false);

        _serverId = _msServer.getId();

        UpHostsInPoolSearch = _storagePoolHostDao.createSearchBuilder(Long.class);
        UpHostsInPoolSearch.selectFields(UpHostsInPoolSearch.entity().getHostId());
        SearchBuilder<HostVO> hostSearch = _hostDao.createSearchBuilder();
        hostSearch.and("status", hostSearch.entity().getStatus(), Op.EQ);
        hostSearch.and("resourceState", hostSearch.entity().getResourceState(), Op.EQ);
        UpHostsInPoolSearch.join("hosts", hostSearch, hostSearch.entity().getId(), UpHostsInPoolSearch.entity().getHostId(), JoinType.INNER);
        UpHostsInPoolSearch.and("pool", UpHostsInPoolSearch.entity().getPoolId(), Op.EQ);
        UpHostsInPoolSearch.done();

        StoragePoolSearch = _vmInstanceDao.createSearchBuilder();

        SearchBuilder<VolumeVO> volumeSearch = volumeDao.createSearchBuilder();
        volumeSearch.and("volumeType", volumeSearch.entity().getVolumeType(), SearchCriteria.Op.EQ);
        volumeSearch.and("poolId", volumeSearch.entity().getPoolId(), SearchCriteria.Op.EQ);
        volumeSearch.and("state", volumeSearch.entity().getState(), SearchCriteria.Op.EQ);
        StoragePoolSearch.join("vmVolume", volumeSearch, volumeSearch.entity().getInstanceId(), StoragePoolSearch.entity().getId(), JoinBuilder.JoinType.INNER);
        StoragePoolSearch.done();

        LocalStorageSearch = _storagePoolDao.createSearchBuilder();
        SearchBuilder<StoragePoolHostVO> storageHostSearch = _storagePoolHostDao.createSearchBuilder();
        storageHostSearch.and("hostId", storageHostSearch.entity().getHostId(), SearchCriteria.Op.EQ);
        LocalStorageSearch.join("poolHost", storageHostSearch, storageHostSearch.entity().getPoolId(), LocalStorageSearch.entity().getId(), JoinBuilder.JoinType.INNER);
        LocalStorageSearch.and("type", LocalStorageSearch.entity().getPoolType(), SearchCriteria.Op.IN);
        LocalStorageSearch.done();

        PoolCapacitySearch = _capacityDao.createSearchBuilder();
        PoolCapacitySearch.and("hostOrPoolId", PoolCapacitySearch.entity().getHostOrPoolId(), SearchCriteria.Op.EQ);
        PoolCapacitySearch.and("dataCenterId", PoolCapacitySearch.entity().getDataCenterId(), SearchCriteria.Op.EQ);
        PoolCapacitySearch.and("capacityType", PoolCapacitySearch.entity().getCapacityType(), SearchCriteria.Op.EQ);
        PoolCapacitySearch.done();

        PoolUsedStatsSearch = _storagePoolDao.createSearchBuilder();
        PoolUsedStatsSearch.and("dataCenterId", PoolUsedStatsSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ);
        PoolUsedStatsSearch.and("podId", PoolUsedStatsSearch.entity().getPodId(), SearchCriteria.Op.EQ);
        PoolUsedStatsSearch.and("clusterId", PoolUsedStatsSearch.entity().getClusterId(), SearchCriteria.Op.EQ);
        PoolUsedStatsSearch.and("ids", PoolUsedStatsSearch.entity().getId(), SearchCriteria.Op.IN);
        PoolUsedStatsSearch.and("parent", PoolUsedStatsSearch.entity().getParent(), SearchCriteria.Op.EQ);
        PoolUsedStatsSearch.done();

        Volume.State.getStateMachine().registerListener(new VolumeStateListener(_configDao, _vmInstanceDao));

        return true;
    }

    @Override
    public String getStoragePoolTags(long poolId) {
        return com.cloud.utils.StringUtils.listToCsvTags(getStoragePoolTagList(poolId));
    }

    @Override
    public List<String> getStoragePoolTagList(long poolId) {
        return _storagePoolDao.searchForStoragePoolTags(poolId);
    }

    @Override
    public boolean start() {
        if (StorageCleanupEnabled.value()) {
            Random generator = new Random();
            int initialDelay = generator.nextInt(StorageCleanupInterval.value());
            _executor.scheduleWithFixedDelay(new StorageGarbageCollector(), initialDelay, StorageCleanupInterval.value(), TimeUnit.SECONDS);
        } else {
            logger.debug("Storage cleanup is not enabled, so the storage cleanup thread is not being scheduled.");
        }

        _executor.scheduleWithFixedDelay(new DownloadURLGarbageCollector(), _downloadUrlCleanupInterval, _downloadUrlCleanupInterval, TimeUnit.SECONDS);
        enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
        return true;
    }

    @Override
    public boolean stop() {
        if (StorageCleanupEnabled.value()) {
            _executor.shutdown();
        }
        return true;
    }

    protected String getValidatedPareForLocalStorage(Object obj, String paramName) {
        String result = obj == null ? null : obj.toString();
        if (StringUtils.isEmpty(result)) {
            throw new InvalidParameterValueException(String.format("Invalid %s provided", paramName));
        }
        return result;
    }

    protected DataStore createLocalStorage(Map<String, Object> poolInfos) throws ConnectionException{
        Object existingUuid = poolInfos.get("uuid");
        if( existingUuid == null ){
            poolInfos.put("uuid", UUID.randomUUID().toString());
        }
        String hostAddress = getValidatedPareForLocalStorage(poolInfos.get("host"), "host");
        String hostPath = getValidatedPareForLocalStorage(poolInfos.get("hostPath"), "path");
        Host host = _hostDao.findByName(hostAddress);

        if( host == null ) {
            host = _hostDao.findByIp(hostAddress);

            if( host == null ) {
                host = _hostDao.findByPublicIp(hostAddress);

                if( host == null ) {
                    throw new InvalidParameterValueException(String.format("host %s not found",hostAddress));
                }
             }
         }

        long capacityBytes = poolInfos.get("capacityBytes") != null ? Long.parseLong(poolInfos.get("capacityBytes").toString()) : 0;

        @SuppressWarnings("unchecked") // the local storage configuration is parsed into untyped maps
        Map<String, String> poolDetails = (Map<String, String>) poolInfos.get("details");
        StoragePoolInfo pInfo = new StoragePoolInfo(poolInfos.get("uuid").toString(),
                                                    host.getPrivateIpAddress(),
                                                    hostPath,
                                                    hostPath,
                                                    StoragePoolType.Filesystem,
                                                    capacityBytes,
                                                    0,
                                                    poolDetails,
                                                    poolInfos.get("name").toString());

        return createLocalStorage(host, pInfo);
    }

    private boolean isLocalStorageEnabledForZone(DataCenterVO zone) {
        return zone.isLocalStorageEnabled() || BooleanUtils.toBoolean(SystemVMUseLocalStorage.valueIn(zone.getId()));
    }

    @DB
    @Override
    public DataStore createLocalStorage(Host host, StoragePoolInfo pInfo) throws ConnectionException {
        DataCenterVO dc = _dcDao.findById(host.getDataCenterId());
        if (dc == null) {
            return null;
        }
        if (!isLocalStorageEnabledForZone(dc)) {
            return null;
        }
        DataStore store = null;
        DataStoreProvider provider = _dataStoreProviderMgr.getDefaultPrimaryDataStoreProvider();
        DataStoreLifeCycle lifeCycle = provider.getDataStoreLifeCycle();
        try {
            String hostAddress = pInfo.getHost();
            if (host.getHypervisorType() == Hypervisor.HypervisorType.VMware) {
                hostAddress = "VMFS datastore: " + pInfo.getHostPath();
            }
            StoragePoolVO pool = _storagePoolDao.findPoolByHostPath(host.getDataCenterId(), host.getPodId(), hostAddress, pInfo.getHostPath(), pInfo.getUuid());
            if (pool == null && host.getHypervisorType() == HypervisorType.VMware) {
                // perform run-time upgrade. In versions prior to 2.2.12, there
                // is a bug that we don't save local datastore info (host path
                // is empty), this will cause us
                // not able to distinguish multiple local datastores that may be
                // available on the host, to support smooth migration, we
                // need to perform runtime upgrade here
                if (!pInfo.getHostPath().isEmpty()) {
                    pool = _storagePoolDao.findPoolByHostPath(host.getDataCenterId(), host.getPodId(), hostAddress, "", pInfo.getUuid());
                }
            }
            if (pool == null) {
                //the path can be different, but if they have the same uuid, assume they are the same storage
                pool = _storagePoolDao.findPoolByHostPath(host.getDataCenterId(), host.getPodId(), hostAddress, null, pInfo.getUuid());
                if (pool != null) {
                    logger.debug("Found a storage pool: {}, but with different hostpath {}, still treat it as the same pool",
                            pInfo.getUuid(),
                            pInfo.getHostPath());
                }
            }

            if (pool == null) {
                Map<String, Object> params = new HashMap<>();
                String name = pInfo.getName() != null ? pInfo.getName() : createLocalStoragePoolName(host, pInfo);
                params.put("zoneId", host.getDataCenterId());
                params.put("clusterId", host.getClusterId());
                params.put("podId", host.getPodId());
                params.put("hypervisorType", host.getHypervisorType());
                params.put("name", name);
                params.put("localStorage", true);
                params.put("details", pInfo.getDetails());
                params.put("uuid", pInfo.getUuid());
                params.put("providerName", provider.getName());
                params.put("scheme", pInfo.getPoolType().toString());
                params.put("host", pInfo.getHost());
                params.put("hostPath", pInfo.getHostPath());

                store = lifeCycle.initialize(params);
            } else {
                store = _dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary);
            }

            pool = _storagePoolDao.findById(store.getId());
            if (pool.getStatus() != StoragePoolStatus.Maintenance && pool.getStatus() != StoragePoolStatus.Removed) {
                HostScope scope = new HostScope(host.getId(), host.getClusterId(), host.getDataCenterId());
                lifeCycle.attachHost(store, scope, pInfo);
            }

        } catch (Exception e) {
            logger.warn("Unable to setup the local storage pool for {}", host, e);
            try {
                if (store != null) {
                    logger.debug("Trying to delete storage pool entry if exists {}", store);
                    lifeCycle.deleteDataStore(store);
                }
            } catch (Exception ex) {
                logger.debug("Failed to clean up local storage pool: {}", ex.getMessage());
            }
            throw new ConnectionException(true, "Unable to setup the local storage pool for " + host, e);
        }

        return _dataStoreMgr.getDataStore(store.getId(), DataStoreRole.Primary);
    }

    /**
     * Creates the local storage pool name.
     * The name will follow the pattern: <hostname>-local-<firstBlockOfUuid>
     */
    protected String createLocalStoragePoolName(Host host, StoragePoolInfo storagePoolInformation) {
        return String.format("%s-%s-%s", StringUtils.trim(host.getName()), "local", storagePoolInformation.getUuid().split("-")[0]);
    }

    protected void checkNfsMountOptions(String nfsMountOpts) throws InvalidParameterValueException {
        String[] options = nfsMountOpts.replaceAll("\\s", "").split(",");
        Map<String, String> optionsMap = new HashMap<>();
        for (String option : options) {
            String[] keyValue = option.split("=");
            if (keyValue.length > 2) {
                throw new InvalidParameterValueException("Invalid value for NFS option " + keyValue[0]);
            }
            if (optionsMap.containsKey(keyValue[0])) {
                throw new InvalidParameterValueException("Duplicate NFS option values found for option " + keyValue[0]);
            }
            optionsMap.put(keyValue[0], null);
        }
    }

    protected void checkNFSMountOptionsForCreate(Map<String, String> details, HypervisorType hypervisorType, String scheme) throws InvalidParameterValueException {
        if (!details.containsKey(ApiConstants.NFS_MOUNT_OPTIONS)) {
            return;
        }
        if (!hypervisorType.equals(HypervisorType.KVM) && !hypervisorType.equals(HypervisorType.Simulator)) {
            throw new InvalidParameterValueException("NFS options can not be set for the hypervisor type " + hypervisorType);
        }
        if (!"nfs".equals(scheme)) {
            throw new InvalidParameterValueException("NFS options can only be set on pool type " + StoragePoolType.NetworkFilesystem);
        }
        checkNfsMountOptions(details.get(ApiConstants.NFS_MOUNT_OPTIONS));
    }

    protected void checkNFSMountOptionsForUpdate(Map<String, String> details, StoragePoolVO pool, Long accountId) throws InvalidParameterValueException {
        if (!details.containsKey(ApiConstants.NFS_MOUNT_OPTIONS)) {
            return;
        }
        if (!_accountMgr.isRootAdmin(accountId)) {
            throw new PermissionDeniedException("Only root admin can modify nfs options");
        }
        if (!pool.getHypervisor().equals(HypervisorType.KVM) && !pool.getHypervisor().equals((HypervisorType.Simulator))) {
            throw new InvalidParameterValueException("NFS options can only be set for the hypervisor type " + HypervisorType.KVM);
        }
        if (!pool.getPoolType().equals(StoragePoolType.NetworkFilesystem)) {
            throw new InvalidParameterValueException("NFS options can only be set on pool type " + StoragePoolType.NetworkFilesystem);
        }
        if (!pool.isInMaintenance()) {
            throw new InvalidParameterValueException("The storage pool should be in maintenance mode to edit nfs options");
        }
        checkNfsMountOptions(details.get(ApiConstants.NFS_MOUNT_OPTIONS));
    }

    @Override
    public PrimaryDataStoreInfo createPool(CreateStoragePoolCmd cmd) throws  IllegalArgumentException {
        jsInterpreterHelper.ensureInterpreterEnabledIfParameterProvided(ApiConstants.IS_TAG_A_RULE, Boolean.TRUE.equals(cmd.isTagARule()));

        String providerName = cmd.getStorageProviderName();
        Map<String,String> uriParams = extractUriParamsAsMap(cmd.getUrl());
        boolean isFileScheme = "file".equals(uriParams.get("scheme"));
        DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(providerName);

        if (storeProvider == null) {
            storeProvider = _dataStoreProviderMgr.getDefaultPrimaryDataStoreProvider();
            if (storeProvider == null) {
                throw new InvalidParameterValueException("can't find storage provider: " + providerName);
            }
        }

        Long clusterId = cmd.getClusterId();
        Long podId = cmd.getPodId();
        Long zoneId = cmd.getZoneId();

        ScopeType scopeType = ScopeType.CLUSTER;
        if (isFileScheme) {
            scopeType = ScopeType.HOST;
        }
        String scope = cmd.getScope();
        if (scope != null) {
            try {
                scopeType = Enum.valueOf(ScopeType.class, scope.toUpperCase());
            } catch (Exception e) {
                throw new InvalidParameterValueException("invalid scope for pool " + scope);
            }
        }

        if (scopeType == ScopeType.CLUSTER && clusterId == null) {
            throw new InvalidParameterValueException("cluster id can't be null, if scope is cluster");
        } else if (scopeType == ScopeType.ZONE && zoneId == null) {
            throw new InvalidParameterValueException("zone id can't be null, if scope is zone");
        }

        HypervisorType hypervisorType;
        if (scopeType == ScopeType.ZONE) {
            // ignore passed clusterId and podId
            clusterId = null;
            podId = null;
            String hypervisor = cmd.getHypervisor();
            if (hypervisor != null) {
                try {
                    hypervisorType = HypervisorType.getType(hypervisor);
                } catch (Exception e) {
                    throw new InvalidParameterValueException("invalid hypervisor type " + hypervisor);
                }
            } else {
                throw new InvalidParameterValueException("Missing parameter hypervisor. Hypervisor type is required to create zone wide primary storage.");
            }

            if (!zoneWidePoolSupportedHypervisorTypes.contains(hypervisorType)) {
                throw new InvalidParameterValueException("Zone wide storage pool is not supported for hypervisor type " + hypervisor);
            }
        } else {
            ClusterVO clusterVO = _clusterDao.findById(clusterId);
            hypervisorType = clusterVO.getHypervisorType();
        }

        Map<String, String> details = extractApiParamAsMap(cmd.getDetails());
        checkNFSMountOptionsForCreate(details, hypervisorType, uriParams.get("scheme"));

        DataCenterVO zone = _dcDao.findById(cmd.getZoneId());
        if (zone == null) {
            throw new InvalidParameterValueException("unable to find zone by id " + zoneId);
        }
        // Check if zone is disabled
        Account account = CallContext.current().getCallingAccount();
        if (Grouping.AllocationState.Disabled == zone.getAllocationState() && !_accountMgr.isRootAdmin(account.getId())) {
            throw new PermissionDeniedException(String.format("Cannot perform this operation, Zone is currently disabled: %s", zone));
        }
        // Check if it's local storage and if it's enabled on the zone
        if (isFileScheme && !isLocalStorageEnabledForZone(zone)) {
            throw new InvalidParameterValueException("Local storage is not enabled for zone: " + zone);
        }

        Map<String, Object> params = new HashMap<>();
        params.put("zoneId", zone.getId());
        params.put("clusterId", clusterId);
        params.put("podId", podId);
        params.put("hypervisorType", hypervisorType);
        params.put("url", cmd.getUrl());
        params.put("tags", cmd.getTags());
        params.put(ApiConstants.STORAGE_ACCESS_GROUPS, cmd.getStorageAccessGroups());
        params.put("isTagARule", cmd.isTagARule());
        params.put("name", cmd.getStoragePoolName());
        params.put("details", details);
        params.put("providerName", storeProvider.getName());
        params.put("managed", cmd.isManaged());
        params.put("capacityBytes", cmd.getCapacityBytes());
        params.put("capacityIops", cmd.getCapacityIops());
        params.put("scheme", uriParams.get("scheme"));
        if (MapUtils.isNotEmpty(uriParams)) {
            params.putAll(uriParams);
        }

        DataStoreLifeCycle lifeCycle = storeProvider.getDataStoreLifeCycle();
        DataStore store = null;
        try {
            if (isFileScheme) {
                store = createLocalStorage(params);
            } else {
                store = lifeCycle.initialize(params);
            }
            if (scopeType == ScopeType.CLUSTER) {
                ClusterScope clusterScope = new ClusterScope(clusterId, podId, zoneId);
                lifeCycle.attachCluster(store, clusterScope);
            } else if (scopeType == ScopeType.ZONE) {
                ZoneScope zoneScope = new ZoneScope(zoneId);
                lifeCycle.attachZone(store, zoneScope, hypervisorType);
            }
        } catch (Exception e) {
            logger.debug("Failed to add data store: {}", e.getMessage(), e);
            try {
                // clean up the db, just absorb the exception thrown in deletion with error logged, so that user can get error for adding data store
                // not deleting data store.
                if (store != null) {
                    lifeCycle.deleteDataStore(store);
                }
            } catch (Exception ex) {
                logger.debug("Failed to clean up storage pool: {}", ex.getMessage());
            }
            throw new CloudRuntimeException("Failed to add data store: " + e.getMessage(), e);
        }

        return (PrimaryDataStoreInfo)_dataStoreMgr.getDataStore(store.getId(), DataStoreRole.Primary);
    }

    protected Map<String,String> extractUriParamsAsMap(String url) {
        Map<String,String> uriParams = new HashMap<>();
        UriUtils.UriInfo uriInfo;
        try {
            uriInfo = UriUtils.getUriInfo(url);
        } catch (CloudRuntimeException cre) {
            logger.debug("URI validation for url: {} failed, returning empty uri params", url);
            return uriParams;
        }

        String scheme = uriInfo.getScheme();
        String storageHost = uriInfo.getStorageHost();
        String storagePath = uriInfo.getStoragePath();
        if (scheme == null) {
            logger.debug("Scheme for url: {} is not found, returning empty uri params", url);
            return uriParams;
        }
        boolean isHostOrPathBlank = StringUtils.isAnyBlank(storagePath, storageHost);
        if (scheme.equalsIgnoreCase("nfs")) {
            if (isHostOrPathBlank) {
                throw new InvalidParameterValueException("host or path is null, should be nfs://hostname/path");
            }
        } else if (scheme.equalsIgnoreCase("cifs")) {
            // Don't validate against a URI encoded URI.
            try {
                URI cifsUri = new URI(url);
                String warnMsg = UriUtils.getCifsUriParametersProblems(cifsUri);
                if (warnMsg != null) {
                    throw new InvalidParameterValueException(warnMsg);
                }
            } catch (URISyntaxException e) {
                throw new InvalidParameterValueException(url + " is not a valid uri");
            }
        } else if (scheme.equalsIgnoreCase("sharedMountPoint")) {
            if (storagePath == null) {
                throw new InvalidParameterValueException("host or path is null, should be sharedmountpoint://localhost/path");
            }
        } else if (scheme.equalsIgnoreCase("rbd")) {
            if (storagePath == null) {
                throw new InvalidParameterValueException("host or path is null, should be rbd://hostname/pool");
            }
        } else if (scheme.equalsIgnoreCase("gluster")) {
            if (isHostOrPathBlank) {
                throw new InvalidParameterValueException("host or path is null, should be gluster://hostname/volume");
            }
        } else if (scheme.equalsIgnoreCase("clvm") || scheme.equalsIgnoreCase("clvm_ng")) {
            if (storagePath == null) {
                throw new InvalidParameterValueException("path is null, should be " + scheme.toLowerCase() + "://localhost/volume-group-name");
            }
        }

        String hostPath = URLDecoder.decode(storagePath, StringUtils.getPreferredCharset());
        if (hostPath == null) { // if decoding fails, use getPath() anyway
            hostPath = storagePath;
        }

        uriParams.put("scheme", scheme);
        uriParams.put("host", storageHost);
        uriParams.put("hostPath", hostPath);
        uriParams.put("userInfo", uriInfo.getUserInfo());
        if (uriInfo.getPort() > 0) {
            uriParams.put("port", uriInfo.getPort() + "");
        }
        return uriParams;
    }

    private Map<String, String> extractApiParamAsMap(Map<String, ? extends Map<String, String>> ds) {
        Map<String, String> details = new HashMap<>();
        if (ds != null) {
            for (Map<String, String> d : ds.values()) {
                details.putAll(d);
            }
        }
        return details;
    }

    @Override
    @ActionEvent(eventType = EventTypes.EVENT_DISABLE_PRIMARY_STORAGE, eventDescription = "disable storage pool")
    public StoragePool disablePrimaryStoragePool(Long id) {
        StoragePoolVO primaryStorage = _storagePoolDao.findById(id);
        if (primaryStorage == null) {
            throw new IllegalArgumentException(String.format("Unable to find storage pool with ID: %d", id));
        }
        if (!primaryStorage.getStatus().equals(StoragePoolStatus.Up)) {
            throw new InvalidParameterValueException(String.format("Primary storage %s cannot be disabled. Storage pool state : %s", primaryStorage, primaryStorage.getStatus().toString()));
        }

        DataStoreProvider provider = _dataStoreProviderMgr.getDataStoreProvider(primaryStorage.getStorageProviderName());
        DataStoreLifeCycle dataStoreLifeCycle = provider.getDataStoreLifeCycle();
        DataStore store = _dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary);
        ((PrimaryDataStoreLifeCycle)dataStoreLifeCycle).disableStoragePool(store);

        return (PrimaryDataStoreInfo)_dataStoreMgr.getDataStore(id, DataStoreRole.Primary);
    }

    @Override
    @ActionEvent(eventType = EventTypes.EVENT_ENABLE_PRIMARY_STORAGE, eventDescription = "enable storage pool")
    public StoragePool enablePrimaryStoragePool(Long id) {
        StoragePoolVO primaryStorage = _storagePoolDao.findById(id);
        if (primaryStorage == null) {
            throw new IllegalArgumentException(String.format("Unable to find storage pool with ID: %d", id));
        }
        if (!primaryStorage.getStatus().equals(StoragePoolStatus.Disabled)) {
            throw new InvalidParameterValueException(String.format("Primary storage %s cannot be enabled. Storage pool state : %s", primaryStorage, primaryStorage.getStatus()));
        }

        DataStoreProvider provider = _dataStoreProviderMgr.getDataStoreProvider(primaryStorage.getStorageProviderName());
        DataStoreLifeCycle dataStoreLifeCycle = provider.getDataStoreLifeCycle();
        DataStore store = _dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary);
        ((PrimaryDataStoreLifeCycle)dataStoreLifeCycle).enableStoragePool(store);

        return (PrimaryDataStoreInfo)_dataStoreMgr.getDataStore(id, DataStoreRole.Primary);
    }

    @Override
    @ActionEvent(eventType = EventTypes.EVENT_UPDATE_PRIMARY_STORAGE, eventDescription = "update storage pool")
    public PrimaryDataStoreInfo updateStoragePool(UpdateStoragePoolCmd cmd) throws IllegalArgumentException {
        // Input validation
        jsInterpreterHelper.ensureInterpreterEnabledIfParameterProvided(ApiConstants.IS_TAG_A_RULE, Boolean.TRUE.equals(cmd.isTagARule()));

        Long id = cmd.getId();
        StoragePoolVO pool = _storagePoolDao.findById(id);
        if (pool == null) {
            throw new IllegalArgumentException("Unable to find storage pool with ID: " + id);
        }

        Map<String, String> inputDetails = extractApiParamAsMap(cmd.getDetails());
        checkNFSMountOptionsForUpdate(inputDetails, pool, cmd.getEntityOwnerId());

        String name = cmd.getName();
        if(StringUtils.isNotBlank(name)) {
            logger.debug("Updating Storage Pool name to: {}", name);
            pool.setName(name);
            _storagePoolDao.update(pool.getId(), pool);
        }

        final List<String> storagePoolTags = cmd.getTags();
        if (storagePoolTags != null) {
            logger.debug("Updating Storage Pool Tags to : {}", storagePoolTags);
            if (pool.getPoolType() == StoragePoolType.DatastoreCluster) {
                List<StoragePoolVO> childStoragePools = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(pool.getId());
                for (StoragePoolVO childPool : childStoragePools) {
                    _storagePoolTagsDao.persist(childPool.getId(), storagePoolTags, cmd.isTagARule());
                }
            }
            _storagePoolTagsDao.persist(pool.getId(), storagePoolTags, cmd.isTagARule());
        }

        Long updatedCapacityBytes = null;
        Long capacityBytes = cmd.getCapacityBytes();
        // keep track of what this call actually changes, so only those values are persisted
        Map<String, String> details = _storagePoolDetailsDao.listDetailsKeyPairs(id);
        Map<String, String> changedDetails = new HashMap<>();
        if (!inputDetails.isEmpty()) {
            changedDetails.putAll(inputDetails);
        }

        if (capacityBytes != null) {
            if (capacityBytes != pool.getCapacityBytes()) {
                updatedCapacityBytes = capacityBytes;
                changedDetails.put(PrimaryDataStoreLifeCycle.CAPACITY_BYTES, String.valueOf(updatedCapacityBytes));
            }
        }

        Long updatedCapacityIops = null;
        Long capacityIops = cmd.getCapacityIops();
        if (capacityIops != null) {
            if (!capacityIops.equals(pool.getCapacityIops())) {
                updatedCapacityIops = capacityIops;
                changedDetails.put(PrimaryDataStoreLifeCycle.CAPACITY_IOPS, String.valueOf(updatedCapacityIops));
            }
        }

        if (cmd.getUrl() != null) {
            changedDetails.put("url", cmd.getUrl());
        }

        if (!changedDetails.isEmpty()) {
            details.putAll(changedDetails);
            DataStoreProvider dataStoreProvider = _dataStoreProviderMgr.getDataStoreProvider(pool.getStorageProviderName());
            DataStoreLifeCycle dataStoreLifeCycle = dataStoreProvider.getDataStoreLifeCycle();
            if (dataStoreLifeCycle instanceof PrimaryDataStoreLifeCycle) {
                if (updatedCapacityBytes != null) {
                    pool.setCapacityBytes(updatedCapacityBytes);
                }
                if (updatedCapacityIops != null) {
                    pool.setCapacityIops(updatedCapacityIops);
                }
                ((PrimaryDataStoreLifeCycle)dataStoreLifeCycle).updateStoragePool(pool, details);
                _storagePoolDao.update(id, pool);
                // re-read the details so values written by the lifecycle above are preserved,
                // then overlay only the values changed by this call
                Map<String, String> detailsToPersist = _storagePoolDetailsDao.listDetailsKeyPairs(id);
                detailsToPersist.putAll(changedDetails);
                _storagePoolDao.updateDetails(id, detailsToPersist);
            }
        }
        return (PrimaryDataStoreInfo)_dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary);
    }

    private void changeStoragePoolScopeToZone(StoragePoolVO primaryStorage) {
        /*
         * For cluster wide primary storage the hypervisor type might not be set.
         * So, get it from the clusterVO.
         */
        Long clusterId = primaryStorage.getClusterId();
        ClusterVO clusterVO = _clusterDao.findById(clusterId);
        HypervisorType hypervisorType = clusterVO.getHypervisorType();
        if (!zoneWidePoolSupportedHypervisorTypes.contains(hypervisorType)) {
            throw new InvalidParameterValueException("Primary storage scope change to Zone is not supported for hypervisor type " + hypervisorType);
        }

        DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(primaryStorage.getStorageProviderName());
        PrimaryDataStoreLifeCycle lifeCycle = (PrimaryDataStoreLifeCycle) storeProvider.getDataStoreLifeCycle();

        DataStore primaryStore = _dataStoreMgr.getPrimaryDataStore(primaryStorage.getId());
        ClusterScope clusterScope = new ClusterScope(primaryStorage.getClusterId(), null, primaryStorage.getDataCenterId());

        lifeCycle.changeStoragePoolScopeToZone(primaryStore, clusterScope, hypervisorType);
    }

    private void changeStoragePoolScopeToCluster(StoragePoolVO primaryStorage, Long clusterId) {
        if (clusterId == null) {
            throw new InvalidParameterValueException("Cluster ID not provided");
        }
        ClusterVO clusterVO = _clusterDao.findById(clusterId);
        if (clusterVO == null) {
            throw new InvalidParameterValueException("Unable to find cluster by id " + clusterId);
        }
        if (clusterVO.getAllocationState().equals(Grouping.AllocationState.Disabled)) {
            throw new PermissionDeniedException("Cannot perform this operation, Cluster is currently disabled: " + clusterId);
        }

        long id = primaryStorage.getId();
        Pair<List<VMInstanceVO>, Integer> vmsNotInClusterUsingPool = _vmInstanceDao.listByVmsNotInClusterUsingPool(clusterId, id);
        if (vmsNotInClusterUsingPool.second() != 0) {
            throw new CloudRuntimeException(String.format("Cannot change scope of the storage pool [%s] to cluster [%s] " +
                    "as there are %s VMs with volumes in this pool that are running on other clusters. " +
                    "All such User VMs must be stopped and System VMs must be destroyed before proceeding. " +
                    "Please use the API listAffectedVmsForStorageScopeChange to get the list.",
                    primaryStorage.getName(), clusterVO.getName(), vmsNotInClusterUsingPool.second()));
        }

        DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(primaryStorage.getStorageProviderName());
        PrimaryDataStoreLifeCycle lifeCycle = (PrimaryDataStoreLifeCycle) storeProvider.getDataStoreLifeCycle();

        DataStore primaryStore = _dataStoreMgr.getPrimaryDataStore(id);
        ClusterScope clusterScope = new ClusterScope(clusterId, clusterVO.getPodId(), primaryStorage.getDataCenterId());

        lifeCycle.changeStoragePoolScopeToCluster(primaryStore, clusterScope, primaryStorage.getHypervisor());
    }

    @Override
    @ActionEvent(eventType = EventTypes.EVENT_CHANGE_STORAGE_POOL_SCOPE, eventDescription = "changing storage pool scope")
    public void changeStoragePoolScope(ChangeStoragePoolScopeCmd cmd) throws IllegalArgumentException, InvalidParameterValueException, PermissionDeniedException {
        Long id = cmd.getId();

        Long accountId = cmd.getEntityOwnerId();
        if (!_accountMgr.isRootAdmin(accountId)) {
            throw new PermissionDeniedException("Only root admin can perform this operation");
        }

        ScopeType newScope = EnumUtils.getEnumIgnoreCase(ScopeType.class, cmd.getScope());
        if (newScope != ScopeType.ZONE && newScope != ScopeType.CLUSTER) {
            throw new InvalidParameterValueException("Invalid scope " + cmd.getScope() + "for Primary storage");
        }

        StoragePoolVO primaryStorage = _storagePoolDao.findById(id);
        if (primaryStorage == null) {
            throw new IllegalArgumentException("Unable to find storage pool with ID: " + id);
        }

        String eventDetails = String.format(" Storage pool Id: %s to %s",primaryStorage.getUuid(), newScope);
        CallContext.current().setEventDetails(eventDetails);

        ScopeType currentScope = primaryStorage.getScope();
        if (currentScope.equals(newScope)) {
            throw new InvalidParameterValueException("New scope must be different than the current scope");
        }

        if (currentScope != ScopeType.ZONE && currentScope != ScopeType.CLUSTER) {
            throw new InvalidParameterValueException("This operation is supported only for Primary storages having scope "
                    + ScopeType.CLUSTER + " or " + ScopeType.ZONE);
        }

        if (!primaryStorage.getStatus().equals(StoragePoolStatus.Disabled)) {
            throw new InvalidParameterValueException("Scope of the Primary storage with id "
                    + primaryStorage.getUuid() +
                    " cannot be changed, as it is not in the Disabled state");
        }

        Long zoneId = primaryStorage.getDataCenterId();
        DataCenterVO zone = _dcDao.findById(zoneId);
        if (zone == null) {
            throw new InvalidParameterValueException("Unable to find zone by id " + zoneId);
        }
        if (zone.getAllocationState().equals(Grouping.AllocationState.Disabled)) {
            throw new PermissionDeniedException("Cannot perform this operation, Zone is currently disabled: " + zoneId);
        }

        if (newScope.equals(ScopeType.ZONE)) {
            changeStoragePoolScopeToZone(primaryStorage);
        } else {
            changeStoragePoolScopeToCluster(primaryStorage, cmd.getClusterId());
        }
    }

    @Override
    @ActionEvent(eventType = EventTypes.EVENT_CONFIGURE_STORAGE_ACCESS, eventDescription = "configuring storage groups", async = true)
    public boolean configureStorageAccess(ConfigureStorageAccessCmd cmd) {
        Long zoneId = cmd.getZoneId();
        Long podId = cmd.getPodId();
        Long clusterId = cmd.getClusterId();
        Long hostId = cmd.getHostId();
        Long storagePoolId = cmd.getStorageId();

        long nonNullCount = Stream.of(zoneId, podId, clusterId, hostId, storagePoolId)
                .filter(Objects::nonNull)
                .count();

        if (nonNullCount != 1) {
            throw new IllegalArgumentException("Exactly one of zoneid, podid, clusterid, hostid or storagepoolid is required");
        }

        // SAG -> Storage Access Group
        List<String> storageAccessGroups = cmd.getStorageAccessGroups();
        if (storageAccessGroups == null) {
            throw new InvalidParameterValueException("storageaccessgroups parameter is required");
        }

        if (zoneId != null) {
            DataCenterVO zone = _dcDao.findById(zoneId);
            Set<String> existingSAGsSet = (zone.getStorageAccessGroups() == null || zone.getStorageAccessGroups().isEmpty())
                    ? Collections.emptySet()
                    : new HashSet<>(Arrays.asList(zone.getStorageAccessGroups().split(",")));

            Set<String> storagePoolSAGsSet = new HashSet<>(storageAccessGroups);
            if (!existingSAGsSet.equals(storagePoolSAGsSet)) {
                _resourceMgr.updateZoneStorageAccessGroups(zone.getId(), storageAccessGroups);
                String preparedStoragePoolTags = CollectionUtils.isEmpty(storageAccessGroups) ? null : String.join(",", storageAccessGroups);
                zone.setStorageAccessGroups(preparedStoragePoolTags);

                if (!_dcDao.update(zoneId, zone)) {
                    throw new CloudRuntimeException("Failed to update zone with the storage access groups.");
                }
            }
        }

        if (podId != null) {
            HostPodVO pod = _podDao.findById(podId);
            Set<String> existingTagsSet = (pod.getStorageAccessGroups() == null || pod.getStorageAccessGroups().isEmpty())
                    ? Collections.emptySet()
                    : new HashSet<>(Arrays.asList(pod.getStorageAccessGroups().split(",")));

            if (CollectionUtils.isNotEmpty(storageAccessGroups)) {
                checkIfStorageAccessGroupsExistsOnZone(pod.getDataCenterId(), storageAccessGroups);
            }

            Set<String> storagePoolTagsSet = new HashSet<>(storageAccessGroups);
            if (!existingTagsSet.equals(storagePoolTagsSet)) {
                _resourceMgr.updatePodStorageAccessGroups(podId, storageAccessGroups);
                String preparedStoragePoolTags = CollectionUtils.isEmpty(storageAccessGroups) ? null : String.join(",", storageAccessGroups);
                pod.setStorageAccessGroups(preparedStoragePoolTags);

                if (!_podDao.update(podId, pod)) {
                    throw new CloudRuntimeException("Failed to update pod with the storage access groups.");
                }
            }
        }

        if (clusterId != null) {
            ClusterVO cluster = _clusterDao.findById(clusterId);
            Set<String> existingTagsSet = (cluster.getStorageAccessGroups() == null || cluster.getStorageAccessGroups().isEmpty())
                    ? Collections.emptySet()
                    : new HashSet<>(Arrays.asList(cluster.getStorageAccessGroups().split(",")));

            if (CollectionUtils.isNotEmpty(storageAccessGroups)) {
                checkIfStorageAccessGroupsExistsOnPod(cluster.getPodId(), storageAccessGroups);
            }

            Set<String> storagePoolTagsSet = new HashSet<>(storageAccessGroups);
            if (!existingTagsSet.equals(storagePoolTagsSet)) {
                _resourceMgr.updateClusterStorageAccessGroups(cluster.getId(), storageAccessGroups);
                String preparedStoragePoolTags = CollectionUtils.isEmpty(storageAccessGroups) ? null : String.join(",", storageAccessGroups);
                cluster.setStorageAccessGroups(preparedStoragePoolTags);

                if (!_clusterDao.update(clusterId, cluster)) {
                    throw new CloudRuntimeException("Failed to update cluster with the storage access groups.");
                }
            }
        }

        if (hostId != null) {
            HostVO host = _hostDao.findById(hostId);
            Set<String> existingTagsSet = (host.getStorageAccessGroups() == null || host.getStorageAccessGroups().isEmpty())
                    ? Collections.emptySet()
                    : new HashSet<>(Arrays.asList(host.getStorageAccessGroups().split(",")));

            if (CollectionUtils.isNotEmpty(storageAccessGroups)) {
                checkIfStorageAccessGroupsExistsOnCluster(host.getClusterId(), storageAccessGroups);
            }

            Set<String> storageAccessGroupsSet = new HashSet<>(storageAccessGroups);
            if (!existingTagsSet.equals(storageAccessGroupsSet)) {
                _resourceMgr.updateHostStorageAccessGroups(hostId, storageAccessGroups);
                String preparedStoragePoolTags = CollectionUtils.isEmpty(storageAccessGroups) ? null : String.join(",", storageAccessGroups);
                host.setStorageAccessGroups(preparedStoragePoolTags);

                if (!_hostDao.update(hostId, host)) {
                    throw new CloudRuntimeException("Failed to update host with the storage access groups.");
                }
            }
        }

        if (storagePoolId != null) {
            StoragePoolVO storagePool = _storagePoolDao.findById(storagePoolId);
            if (ScopeType.HOST.equals(storagePool.getScope())) {
                throw new CloudRuntimeException("Storage Access Groups are not suitable for local storage");
            }

            logger.debug("Updating Storage Pool Access Group Maps to : {}", storageAccessGroups);

            if (storagePool.getPoolType() == StoragePoolType.DatastoreCluster) {
                List<StoragePoolVO> childStoragePools = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(storagePool.getId());
                for (StoragePoolVO childPool : childStoragePools) {
                    _resourceMgr.updateStoragePoolConnectionsOnHosts(childPool.getId(), storageAccessGroups);
                    _storagePoolAccessGroupMapDao.persist(childPool.getId(), storageAccessGroups);
                }
            } else {
                _resourceMgr.updateStoragePoolConnectionsOnHosts(storagePool.getId(), storageAccessGroups);
            }

            _storagePoolAccessGroupMapDao.persist(storagePool.getId(), storageAccessGroups);
        }

        return true;
    }

    protected void checkIfStorageAccessGroupsExistsOnZone(long zoneId, List<String> storageAccessGroups) {
        DataCenterVO zoneVO = _dcDao.findById(zoneId);

        String storageAccessGroupsOnZone = zoneVO.getStorageAccessGroups();
        List<String> zoneTagsList = parseTags(storageAccessGroupsOnZone);

        List<String> existingTagsOnZone = new ArrayList<>(CollectionUtils.intersection(storageAccessGroups, zoneTagsList));

        if (CollectionUtils.isNotEmpty(existingTagsOnZone)) {
            throw new CloudRuntimeException(String.format("access groups already exist on the zone: %s", existingTagsOnZone));
        }
    }

    protected void checkIfStorageAccessGroupsExistsOnPod(long podId, List<String> storageAccessGroups) {
        HostPodVO podVO = _podDao.findById(podId);
        DataCenterVO zoneVO = _dcDao.findById(podVO.getDataCenterId());

        String storageAccessGroupsOnPod = podVO.getStorageAccessGroups();
        String storageAccessGroupsOnZone = zoneVO.getStorageAccessGroups();

        List<String> podTagsList = parseTags(storageAccessGroupsOnPod);
        List<String> zoneTagsList = parseTags(storageAccessGroupsOnZone);

        List<String> existingTagsOnPod = new ArrayList<>(CollectionUtils.intersection(storageAccessGroups, podTagsList));
        List<String> existingTagsOnZone = new ArrayList<>(CollectionUtils.intersection(storageAccessGroups, zoneTagsList));

        if (CollectionUtils.isNotEmpty(existingTagsOnPod) || CollectionUtils.isNotEmpty(existingTagsOnZone)) {
            String message = "access groups already exist ";

            if (CollectionUtils.isNotEmpty(existingTagsOnPod)) {
                message += String.format("on the pod: %s", existingTagsOnPod);
            }
            if (CollectionUtils.isNotEmpty(existingTagsOnZone)) {
                if (CollectionUtils.isNotEmpty(existingTagsOnPod)) {
                    message += ", ";
                }
                message += String.format("on the zone: %s", existingTagsOnZone);
            }

            throw new CloudRuntimeException(message);
        }
    }

    protected void checkIfStorageAccessGroupsExistsOnCluster(long clusterId, List<String> storageAccessGroups) {
        ClusterVO clusterVO = _clusterDao.findById(clusterId);
        HostPodVO podVO = _podDao.findById(clusterVO.getPodId());
        DataCenterVO zoneVO = _dcDao.findById(podVO.getDataCenterId());

        String storageAccessGroupsOnCluster = clusterVO.getStorageAccessGroups();
        String storageAccessGroupsOnPod = podVO.getStorageAccessGroups();
        String storageAccessGroupsOnZone = zoneVO.getStorageAccessGroups();

        List<String> podTagsList = parseTags(storageAccessGroupsOnPod);
        List<String> zoneTagsList = parseTags(storageAccessGroupsOnZone);
        List<String> clusterTagsList = parseTags(storageAccessGroupsOnCluster);

        List<String> existingTagsOnCluster = new ArrayList<>(CollectionUtils.intersection(storageAccessGroups, clusterTagsList));
        List<String> existingTagsOnPod = new ArrayList<>(CollectionUtils.intersection(storageAccessGroups, podTagsList));
        List<String> existingTagsOnZone = new ArrayList<>(CollectionUtils.intersection(storageAccessGroups, zoneTagsList));

        if (CollectionUtils.isNotEmpty(existingTagsOnCluster) || CollectionUtils.isNotEmpty(existingTagsOnPod) || CollectionUtils.isNotEmpty(existingTagsOnZone)) {
            String message = "access groups already exist ";

            if (CollectionUtils.isNotEmpty(existingTagsOnCluster)) {
                message += String.format("on the cluster: %s", existingTagsOnCluster);
            }
            if (CollectionUtils.isNotEmpty(existingTagsOnPod)) {
                if (CollectionUtils.isNotEmpty(existingTagsOnCluster)) {
                    message += ", ";
                }
                message += String.format("on the pod: %s", existingTagsOnPod);
            }
            if (CollectionUtils.isNotEmpty(existingTagsOnZone)) {
                if (CollectionUtils.isNotEmpty(existingTagsOnCluster) || CollectionUtils.isNotEmpty(existingTagsOnPod)) {
                    message += ", ";
                }
                message += String.format("on the zone: %s", existingTagsOnZone);
            }

            throw new CloudRuntimeException(message);
        }
    }

    private List<String> parseTags(String tags) {
        if (tags == null || tags.trim().isEmpty()) {
            return Collections.emptyList();
        }
        return Arrays.asList(tags.split(","));
    }

    @Override
    public void removeStoragePoolFromCluster(long hostId, String iScsiName, StoragePool storagePool) {
        final Map<String, String> details = new HashMap<>();

        details.put(DeleteStoragePoolCommand.DATASTORE_NAME, iScsiName);
        details.put(DeleteStoragePoolCommand.IQN, iScsiName);
        details.put(DeleteStoragePoolCommand.STORAGE_HOST, storagePool.getHostAddress());
        details.put(DeleteStoragePoolCommand.STORAGE_PORT, String.valueOf(storagePool.getPort()));

        final DeleteStoragePoolCommand cmd = new DeleteStoragePoolCommand();

        cmd.setDetails(details);
        cmd.setRemoveDatastore(true);

        final Answer answer = _agentMgr.easySend(hostId, cmd);

        if (answer == null || !answer.getResult()) {
            String errMsg = "Error interacting with host (related to DeleteStoragePoolCommand)" + (answer == null ? "" : (StringUtils.isNotBlank(answer.getDetails()) ? ": " + answer.getDetails() : ""));

            logger.error(errMsg);

            throw new CloudRuntimeException(errMsg);
        }
    }

    @Override
    @DB
    public boolean deletePool(DeletePoolCmd cmd) {
        Long id = cmd.getId();
        boolean forced = cmd.isForced();

        StoragePoolVO sPool = _storagePoolDao.findById(id);
        if (sPool == null) {
            logger.warn("Unable to find pool: {}", id);
            throw new InvalidParameterValueException("Unable to find pool by id " + id);
        }
        if (sPool.getStatus() != StoragePoolStatus.Maintenance) {
            logger.warn("Unable to delete storage pool: {} due to it is not in Maintenance state", sPool);
            throw new InvalidParameterValueException(String.format("Unable to delete storage due to it is not in Maintenance state, pool: %s", sPool));
        }

        if (sPool.getPoolType() == StoragePoolType.DatastoreCluster) {
            // FR41 yet to handle on failure of deletion of any child storage pool
            if (checkIfDataStoreClusterCanbeDeleted(sPool, forced)) {
                Transaction.execute(new TransactionCallbackNoReturn() {
                    @Override
                    public void doInTransactionWithoutResult(TransactionStatus status) {
                        List<StoragePoolVO> childStoragePools = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(sPool.getId());
                        for (StoragePoolVO childPool : childStoragePools) {
                            deleteDataStoreInternal(childPool, forced);
                        }
                    }
                });
            } else {
                logger.debug("Cannot delete storage pool {} as the following non-destroyed volumes are on it: {}.", sPool::toString, () -> getStoragePoolNonDestroyedVolumesLog(sPool.getId()));
                throw new CloudRuntimeException(String.format("Cannot delete pool %s as there are associated non-destroyed vols for this pool", sPool));
            }
        }
        return deleteDataStoreInternal(sPool, forced);
    }

    @Override
    public Pair<Map<String, String>, Boolean> getStoragePoolNFSMountOpts(StoragePool pool, Map<String, String> details) {
        boolean details_added = false;
        if (pool.getPoolType().equals(Storage.StoragePoolType.NetworkFilesystem)) {
            StoragePoolDetailVO nfsMountOpts = _storagePoolDetailsDao.findDetail(pool.getId(), ApiConstants.NFS_MOUNT_OPTIONS);
            if (nfsMountOpts != null) {
                if (details == null) {
                    details = new HashMap<>();
                }
                details.put(ApiConstants.NFS_MOUNT_OPTIONS, nfsMountOpts.getValue());
                details_added = true;
            }
        }
        return new Pair<>(details, details_added);
    }

    public String getStoragePoolMountFailureReason(String reason) {
        if (reason.toLowerCase().contains(NFS_MOUNT_OPTIONS_INCORRECT.toLowerCase())) {
            return NFS_MOUNT_OPTIONS_INCORRECT;
        } else {
            return null;
        }
    }

    private boolean checkIfDataStoreClusterCanbeDeleted(StoragePoolVO sPool, boolean forced) {
        List<StoragePoolVO> childStoragePools = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(sPool.getId());
        boolean canDelete = true;
        for (StoragePoolVO childPool : childStoragePools) {
            Pair<Long, Long> vlms = volumeDao.getCountAndTotalByPool(childPool.getId());
            if (forced) {
                if (vlms.first() > 0) {
                    Pair<Long, Long> nonDstrdVlms = volumeDao.getNonDestroyedCountAndTotalByPool(childPool.getId());
                    if (nonDstrdVlms.first() > 0) {
                        canDelete = false;
                        break;
                    }
                }
            } else {
                if (vlms.first() > 0) {
                    canDelete = false;
                    break;
                }
            }
        }
        return canDelete;
    }

    private boolean deleteDataStoreInternal(StoragePoolVO sPool, boolean forced) {
        Pair<Long, Long> vlms = volumeDao.getCountAndTotalByPool(sPool.getId());
        if (forced) {
            if (vlms.first() > 0) {
                Pair<Long, Long> nonDstrdVlms = volumeDao.getNonDestroyedCountAndTotalByPool(sPool.getId());
                if (nonDstrdVlms.first() > 0) {
                    logger.debug("Cannot delete storage pool {} as the following non-destroyed volumes are on it: {}.", sPool::toString, () -> getStoragePoolNonDestroyedVolumesLog(sPool.getId()));
                    throw new CloudRuntimeException(String.format("Cannot delete pool %s as there are non-destroyed volumes associated to this pool.", sPool));
                }
                // force expunge non-destroyed volumes
                List<VolumeVO> vols = volumeDao.listVolumesToBeDestroyed();
                for (VolumeVO vol : vols) {
                    AsyncCallFuture<VolumeApiResult> future = volService.expungeVolumeAsync(volFactory.getVolume(vol.getId()));
                    try {
                        future.get();
                    } catch (InterruptedException | ExecutionException e) {
                        logger.debug("expunge volume failed: {}", vol, e);
                    }
                }
            }
        } else {
            // Check if the pool has associated volumes in the volumes table
            // If it does , then you cannot delete the pool
            if (vlms.first() > 0) {
                logger.debug("Cannot delete storage pool {} as the following non-destroyed volumes are on it: {}.", sPool::toString, () -> getStoragePoolNonDestroyedVolumesLog(sPool.getId()));
                throw new CloudRuntimeException(String.format("Cannot delete pool %s as there are non-destroyed volumes associated to this pool.", sPool));
            }
        }

        // First get the host_id from storage_pool_host_ref for given pool id
        StoragePoolVO lock = _storagePoolDao.acquireInLockTable(sPool.getId());

        if (lock == null) {
            logger.debug("Failed to acquire lock when deleting PrimaryDataStoreVO: {}", sPool);
            return false;
        }

        _storagePoolDao.releaseFromLockTable(lock.getId());
        logger.trace("Released lock for storage pool {}", sPool);

        DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(sPool.getStorageProviderName());
        DataStoreLifeCycle lifeCycle = storeProvider.getDataStoreLifeCycle();
        DataStore store = _dataStoreMgr.getDataStore(sPool.getId(), DataStoreRole.Primary);
        return lifeCycle.deleteDataStore(store);
    }

    protected String getStoragePoolNonDestroyedVolumesLog(long storagePoolId) {
        StringBuilder sb = new StringBuilder();
        List<VolumeVO> nonDestroyedVols = volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null);
        VMInstanceVO volInstance;
        List<String> logMessageInfo = new ArrayList<>();

        sb.append("[");
        for (VolumeVO vol : nonDestroyedVols) {
            if (vol.getInstanceId() != null) {
                volInstance = _vmInstanceDao.findById(vol.getInstanceId());
                if (volInstance != null) {
                    logMessageInfo.add(String.format("Volume [%s] (attached to VM [%s])", vol.getUuid(), volInstance.getUuid()));
                } else {
                    logMessageInfo.add(String.format("Volume [%s] (attached VM with ID [%d] doesn't exists)", vol.getUuid(), vol.getInstanceId()));
                }
            } else {
                logMessageInfo.add(String.format("Volume [%s] (not attached to any VM)", vol.getUuid()));
            }
        }
        sb.append(String.join(", ", logMessageInfo));
        sb.append("]");

        return sb.toString();
    }

    protected void cleanupConnectedHostConnectionForFailedStorage(DataStore primaryStore, List<Long> poolHostIds) {
        List<HostVO> hosts = _hostDao.listByIds(poolHostIds);
        StoragePool pool = _storagePoolDao.findById(primaryStore.getId());
        for (HostVO host : hosts) {
            try {
                disconnectHostFromSharedPool(host, pool);
            } catch (StorageUnavailableException | StorageConflictException e) {
                logger.error("Error during cleaning up failed storage host connection", e);
            }
        }
    }

    @Override
    public void connectHostsToPool(DataStore primaryStore, List<Long> hostIds, Scope scope,
              boolean handleExceptionsPartially, boolean errorOnNoUpHost) throws CloudRuntimeException {
        if (CollectionUtils.isEmpty(hostIds)) {
            return;
        }
        Map<Long, HostVO> hostMap = new HashMap<>();
        for (HostVO h : _hostDao.listByIds(hostIds)) {
            hostMap.put(h.getId(), h);
        }
        CopyOnWriteArrayList<Long> poolHostIds = new CopyOnWriteArrayList<>();
        ExecutorService executorService = Executors.newFixedThreadPool(Math.max(1, Math.min(hostIds.size(),
                StoragePoolHostConnectWorkers.value())));
        List<Future<Void>> futures = new ArrayList<>();
        AtomicBoolean exceptionOccurred = new AtomicBoolean(false);
        for (Long hostId : hostIds) {
            futures.add(executorService.submit(() -> {
                if (exceptionOccurred.get()) {
                    return null;
                }
                HostVO host = hostMap.get(hostId);
                if (host == null) {
                    logger.warn("Host [{}] not found, skipping pool connection for {}", hostId, primaryStore);
                    return null;
                }
                Transaction.execute(new TransactionCallbackWithExceptionNoReturn<Exception>() {
                    @Override
                    public void doInTransactionWithoutResult(TransactionStatus status) throws Exception {
                        try {
                            connectHostToSharedPool(host, primaryStore.getId());
                            poolHostIds.add(hostId);
                        } catch (Exception e) {
                            if (handleExceptionsPartially && e.getCause() instanceof StorageConflictException) {
                                exceptionOccurred.set(true);
                                throw e;
                            }
                            logger.warn("Unable to establish a connection between {} and {}", host, primaryStore, e);
                            String reason = getStoragePoolMountFailureReason(e.getMessage());
                            if (handleExceptionsPartially && reason != null) {
                                exceptionOccurred.set(true);
                                throw new CloudRuntimeException(reason);
                            }
                        }
                    }
                });
                return null;
            }));
        }
        for (Future<Void> future : futures) {
            try {
                future.get();
            } catch (Exception e) {
                Throwable cause = e.getCause();
                if (cause instanceof StorageConflictException || cause instanceof CloudRuntimeException) {
                    executorService.shutdown();
                    cleanupConnectedHostConnectionForFailedStorage(primaryStore, poolHostIds);
                    primaryStoreDao.expunge(primaryStore.getId());
                    if (cause instanceof CloudRuntimeException) {
                        throw (CloudRuntimeException)cause;
                    }
                    throw new CloudRuntimeException("Storage has already been added as local storage", e);
                } else {
                    logger.error("Error occurred while connecting host to shared pool", e);
                }
            }
        }
        executorService.shutdown();
        if (poolHostIds.isEmpty()) {
            logger.warn("No host can access storage pool {} in {}: {}.",
                    primaryStore, scope.getScopeType(), scope.getScopeId());
            if (errorOnNoUpHost) {
                primaryStoreDao.expunge(primaryStore.getId());
                throw new CloudRuntimeException("Failed to access storage pool");
            }
        }
    }

    @Override
    public boolean connectHostToSharedPool(Host host, long poolId) throws StorageUnavailableException, StorageConflictException {
        StoragePool pool = (StoragePool)_dataStoreMgr.getDataStore(poolId, DataStoreRole.Primary);
        assert (pool.isShared()) : "Now, did you actually read the name of this method?";
        logger.debug("Adding pool {} to  host {}", pool, host);

        DataStoreProvider provider = _dataStoreProviderMgr.getDataStoreProvider(pool.getStorageProviderName());
        HypervisorHostListener listener = hostListeners.get(provider.getName());
        return listener.hostConnect(host, pool);
    }

    @Override
    public void disconnectHostFromSharedPool(Host host, StoragePool pool) throws StorageUnavailableException, StorageConflictException {
        assert (pool.isShared()) : "Now, did you actually read the name of this method?";
        logger.debug("Removing pool {} from host {}", pool, host);

        DataStoreProvider provider = _dataStoreProviderMgr.getDataStoreProvider(pool.getStorageProviderName());
        HypervisorHostListener listener = hostListeners.get(provider.getName());
        listener.hostDisconnected(host, pool);
    }

    @Override
    public void enableHost(long hostId) {
        List<DataStoreProvider> providers = _dataStoreProviderMgr.getProviders();
        if (providers != null) {
            for (DataStoreProvider provider : providers) {
                if (provider instanceof PrimaryDataStoreProvider) {
                    try {
                        HypervisorHostListener hypervisorHostListener = provider.getHostListener();
                        if (hypervisorHostListener != null) {
                            hypervisorHostListener.hostEnabled(hostId);
                        }
                    }
                    catch (Exception ex) {
                        logger.error("hostEnabled(long) failed for storage provider {}", provider.getName(), ex);
                    }
                }
            }
        }
    }

    @Override
    public BigDecimal getStorageOverProvisioningFactor(Long poolId) {
        return BigDecimal.valueOf(CapacityManager.StorageOverprovisioningFactor.valueIn(poolId));
    }

    @Override
    public void createCapacityEntry(StoragePoolVO storagePool, short capacityType, long allocated) {
        SearchCriteria<CapacityVO> capacitySC = PoolCapacitySearch.create();
        capacitySC.setParameters("hostOrPoolId", storagePool.getId());
        capacitySC.setParameters("dataCenterId", storagePool.getDataCenterId());
        capacitySC.setParameters("capacityType", capacityType);

        List<CapacityVO> capacities = _capacityDao.search(capacitySC, null);

        long totalOverProvCapacity;
        if (storagePool.getPoolType().supportsOverProvisioning()) {
            // All this is for the inaccuracy of floats for big number multiplication.
            BigDecimal overProvFactor = getStorageOverProvisioningFactor(storagePool.getId());
            totalOverProvCapacity = overProvFactor.multiply(new BigDecimal(storagePool.getCapacityBytes())).longValue();
            logger.debug("Found storage pool {} of type {} with overprovisioning factor {}", storagePool, storagePool.getPoolType(), overProvFactor);
            logger.debug("Total over provisioned capacity calculated is {} * {}", overProvFactor, toHumanReadableSize(storagePool.getCapacityBytes()));
        } else {
            logger.debug("Found storage pool {} of type {}", storagePool, storagePool.getPoolType());
            totalOverProvCapacity = storagePool.getCapacityBytes();
        }

        logger.debug("Total over provisioned capacity of the pool {} is {}", storagePool, toHumanReadableSize(totalOverProvCapacity));
        CapacityState capacityState = CapacityState.Enabled;
        if (storagePool.getScope() == ScopeType.ZONE) {
            DataCenterVO dc = _dcDao.findById(storagePool.getDataCenterId());
            AllocationState allocationState = dc.getAllocationState();
            capacityState = (allocationState == AllocationState.Disabled) ? CapacityState.Disabled : CapacityState.Enabled;
        } else {
            if (storagePool.getClusterId() != null) {
                ClusterVO cluster = ApiDBUtils.findClusterById(storagePool.getClusterId());
                if (cluster != null) {
                    AllocationState allocationState = _configMgr.findClusterAllocationState(cluster);
                    capacityState = (allocationState == AllocationState.Disabled) ? CapacityState.Disabled : CapacityState.Enabled;
                }
            }
        }

        if (storagePool.getScope() == ScopeType.HOST) {
            List<StoragePoolHostVO> stoargePoolHostVO = _storagePoolHostDao.listByPoolId(storagePool.getId());

            if (stoargePoolHostVO != null && !stoargePoolHostVO.isEmpty()) {
                HostVO host = _hostDao.findById(stoargePoolHostVO.get(0).getHostId());

                if (host != null) {
                    capacityState = (host.getResourceState() == ResourceState.Disabled) ? CapacityState.Disabled : CapacityState.Enabled;
                }
            }
        }

        if (capacities.isEmpty()) {
            CapacityVO capacity = new CapacityVO(storagePool.getId(), storagePool.getDataCenterId(), storagePool.getPodId(), storagePool.getClusterId(), allocated, totalOverProvCapacity,
                    capacityType);
            capacity.setCapacityState(capacityState);
            _capacityDao.persist(capacity);
        } else {
            CapacityVO capacity = capacities.get(0);
            if (capacity.getTotalCapacity() != totalOverProvCapacity || allocated != capacity.getUsedCapacity() || capacity.getCapacityState() != capacityState) {
                capacity.setTotalCapacity(totalOverProvCapacity);
                capacity.setUsedCapacity(allocated);
                capacity.setCapacityState(capacityState);
                _capacityDao.update(capacity.getId(), capacity);
            }
        }
        logger.debug("Successfully set Capacity - {} for capacity type - {} , DataCenterId - {}, Pool - {}, PodId {}",
                toHumanReadableSize(totalOverProvCapacity), capacityType, storagePool.getDataCenterId(), storagePool, storagePool.getPodId());
    }

    @Override
    public List<Long> getUpHostsInPool(long poolId) {
        SearchCriteria<Long> sc = UpHostsInPoolSearch.create();
        sc.setParameters("pool", poolId);
        sc.setJoinParameters("hosts", "status", Status.Up);
        sc.setJoinParameters("hosts", "resourceState", ResourceState.Enabled);
        return _storagePoolHostDao.customSearch(sc, null);
    }

    @Override
    public Pair<Long, Answer[]> sendToPool(StoragePool pool, long[] hostIdsToTryFirst, List<Long> hostIdsToAvoid, Commands cmds) throws StorageUnavailableException {
        List<Long> hostIds = getUpHostsInPool(pool.getId());
        Collections.shuffle(hostIds);
        if (hostIdsToTryFirst != null) {
            for (int i = hostIdsToTryFirst.length - 1; i >= 0; i--) {
                if (hostIds.remove(hostIdsToTryFirst[i])) {
                    hostIds.add(0, hostIdsToTryFirst[i]);
                }
            }
        }

        if (hostIdsToAvoid != null) {
            hostIds.removeAll(hostIdsToAvoid);
        }
        if (hostIds.isEmpty()) {
            throw new StorageUnavailableException(String.format("Unable to send command to the pool %s due to there is no enabled hosts up in this cluster", pool), pool.getId());
        }
        for (Long hostId : hostIds) {
            try {
                List<Answer> answers = new ArrayList<>();
                Command[] cmdArray = cmds.toCommands();
                for (Command cmd : cmdArray) {
                    long targetHostId = _hvGuruMgr.getGuruProcessedCommandTargetHost(hostId, cmd);
                    answers.add(_agentMgr.send(targetHostId, cmd));
                }
                return new Pair<>(hostId, answers.toArray(new Answer[0]));
            } catch (AgentUnavailableException | OperationTimedoutException e) {
                logger.debug("Unable to send storage pool command to {} via {}", pool::toString, () -> _hostDao.findById(hostId), () -> e);
            }
        }

        throw new StorageUnavailableException("Unable to send command to the pool ", pool.getId());
    }

    @Override
    public Pair<Long, Answer> sendToPool(StoragePool pool, long[] hostIdsToTryFirst, List<Long> hostIdsToAvoid, Command cmd) throws StorageUnavailableException {
        Commands cmds = new Commands(cmd);
        Pair<Long, Answer[]> result = sendToPool(pool, hostIdsToTryFirst, hostIdsToAvoid, cmds);
        return new Pair<>(result.first(), result.second()[0]);
    }

    private void cleanupInactiveTemplates() {
        List<VMTemplateVO> vmTemplateVOS = _templateDao.listUnRemovedTemplatesByStates(VirtualMachineTemplate.State.Inactive);
        for (VMTemplateVO template: vmTemplateVOS) {
            template.setRemoved(new Date());
            _templateDao.update(template.getId(), template);
        }
    }

    @Override
    public void cleanupStorage() {
        GlobalLock scanLock = GlobalLock.getInternLock("storagemgr.cleanup");

        try {
            if (scanLock.lock(3)) {
                try {
                    // Cleanup primary storage pools
                    if (TemplateCleanupEnabled.value()) {
                        List<StoragePoolVO> storagePools = _storagePoolDao.listAll();
                        for (StoragePoolVO pool : storagePools) {
                            try {

                                List<VMTemplateStoragePoolVO> unusedTemplatesInPool = _tmpltMgr.getUnusedTemplatesInPool(pool);
                                logger.debug("Storage pool garbage collector found [{}] Templates to be cleaned up in storage pool [{}].", unusedTemplatesInPool.size(), pool);
                                for (VMTemplateStoragePoolVO templatePoolVO : unusedTemplatesInPool) {
                                    if (templatePoolVO.getDownloadState() != VMTemplateStorageResourceAssoc.Status.DOWNLOADED) {
                                        logger.debug("Storage pool garbage collector is skipping " +
                                                "template: {} on pool {} because it is not completely downloaded.",
                                                () -> _templateDao.findById(templatePoolVO.getTemplateId()), () -> _storagePoolDao.findById(templatePoolVO.getPoolId()));
                                        continue;
                                    }

                                    if (!templatePoolVO.getMarkedForGC()) {
                                        templatePoolVO.setMarkedForGC(true);
                                        _vmTemplatePoolDao.update(templatePoolVO.getId(), templatePoolVO);
                                        logger.debug("Storage pool garbage collector has marked template [{}] on pool [{}] " +
                                                "for garbage collection.",
                                                () -> _templateDao.findById(templatePoolVO.getTemplateId()), () -> _storagePoolDao.findById(templatePoolVO.getPoolId()));
                                        continue;
                                    }

                                    _tmpltMgr.evictTemplateFromStoragePool(templatePoolVO);
                                }
                            } catch (Exception e) {
                                logger.error("Failed to clean up primary storage pool [{}] due to: [{}].", pool, e.getMessage());
                                logger.debug("Failed to clean up primary storage pool [{}].", pool, e);
                            }
                        }
                    }

                    cleanupSnapshotsFromStoreRefInDestroyingState();
                    cleanupSecondaryStorage();

                    List<VolumeVO> vols = volumeDao.listVolumesToBeDestroyed(new Date(System.currentTimeMillis() - ((long)StorageCleanupDelay.value() << 10)));
                    List<Long> rootVolumeVmIds = vols.stream()
                            .filter(v -> Type.ROOT.equals(v.getVolumeType()) && v.getInstanceId() != null)
                            .map(VolumeVO::getInstanceId)
                            .distinct()
                            .collect(Collectors.toList());
                    Map<Long, VMInstanceVO> vmMap = new HashMap<>();
                    for (VMInstanceVO vm : _vmInstanceDao.listByIds(rootVolumeVmIds)) {
                        vmMap.put(vm.getId(), vm);
                    }
                    for (VolumeVO vol : vols) {
                        if (Type.ROOT.equals(vol.getVolumeType())) {
                            VMInstanceVO vmInstanceVO = vol.getInstanceId() != null ? vmMap.get(vol.getInstanceId()) : null;
                             if (vmInstanceVO != null && vmInstanceVO.getState() == State.Destroyed) {
                                 logger.debug("ROOT volume [{}] will not be expunged because the VM is [{}], therefore this volume will be expunged with the VM"
                                         + " cleanup job.", vol, vmInstanceVO.getState());
                                 continue;
                             }
                        }
                        if (isVolumeSuspectedDestroyDuplicateOfVmVolume(vol)) {
                            logger.warn("Skipping cleaning up {} as it could be a duplicate for another volume on same pool", vol);
                            continue;
                        }
                        try {
                            // If this fails, just log a warning. It's ideal if we clean up the host-side clustered file
                            // system, but not necessary.
                            handleManagedStorage(vol);
                        } catch (Exception e) {
                            logger.error("Unable to destroy host-side clustered file system [{}] due to: [{}].", vol, e.getMessage());
                            logger.debug("Unable to destroy host-side clustered file system [{}].", vol, e);
                        }

                        try {
                            VolumeInfo volumeInfo = volFactory.getVolume(vol.getId());
                            if (volumeInfo != null) {
                                volService.ensureVolumeIsExpungeReady(vol.getId());
                                volService.expungeVolumeAsync(volumeInfo);
                            } else {
                                logger.debug("Volume [{}] is already destroyed.", vol);
                            }
                        } catch (Exception e) {
                            logger.error("Unable to destroy volume [{}] due to: [{}].", vol, e.getMessage());
                            logger.debug("Unable to destroy volume [{}].", vol, e);
                        }
                    }

                    removeSnapshotsInErrorStatus();

                    // destroy uploaded volumes in abandoned/error state
                    List<VolumeDataStoreVO> volumeDataStores = _volumeDataStoreDao.listByVolumeState(Volume.State.UploadError, Volume.State.UploadAbandoned);
                    for (VolumeDataStoreVO volumeDataStore : volumeDataStores) {
                        VolumeVO volume = volumeDao.findById(volumeDataStore.getVolumeId());
                        if (volume == null) {
                            logger.warn("Uploaded volume [{}] not found, so cannot be destroyed.", volumeDataStore.getVolumeId());
                            continue;
                        }
                        try {
                            DataStore dataStore = _dataStoreMgr.getDataStore(volumeDataStore.getDataStoreId(), DataStoreRole.Image);
                            EndPoint ep = _epSelector.select(dataStore, volumeDataStore.getExtractUrl());
                            if (ep == null) {
                                logger.warn("There is no secondary storage VM for image store {}, cannot destroy uploaded volume {}.", dataStore, volume);
                                continue;
                            }
                            Host host = _hostDao.findById(ep.getId());
                            if (host != null && host.getManagementServerId() != null) {
                                if (_serverId == host.getManagementServerId()) {
                                    volService.destroyVolume(volume.getId());
                                    // decrement volume resource count
                                    _resourceLimitMgr.decrementVolumeResourceCount(volume.getAccountId(), volume.isDisplayVolume(),
                                            null, _diskOfferingDao.findByIdIncludingRemoved(volume.getDiskOfferingId()), null);
                                    // expunge volume from secondary if volume is on image store
                                    VolumeInfo volOnSecondary = volFactory.getVolume(volume.getId(), DataStoreRole.Image);
                                    if (volOnSecondary != null) {
                                        logger.info("Expunging volume [{}] uploaded using HTTP POST from secondary data store.", volume);
                                        AsyncCallFuture<VolumeApiResult> future = volService.expungeVolumeAsync(volOnSecondary);
                                        VolumeApiResult result = future.get();
                                        if (!result.isSuccess()) {
                                            logger.warn("Failed to expunge volume {} from the image store {} due to: {}", volume, dataStore, result.getResult());
                                        }
                                    }
                                }
                            }
                        } catch (Throwable th) {
                            logger.error("Unable to destroy uploaded volume [{}] due to: [{}].", volume, th.getMessage());
                            logger.debug("Unable to destroy uploaded volume [{}].", volume, th);
                        }
                    }

                    // destroy uploaded templates in abandoned/error state
                    List<TemplateDataStoreVO> templateDataStores = _templateStoreDao.listByTemplateState(VirtualMachineTemplate.State.UploadError, VirtualMachineTemplate.State.UploadAbandoned);
                    for (TemplateDataStoreVO templateDataStore : templateDataStores) {
                        VMTemplateVO template = _templateDao.findById(templateDataStore.getTemplateId());
                        if (template == null) {
                            logger.warn("Uploaded template [{}] not found, so cannot be destroyed.", templateDataStore.getTemplateId());
                            continue;
                        }
                        try {
                            DataStore dataStore = _dataStoreMgr.getDataStore(templateDataStore.getDataStoreId(), DataStoreRole.Image);
                            EndPoint ep = _epSelector.select(dataStore, templateDataStore.getExtractUrl());
                            if (ep == null) {
                                logger.warn("Cannot destroy uploaded template {} as there is no secondary storage VM for image store {}.", template, dataStore);
                                continue;
                            }
                            Host host = _hostDao.findById(ep.getId());
                            if (host != null && host.getManagementServerId() != null) {
                                if (_serverId == host.getManagementServerId()) {
                                    AsyncCallFuture<TemplateApiResult> future = _imageSrv.deleteTemplateAsync(tmplFactory.getTemplate(template.getId(), dataStore));
                                    TemplateApiResult result = future.get();
                                    if (!result.isSuccess()) {
                                        logger.warn("Failed to delete template {} from the image store {} due to: {}", template, dataStore, result.getResult());
                                        continue;
                                    }
                                    // remove from template_zone_ref
                                    List<VMTemplateZoneVO> templateZones = _vmTemplateZoneDao.listByZoneTemplate(((ImageStoreEntity)dataStore).getDataCenterId(), template.getId());
                                    if (templateZones != null) {
                                        for (VMTemplateZoneVO templateZone : templateZones) {
                                            _vmTemplateZoneDao.remove(templateZone.getId());
                                        }
                                    }
                                    // mark all the occurrences of this template in the given store as destroyed
                                    _templateStoreDao.removeByTemplateStore(template.getId(), dataStore.getId());
                                    // find all eligible image stores for this template
                                    List<DataStore> imageStores = _tmpltMgr.getImageStoreByTemplate(template.getId(), null);
                                    if (imageStores == null || imageStores.isEmpty()) {
                                        template.setState(VirtualMachineTemplate.State.Inactive);
                                        _templateDao.update(template.getId(), template);

                                        // decrement template resource count
                                        _resourceLimitMgr.decrementResourceCount(template.getAccountId(), ResourceType.template);
                                    }
                                }
                            }
                        } catch (Throwable th) {
                            logger.error("Unable to destroy uploaded template [{}] due to: [{}].", template, th.getMessage());
                            logger.debug("Unable to destroy uploaded template [{}].", template, th);
                        }
                    }
                    cleanupInactiveTemplates();
                } finally {
                    scanLock.unlock();
                }
            }
        } finally {
            scanLock.releaseRef();
        }
    }

    private void cleanupSnapshotsFromStoreRefInDestroyingState() {
        List<SnapshotDataStoreVO> storeRefSnapshotsInDestroyingState = _snapshotStoreDao.listByState(ObjectInDataStoreStateMachine.State.Destroying);
        for (SnapshotDataStoreVO snapshotDataStoreVO : storeRefSnapshotsInDestroyingState) {
            SnapshotVO snapshot = _snapshotDao.findById(snapshotDataStoreVO.getSnapshotId());
            if (snapshot == null) {
                logger.warn("Did not find snapshot [ID: {}] for which store reference is in destroying state; therefore, it cannot be destroyed.", snapshotDataStoreVO.getSnapshotId());
                continue;
            }
            deleteSnapshot(snapshot, snapshotDataStoreVO);
        }
    }

    private void deleteSnapshot(SnapshotVO snapshot, SnapshotDataStoreVO snapshotDataStoreVO) {
        if (snapshot == null || snapshotDataStoreVO == null) {
            return;
        }

        try {
            final String snapshotUuid = snapshot.getUuid();
            final String storeRole = snapshotDataStoreVO.getRole().toString().toLowerCase();
            logger.debug("Snapshot [{}] is in {} state on {} data store ID: {}.", snapshotUuid, snapshotDataStoreVO.getState(), storeRole, snapshotDataStoreVO.getDataStoreId());
            SnapshotInfo snapshotInfo = snapshotFactory.getSnapshotIncludingRemoved(snapshotDataStoreVO.getSnapshotId(), snapshotDataStoreVO.getDataStoreId(), snapshotDataStoreVO.getRole());
            if (snapshotInfo != null) {
                logger.debug("Snapshot [{}] in {} state found on {} data store [{}], it will be deleted.", snapshotUuid, snapshotDataStoreVO.getState(), storeRole, snapshotInfo.getDataStore().getUuid());
                _snapshotService.deleteSnapshot(snapshotInfo);
            } else {
                logger.debug("Did not find snapshot [{}] in {} state on {} data store ID: {}.", snapshotUuid, snapshotDataStoreVO.getState(), storeRole, snapshotDataStoreVO.getDataStoreId());
            }
        } catch (Exception e) {
            logger.error("Failed to delete snapshot [{}] from storage due to: [{}].", snapshot, e.getMessage(), e);
        }
    }

    private void removeSnapshotsInErrorStatus() {
        List<SnapshotVO> snapshotsInErrorStatus = _snapshotDao.listAllByStatusIncludingRemoved(Snapshot.State.Error);
        for (SnapshotVO snapshotVO : snapshotsInErrorStatus) {
            try {
                List<SnapshotDataStoreVO> storeRefSnapshotsInErrorStatus = _snapshotStoreDao.findBySnapshotId(snapshotVO.getId());
                for (SnapshotDataStoreVO snapshotDataStoreVO : storeRefSnapshotsInErrorStatus) {
                    deleteSnapshot(snapshotVO, snapshotDataStoreVO);
                    _snapshotStoreDao.expunge(snapshotDataStoreVO.getId());
                }
                _snapshotDao.expunge(snapshotVO.getId());
            } catch (Exception e) {
                logger.error("Unable to destroy snapshot [{}] due to: [{}].", snapshotVO, e.getMessage());
                logger.debug("Unable to destroy snapshot [{}].", snapshotVO, e);
            }
        }
    }

    protected boolean isVolumeSuspectedDestroyDuplicateOfVmVolume(VolumeVO gcVolume) {
        if (gcVolume.getPath() == null) {
            return false;
        }
        if (gcVolume.getPoolId() == null) {
            return false;
        }
        Long vmId = gcVolume.getInstanceId();
        if (vmId == null) {
            return false;
        }
        VMInstanceVO vm = _vmInstanceDao.findById(vmId);
        if (vm == null) {
            return false;
        }
        List<VolumeVO> vmUsableVolumes = volumeDao.findUsableVolumesForInstance(vmId);
        for (VolumeVO vol : vmUsableVolumes) {
            if (gcVolume.getPoolId().equals(vol.getPoolId()) && gcVolume.getPath().equals(vol.getPath())) {
                logger.debug("{} meant for garbage collection could a possible duplicate for {}", gcVolume, vol);
                return true;
            }
        }
        return false;
    }

    /**
     * This method only applies for managed storage.
     * For XenServer and vSphere, see if we need to remove an SR or a datastore, then remove the underlying volume
     * from any applicable access control list (before other code attempts to delete the volume that supports it).
     * For KVM, just tell the underlying storage plug-in to remove the volume from any applicable access control list
     * (before other code attempts to delete the volume that supports it).
     */
    private void handleManagedStorage(Volume volume) {
        Long instanceId = volume.getInstanceId();

        if (instanceId != null) {
            StoragePoolVO storagePool = _storagePoolDao.findById(volume.getPoolId());

            if (storagePool != null && storagePool.isManaged()) {
                VMInstanceVO vmInstanceVO = _vmInstanceDao.findById(instanceId);
                if (vmInstanceVO == null) {
                    return;
                }

                Long lastHostId = vmInstanceVO.getLastHostId();

                if (lastHostId != null) {
                    HostVO host = _hostDao.findById(lastHostId);
                    ClusterVO cluster = _clusterDao.findById(host.getClusterId());
                    VolumeInfo volumeInfo = volFactory.getVolume(volume.getId());

                    if (cluster.getHypervisorType() == HypervisorType.KVM) {
                        volService.revokeAccess(volumeInfo, host, volumeInfo.getDataStore());
                    } else {
                        DataTO volTO = volFactory.getVolume(volume.getId()).getTO();
                        DiskTO disk = new DiskTO(volTO, volume.getDeviceId(), volume.getPath(), volume.getVolumeType());

                        DettachCommand cmd = new DettachCommand(disk, null);

                        cmd.setManaged(true);

                        cmd.setStorageHost(storagePool.getHostAddress());
                        cmd.setStoragePort(storagePool.getPort());

                        cmd.set_iScsiName(volume.get_iScsiName());

                        Answer answer = _agentMgr.easySend(lastHostId, cmd);

                        if (answer != null && answer.getResult()) {
                            volService.revokeAccess(volumeInfo, host, volumeInfo.getDataStore());
                        } else {
                            logger.warn("Unable to remove host-side clustered file system for the following volume: {}", volume);
                        }
                    }
                }
            }
        }
    }

    @Override
    @DB
    public void cleanupSecondaryStorage() {
        // NOTE that object_store refactor will immediately delete the object from secondary storage when deleteTemplate etc. api is issued.
        // so here we don't need to issue DeleteCommand to resource anymore, only need to remove db entry.
        try {
            // Cleanup templates in template_store_ref
            List<DataStore> imageStores = _dataStoreMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(null));
            for (DataStore store : imageStores) {
                try {
                    long storeId = store.getId();
                    List<TemplateDataStoreVO> destroyedTemplateStoreVOs = _templateStoreDao.listDestroyed(storeId);
                    logger.debug("Secondary storage garbage collector found {} Templates to cleanup on template_store_ref for store: {}", destroyedTemplateStoreVOs.size(), store);
                    for (TemplateDataStoreVO destroyedTemplateStoreVO : destroyedTemplateStoreVOs) {
                        logger.debug("Deleting template store DB entry: {}", destroyedTemplateStoreVO);
                        _templateStoreDao.remove(destroyedTemplateStoreVO.getId());
                    }
                } catch (Exception e) {
                    logger.warn("Problem cleaning up Templates in template_store_ref for store: {}", store, e);
                }
            }

            // CleanUp snapshots on snapshot_store_ref
            for (DataStore store : imageStores) {
                try {
                    List<SnapshotDataStoreVO> destroyedSnapshotStoreVOs = _snapshotStoreDao.listDestroyed(store.getId());
                    logger.debug("Secondary storage garbage collector found {} snapshots to cleanup on snapshot_store_ref for store: {}", destroyedSnapshotStoreVOs.size(), store);
                    for (SnapshotDataStoreVO destroyedSnapshotStoreVO : destroyedSnapshotStoreVOs) {
                        // check if this snapshot has child
                        SnapshotInfo snap = snapshotFactory.getSnapshot(destroyedSnapshotStoreVO.getSnapshotId(), store);
                        if (snap.getChild() != null) {
                            logger.debug("Skip snapshot on store: {} , because it has child", destroyedSnapshotStoreVO);
                            continue;
                        }

                        logger.debug("Deleting snapshot store DB entry: {}", destroyedSnapshotStoreVO);

                        List<SnapshotDataStoreVO> imageStoreRefs = _snapshotStoreDao.listBySnapshotAndDataStoreRole(destroyedSnapshotStoreVO.getSnapshotId(), DataStoreRole.Image);
                        if (imageStoreRefs.size() <= 1) {
                            _snapshotDao.remove(destroyedSnapshotStoreVO.getSnapshotId());
                        }
                        SnapshotDataStoreVO snapshotOnPrimary = _snapshotStoreDao.findDestroyedReferenceBySnapshot(destroyedSnapshotStoreVO.getSnapshotId(), DataStoreRole.Primary);
                        if (snapshotOnPrimary != null) {
                            logger.debug("Deleting snapshot on primary store reference DB entry: {}", snapshotOnPrimary);
                            _snapshotStoreDao.remove(snapshotOnPrimary.getId());
                        }
                        _snapshotStoreDao.remove(destroyedSnapshotStoreVO.getId());
                    }

                } catch (Exception e2) {
                    logger.warn("problem cleaning up snapshots in snapshot_store_ref for store: {}", store, e2);
                }

            }

            // CleanUp volumes on volume_store_ref
            for (DataStore store : imageStores) {
                try {
                    List<VolumeDataStoreVO> destroyedStoreVOs = _volumeStoreDao.listDestroyed(store.getId());
                    destroyedStoreVOs.addAll(_volumeDataStoreDao.listByVolumeState(Volume.State.Expunged));
                    logger.debug("Secondary storage garbage collector found {} volumes to cleanup on volume_store_ref for store: {}", destroyedStoreVOs.size(), store);
                    for (VolumeDataStoreVO destroyedStoreVO : destroyedStoreVOs) {
                        logger.debug("Deleting volume store DB entry: {}", destroyedStoreVO);
                        _volumeStoreDao.remove(destroyedStoreVO.getId());
                    }

                } catch (Exception e2) {
                    logger.warn("problem cleaning up volumes in volume_store_ref for store: {}", store, e2);
                }
            }
        } catch (Exception e3) {
            logger.warn("problem cleaning up secondary storage DB entries. ", e3);
        }
    }

    @Override
    @DB
    @ActionEvent(eventType = EventTypes.EVENT_MAINTENANCE_PREPARE_PRIMARY_STORAGE,
            eventDescription = "preparing storage pool for maintenance", async = true)
    public PrimaryDataStoreInfo preparePrimaryStorageForMaintenance(Long primaryStorageId) {
        StoragePoolVO primaryStorage;
        primaryStorage = _storagePoolDao.findById(primaryStorageId);

        if (primaryStorage == null) {
            String msg = "Unable to obtain lock on the storage pool record in preparePrimaryStorageForMaintenance()";
            logger.error(msg);
            throw new InvalidParameterValueException(msg);
        }

        if (!primaryStorage.getStatus().equals(StoragePoolStatus.Up) && !primaryStorage.getStatus().equals(StoragePoolStatus.ErrorInMaintenance)) {
            throw new InvalidParameterValueException(String.format("Primary storage %s is not ready for migration, as the status is:%s", primaryStorage, primaryStorage.getStatus().toString()));
        }

        DataStoreProvider provider = _dataStoreProviderMgr.getDataStoreProvider(primaryStorage.getStorageProviderName());
        DataStoreLifeCycle lifeCycle = provider.getDataStoreLifeCycle();
        DataStore store = _dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary);

        if (primaryStorage.getPoolType() == StoragePoolType.DatastoreCluster) {
            if (primaryStorage.getStatus() == StoragePoolStatus.PrepareForMaintenance) {
                throw new CloudRuntimeException(String.format("There is already a job running for preparation for maintenance of the storage pool %s", primaryStorage));
            }
            handlePrepareDatastoreClusterMaintenance(lifeCycle, primaryStorageId);
        }
        lifeCycle.maintain(store);

        return (PrimaryDataStoreInfo)_dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary);
    }

    private void handlePrepareDatastoreClusterMaintenance(DataStoreLifeCycle lifeCycle, Long primaryStorageId) {
        StoragePoolVO datastoreCluster = _storagePoolDao.findById(primaryStorageId);
        datastoreCluster.setStatus(StoragePoolStatus.PrepareForMaintenance);
        _storagePoolDao.update(datastoreCluster.getId(), datastoreCluster);

        // Before preparing the datastorecluster to maintenance mode, the storagepools in the datastore cluster needs to put in maintenance
        List<StoragePoolVO> childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(primaryStorageId);
        Transaction.execute(new TransactionCallbackNoReturn() {
            @Override
            public void doInTransactionWithoutResult(TransactionStatus status) {
                for (StoragePoolVO childDatastore : childDatastores) {
                    // set the pool state to prepare for maintenance, so that VMs will not migrate to the storagepools in the same cluster
                    childDatastore.setStatus(StoragePoolStatus.PrepareForMaintenance);
                    _storagePoolDao.update(childDatastore.getId(), childDatastore);
                }
            }
        });
        for (StoragePoolVO datastore : childDatastores) {
            DataStore childStore = _dataStoreMgr.getDataStore(datastore.getId(), DataStoreRole.Primary);
            try {
                lifeCycle.maintain(childStore);
            } catch (Exception e) {
                logger.debug("Exception on maintenance preparation of one of the child datastores in datastore cluster {}", datastoreCluster, e);
                // Set to ErrorInMaintenance state of all child storage pools and datastore cluster
                for (StoragePoolVO childDatastore : childDatastores) {
                    childDatastore.setStatus(StoragePoolStatus.ErrorInMaintenance);
                    _storagePoolDao.update(childDatastore.getId(), childDatastore);
                }
                datastoreCluster.setStatus(StoragePoolStatus.ErrorInMaintenance);
                _storagePoolDao.update(datastoreCluster.getId(), datastoreCluster);
                throw new CloudRuntimeException(String.format("Failed to prepare maintenance mode for datastore cluster %s with error %s %s", datastoreCluster, e.getMessage(), e));
            }
        }
    }

    @Override
    @DB
    @ActionEvent(eventType = EventTypes.EVENT_MAINTENANCE_CANCEL_PRIMARY_STORAGE,
            eventDescription = "canceling maintenance for primary storage pool", async = true)
    public PrimaryDataStoreInfo cancelPrimaryStorageForMaintenance(CancelPrimaryStorageMaintenanceCmd cmd) throws ResourceUnavailableException {
        Long primaryStorageId = cmd.getId();
        StoragePoolVO primaryStorage;

        primaryStorage = _storagePoolDao.findById(primaryStorageId);

        if (primaryStorage == null) {
            String msg = "Unable to obtain lock on the storage pool in cancelPrimaryStorageForMaintenance()";
            logger.error(msg);
            throw new InvalidParameterValueException(msg);
        }

        if (primaryStorage.getStatus().equals(StoragePoolStatus.Up) || primaryStorage.getStatus().equals(StoragePoolStatus.PrepareForMaintenance)) {
            throw new StorageUnavailableException("Primary storage " + primaryStorage + " is not ready to complete migration, as the status is:" + primaryStorage.getStatus().toString(),
                    primaryStorageId);
        }

        DataStoreProvider provider = _dataStoreProviderMgr.getDataStoreProvider(primaryStorage.getStorageProviderName());
        DataStoreLifeCycle lifeCycle = provider.getDataStoreLifeCycle();
        DataStore store = _dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary);
        if (primaryStorage.getPoolType() == StoragePoolType.DatastoreCluster) {
            primaryStorage.setStatus(StoragePoolStatus.Up);
            _storagePoolDao.update(primaryStorage.getId(), primaryStorage);
            //FR41 need to handle when one of the primary stores is unable to cancel the maintenance mode
            List<StoragePoolVO> childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(primaryStorageId);
            for (StoragePoolVO childDatastore : childDatastores) {
                DataStore childStore = _dataStoreMgr.getDataStore(childDatastore.getId(), DataStoreRole.Primary);
                lifeCycle.cancelMaintain(childStore);
            }
        }
        lifeCycle.cancelMaintain(store);

        return (PrimaryDataStoreInfo)_dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary);
    }

    @Override
    @ActionEvent(eventType = EventTypes.EVENT_SYNC_STORAGE_POOL, eventDescription = "synchronising storage pool with management server", async = true)
    public StoragePool syncStoragePool(SyncStoragePoolCmd cmd) {
        Long poolId = cmd.getPoolId();
        StoragePoolVO pool = _storagePoolDao.findById(poolId);

        if (pool == null) {
            String msg = String.format("Unable to find the storage pool with id %d record while syncing storage pool with management server", poolId);
            logger.error(msg);
            throw new InvalidParameterValueException(msg);
        }

        if (!pool.getPoolType().equals(StoragePoolType.DatastoreCluster)) {
            throw new InvalidParameterValueException("SyncStoragePool API is currently supported only for storage type of datastore cluster");
        }

        if (!pool.getStatus().equals(StoragePoolStatus.Up)) {
            throw new InvalidParameterValueException(String.format("Primary storage %s is not ready for syncing, as the status is %s", pool, pool.getStatus().toString()));
        }

        // find the host
        List<Long> poolIds = new ArrayList<>();
        poolIds.add(poolId);
        List<Long> hosts = _storagePoolHostDao.findHostsConnectedToPools(poolIds);
        if (!hosts.isEmpty()) {
            Long hostId = hosts.get(0);
            ModifyStoragePoolCommand modifyStoragePoolCommand = new ModifyStoragePoolCommand(true, pool);
            final Answer answer = _agentMgr.easySend(hostId, modifyStoragePoolCommand);

            if (answer == null) {
                throw new CloudRuntimeException(String.format("Unable to get an answer to the modify storage pool command %s", pool));
            }

            if (!answer.getResult()) {
                throw new CloudRuntimeException(String.format("Unable to process ModifyStoragePoolCommand for pool %s on the host %s due to %s", pool, _hostDao.findById(hostId), answer.getDetails()));
            }

            assert (answer instanceof ModifyStoragePoolAnswer) : "Well, now why won't you actually return the ModifyStoragePoolAnswer when it's ModifyStoragePoolCommand? Pool=" +
                    pool.getId() + "Host=" + hostId;
            ModifyStoragePoolAnswer mspAnswer = (ModifyStoragePoolAnswer) answer;
            StoragePoolVO poolVO = _storagePoolDao.findById(poolId);
            updateStoragePoolHostVOAndBytes(poolVO, hostId, mspAnswer);
            validateChildDatastoresToBeAddedInUpState(poolVO, mspAnswer.getDatastoreClusterChildren());
            syncDatastoreClusterStoragePool(poolId, mspAnswer.getDatastoreClusterChildren(), hostId);
            for (ModifyStoragePoolAnswer childDataStoreAnswer : mspAnswer.getDatastoreClusterChildren()) {
                StoragePoolInfo childStoragePoolInfo = childDataStoreAnswer.getPoolInfo();
                StoragePoolVO dataStoreVO = _storagePoolDao.findPoolByUUID(childStoragePoolInfo.getUuid());
                for (Long host : hosts) {
                    updateStoragePoolHostVOAndBytes(dataStoreVO, host, childDataStoreAnswer);
                }
            }

        } else {
            throw new CloudRuntimeException(String.format("Unable to sync storage pool [%s] as there no connected hosts to the storage pool", pool));
        }
        return (PrimaryDataStoreInfo) _dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary);
    }


    @Override
    public Heuristic createSecondaryStorageHeuristic(CreateSecondaryStorageSelectorCmd cmd) {
        String name = cmd.getName();
        String description = cmd.getDescription();
        long zoneId = cmd.getZoneId();
        String heuristicRule = cmd.getHeuristicRule();
        String type = cmd.getType();
        HeuristicType formattedType = EnumUtils.getEnumIgnoreCase(HeuristicType.class, type);

        if (formattedType == null) {
            throw new IllegalArgumentException(String.format("The given heuristic type [%s] is not valid for creating a new secondary storage selector." +
                    " The valid options are %s.", type, Arrays.asList(HeuristicType.values())));
        }

        HeuristicVO heuristic = secondaryStorageHeuristicDao.findByZoneIdAndType(zoneId, formattedType);

        if (heuristic != null) {
            DataCenterVO dataCenter = _dcDao.findById(zoneId);
            throw new CloudRuntimeException(String.format("There is already a heuristic rule in the specified %s with the type [%s].",
                    dataCenter, type));
        }

        validateHeuristicRule(heuristicRule);

        HeuristicVO heuristicVO = new HeuristicVO(name, description, zoneId, formattedType.toString(), heuristicRule);
        return secondaryStorageHeuristicDao.persist(heuristicVO);
    }

    @Override
    public Heuristic updateSecondaryStorageHeuristic(UpdateSecondaryStorageSelectorCmd cmd) {
        long heuristicId = cmd.getId();
        String heuristicRule = cmd.getHeuristicRule();

        HeuristicVO heuristicVO = secondaryStorageHeuristicDao.findById(heuristicId);
        validateHeuristicRule(heuristicRule);
        heuristicVO.setHeuristicRule(heuristicRule);

        return secondaryStorageHeuristicDao.persist(heuristicVO);
    }

    @Override
    public void removeSecondaryStorageHeuristic(RemoveSecondaryStorageSelectorCmd cmd) {
        Long heuristicId = cmd.getId();
        HeuristicVO heuristicVO = secondaryStorageHeuristicDao.findById(heuristicId);

        if (heuristicVO != null) {
            secondaryStorageHeuristicDao.remove(heuristicId);
        } else {
            throw new CloudRuntimeException("Unable to find an active heuristic with the specified UUID.");
        }
    }

    protected void validateHeuristicRule(String heuristicRule) {
        if (StringUtils.isBlank(heuristicRule)) {
            throw new IllegalArgumentException("Unable to create a new secondary storage selector as the given heuristic rule is blank.");
        }
        jsInterpreterHelper.ensureInterpreterEnabledIfParameterProvided(ApiConstants.HEURISTIC_RULE, true);
    }

    public void syncDatastoreClusterStoragePool(long datastoreClusterPoolId, List<ModifyStoragePoolAnswer> childDatastoreAnswerList, long hostId) {
        StoragePoolVO datastoreClusterPool = _storagePoolDao.findById(datastoreClusterPoolId);
        List<StoragePoolTagVO> storageTags = _storagePoolTagsDao.findStoragePoolTags(datastoreClusterPoolId);
        List<StoragePoolVO> childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(datastoreClusterPoolId);
        Set<String> childDatastoreUUIDs = new HashSet<>();
        for (StoragePoolVO childDatastore : childDatastores) {
            childDatastoreUUIDs.add(childDatastore.getUuid());
        }

        for (ModifyStoragePoolAnswer childDataStoreAnswer : childDatastoreAnswerList) {
            StoragePoolInfo childStoragePoolInfo = childDataStoreAnswer.getPoolInfo();
            StoragePoolVO dataStoreVO = getExistingPoolByUuid(childStoragePoolInfo.getUuid());
            if (dataStoreVO == null && childDataStoreAnswer.getPoolType().equalsIgnoreCase("NFS")) {
                List<StoragePoolVO> nfsStoragePools = _storagePoolDao.findPoolsByStorageType(StoragePoolType.NetworkFilesystem);
                for (StoragePoolVO storagePool : nfsStoragePools) {
                    String storagePoolUUID = storagePool.getUuid();
                    if (childStoragePoolInfo.getName().equalsIgnoreCase(storagePoolUUID.replaceAll("-", ""))) {
                        dataStoreVO = storagePool;
                        break;
                    }
                }
            }
            if (dataStoreVO != null) {
                if (dataStoreVO.getParent() != datastoreClusterPoolId) {
                    logger.debug("Storage pool {} with uuid {} is found to be under datastore cluster {} at vCenter, " +
                                    "so moving the storage pool to be a child storage pool under the datastore cluster in CloudStack management server",
                            childStoragePoolInfo.getName(), childStoragePoolInfo.getUuid(), datastoreClusterPool.getName());
                    dataStoreVO.setParent(datastoreClusterPoolId);
                    _storagePoolDao.update(dataStoreVO.getId(), dataStoreVO);
                    if (CollectionUtils.isNotEmpty(storageTags)) {
                        storageTags.addAll(_storagePoolTagsDao.findStoragePoolTags(dataStoreVO.getId()));
                    } else {
                        storageTags = _storagePoolTagsDao.findStoragePoolTags(dataStoreVO.getId());
                    }
                    if (CollectionUtils.isNotEmpty(storageTags)) {
                        Set<StoragePoolTagVO> set = new LinkedHashSet<>(storageTags);
                        storageTags.clear();
                        storageTags.addAll(set);
                        logger.debug("Updating Storage Pool Tags to : {}", storageTags);
                        _storagePoolTagsDao.persist(storageTags);
                    }
                } else {
                    // This is to find datastores which are removed from datastore cluster.
                    // The final set childDatastoreUUIDs contains the UUIDs of child datastores which needs to be removed from datastore cluster
                    childDatastoreUUIDs.remove(dataStoreVO.getUuid());
                }
            } else {
                dataStoreVO = createChildDatastoreVO(datastoreClusterPool, childDataStoreAnswer, storageTags);
            }
            updateStoragePoolHostVOAndBytes(dataStoreVO, hostId, childDataStoreAnswer);
        }

        handleRemoveChildStoragePoolFromDatastoreCluster(childDatastoreUUIDs);
    }

    private StoragePoolVO getExistingPoolByUuid(String uuid){
        if(!uuid.contains("-")){
            UUID poolUuid = new UUID(
                new BigInteger(uuid.substring(0, 16), 16).longValue(),
                new BigInteger(uuid.substring(16), 16).longValue()
            );
            uuid = poolUuid.toString();
        }
        return _storagePoolDao.findByUuid(uuid);
    }

    public void validateChildDatastoresToBeAddedInUpState(StoragePoolVO datastoreClusterPool, List<ModifyStoragePoolAnswer> childDatastoreAnswerList) {
        for (ModifyStoragePoolAnswer childDataStoreAnswer : childDatastoreAnswerList) {
            StoragePoolInfo childStoragePoolInfo = childDataStoreAnswer.getPoolInfo();
            StoragePoolVO dataStoreVO = _storagePoolDao.findPoolByUUID(childStoragePoolInfo.getUuid());
            if (dataStoreVO == null && childDataStoreAnswer.getPoolType().equalsIgnoreCase("NFS")) {
                List<StoragePoolVO> nfsStoragePools = _storagePoolDao.findPoolsByStorageType(StoragePoolType.NetworkFilesystem);
                for (StoragePoolVO storagePool : nfsStoragePools) {
                    String storagePoolUUID = storagePool.getUuid();
                    if (childStoragePoolInfo.getName().equalsIgnoreCase(storagePoolUUID.replaceAll("-", ""))) {
                          dataStoreVO = storagePool;
                          break;
                    }
                }
            }
            if (dataStoreVO != null && !dataStoreVO.getStatus().equals(StoragePoolStatus.Up)) {
                String msg = String.format("Cannot synchronise datastore cluster %s because primary storage %s is not in Up state, " +
                        "current state is %s", datastoreClusterPool, dataStoreVO, dataStoreVO.getStatus().toString());
                throw new CloudRuntimeException(msg);
            }
        }
    }

    private StoragePoolVO createChildDatastoreVO(StoragePoolVO datastoreClusterPool, ModifyStoragePoolAnswer childDataStoreAnswer, List<StoragePoolTagVO> storagePoolTagVOList) {
        StoragePoolInfo childStoragePoolInfo = childDataStoreAnswer.getPoolInfo();

        StoragePoolVO dataStoreVO = new StoragePoolVO();
        dataStoreVO.setStorageProviderName(datastoreClusterPool.getStorageProviderName());
        dataStoreVO.setHostAddress(childStoragePoolInfo.getHost());
        dataStoreVO.setPoolType(Storage.StoragePoolType.PreSetup);
        dataStoreVO.setPath(childStoragePoolInfo.getHostPath());
        dataStoreVO.setPort(datastoreClusterPool.getPort());
        dataStoreVO.setName(childStoragePoolInfo.getName());
        dataStoreVO.setUuid(childStoragePoolInfo.getUuid());
        dataStoreVO.setDataCenterId(datastoreClusterPool.getDataCenterId());
        dataStoreVO.setPodId(datastoreClusterPool.getPodId());
        dataStoreVO.setClusterId(datastoreClusterPool.getClusterId());
        dataStoreVO.setStatus(StoragePoolStatus.Up);
        dataStoreVO.setUserInfo(datastoreClusterPool.getUserInfo());
        dataStoreVO.setManaged(datastoreClusterPool.isManaged());
        dataStoreVO.setCapacityIops(datastoreClusterPool.getCapacityIops());
        dataStoreVO.setCapacityBytes(childDataStoreAnswer.getPoolInfo().getCapacityBytes());
        dataStoreVO.setUsedBytes(childDataStoreAnswer.getPoolInfo().getCapacityBytes() - childDataStoreAnswer.getPoolInfo().getAvailableBytes());
        dataStoreVO.setHypervisor(datastoreClusterPool.getHypervisor());
        dataStoreVO.setScope(datastoreClusterPool.getScope());
        dataStoreVO.setParent(datastoreClusterPool.getId());

        Map<String, String> details = new HashMap<>();
        if(StringUtils.isNotEmpty(childDataStoreAnswer.getPoolType())) {
            details.put("pool_type", childDataStoreAnswer.getPoolType());
        }

        List<String> storagePoolTags = new ArrayList<>();
        boolean isTagARule = false;
        if (CollectionUtils.isNotEmpty(storagePoolTagVOList)) {
            storagePoolTags = storagePoolTagVOList.parallelStream().map(StoragePoolTagVO::getTag).collect(Collectors.toList());
            isTagARule = storagePoolTagVOList.get(0).isTagARule();
        }
        List<String> storageAccessGroups = _storagePoolAccessGroupMapDao.getStorageAccessGroups(datastoreClusterPool.getId());

        _storagePoolDao.persist(dataStoreVO, details, storagePoolTags, isTagARule, storageAccessGroups);
        return dataStoreVO;
    }

    @Override
    public boolean checkIfHostAndStoragePoolHasCommonStorageAccessGroups(Host host, StoragePool pool) {
        String[] hostStorageAccessGroups = getStorageAccessGroups(null, null, null, host.getId());
        List<String> storagePoolAccessGroups = _storagePoolAccessGroupMapDao.getStorageAccessGroups(pool.getId());

        if (CollectionUtils.isEmpty(storagePoolAccessGroups)) {
            return true;
        }

        if (ArrayUtils.isEmpty(hostStorageAccessGroups)) {
            return false;
        }

        if (ArrayUtils.isNotEmpty(hostStorageAccessGroups)) {
            logger.debug("Storage access groups on the host {} are {}", host, hostStorageAccessGroups);
        }

        if (CollectionUtils.isNotEmpty(storagePoolAccessGroups)) {
            logger.debug("Storage access groups on the storage pool {} are {}", host, storagePoolAccessGroups);
        }

        List<String> hostTagList = Arrays.asList(hostStorageAccessGroups);
        return CollectionUtils.containsAny(hostTagList, storagePoolAccessGroups);
    }

    @Override
    public Pair<Boolean, String> checkIfReadyVolumeFitsInStoragePoolWithStorageAccessGroups(StoragePool destPool, Volume volume) {
        if (Volume.State.Ready.equals(volume.getState())) {
            Long vmId = volume.getInstanceId();
            VMInstanceVO vm = null;
            if (vmId != null) {
                vm = _vmInstanceDao.findById(vmId);
            }

            if (vm == null || State.Stopped.equals(vm.getState())) {
                Long srcPoolId = volume.getPoolId();
                StoragePoolVO srcPool = _storagePoolDao.findById(srcPoolId);
                List<String> srcStorageAccessGroups = _storagePoolAccessGroupMapDao.getStorageAccessGroups(srcPoolId);
                List<String> destStorageAccessGroups = _storagePoolAccessGroupMapDao.getStorageAccessGroups(destPool.getId());

                if (CollectionUtils.isNotEmpty(srcStorageAccessGroups) && CollectionUtils.isNotEmpty(destStorageAccessGroups)) {
                    logger.debug("Storage access groups on source storage {} are {} and destination storage {} are {}",
                            srcPool, srcStorageAccessGroups, destPool, destStorageAccessGroups);
                    List<String> intersection = new ArrayList<>(srcStorageAccessGroups);
                    intersection.retainAll(destStorageAccessGroups);
                    if (CollectionUtils.isNotEmpty(intersection)) {
                        return new Pair<>(true, "Success");
                    } else {
                        List<Long> poolIds = new ArrayList<>();
                        poolIds.add(srcPool.getId());
                        poolIds.add(destPool.getId());
                        Host hostWithPoolsAccess = findUpAndEnabledHostWithAccessToStoragePools(poolIds);
                        if (hostWithPoolsAccess == null) {
                            logger.debug("Storage access groups on source and destination storages do not match, and there is no common host connected to these storages");
                            return new Pair<>(false, "No common host connected to source and destination storages");
                        }
                    }
                }
                return new Pair<>(true, "Success");
            } else {
                if (State.Running.equals(vm.getState())) {
                    Long hostId = vm.getHostId();
                    String[] hostStorageAccessGroups = getStorageAccessGroups(null, null, null, hostId);
                    Long srcPoolId = volume.getPoolId();
                    StoragePoolVO srcPool = _storagePoolDao.findById(srcPoolId);
                    List<String> srcStorageAccessGroups = _storagePoolAccessGroupMapDao.getStorageAccessGroups(srcPoolId);
                    List<String> destStorageAccessGroups = _storagePoolAccessGroupMapDao.getStorageAccessGroups(destPool.getId());

                    logger.debug("Storage access groups on source storage {} are {} and destination storage {} are {}",
                            srcPool, srcStorageAccessGroups, destPool, destStorageAccessGroups);

                    if (CollectionUtils.isEmpty(srcStorageAccessGroups) && CollectionUtils.isEmpty(destStorageAccessGroups)) {
                        return new Pair<>(true, "Success");
                    }

                    if (CollectionUtils.isNotEmpty(srcStorageAccessGroups) && CollectionUtils.isNotEmpty(destStorageAccessGroups)) {

                        if (ArrayUtils.isNotEmpty(hostStorageAccessGroups)) {
                            boolean hasSrcCommon = srcStorageAccessGroups.stream()
                                    .anyMatch(group -> Arrays.asList(hostStorageAccessGroups).contains(group));
                            boolean hasDestCommon = destStorageAccessGroups.stream()
                                    .anyMatch(group -> Arrays.asList(hostStorageAccessGroups).contains(group));
                            if (hasSrcCommon && hasDestCommon) {
                                return new Pair<>(true, "Success");
                            }
                        }

                        return new Pair<>(false, "No common storage access groups between source, destination pools and host");
                    }

                    if (CollectionUtils.isEmpty(srcStorageAccessGroups)) {
                        if (ArrayUtils.isNotEmpty(hostStorageAccessGroups)) {
                            List<String> hostAccessGroupList = Arrays.asList(hostStorageAccessGroups);
                            hostAccessGroupList.retainAll(destStorageAccessGroups);
                            if (CollectionUtils.isNotEmpty(hostAccessGroupList)) {
                                return new Pair<>(true, "Success");
                            }
                        }
                        return new Pair<>(false, "Host lacks access to destination storage groups");
                    }

                    return new Pair<>(true, "Success");
                }
            }
        }
        return new Pair<>(true, "Success");
    }

    @Override
    public String[] getStorageAccessGroups(Long zoneId, Long podId, Long clusterId, Long hostId) {
        List<String> storageAccessGroups = new ArrayList<>();
        if (hostId != null) {
            HostVO host = _hostDao.findById(hostId);
            ClusterVO cluster = _clusterDao.findById(host.getClusterId());
            HostPodVO pod = _podDao.findById(cluster.getPodId());
            DataCenterVO zone = _dcDao.findById(pod.getDataCenterId());
            storageAccessGroups.addAll(List.of(com.cloud.utils.StringUtils.splitCommaSeparatedStrings(host.getStorageAccessGroups(), cluster.getStorageAccessGroups(), pod.getStorageAccessGroups(), zone.getStorageAccessGroups())));
        } else if (clusterId != null) {
            ClusterVO cluster = _clusterDao.findById(clusterId);
            HostPodVO pod = _podDao.findById(cluster.getPodId());
            DataCenterVO zone = _dcDao.findById(pod.getDataCenterId());
            storageAccessGroups.addAll(List.of(com.cloud.utils.StringUtils.splitCommaSeparatedStrings(cluster.getStorageAccessGroups(), pod.getStorageAccessGroups(), zone.getStorageAccessGroups())));
        } else if (podId != null) {
            HostPodVO pod = _podDao.findById(podId);
            DataCenterVO zone = _dcDao.findById(pod.getDataCenterId());
            storageAccessGroups.addAll(List.of(com.cloud.utils.StringUtils.splitCommaSeparatedStrings(pod.getStorageAccessGroups(), zone.getStorageAccessGroups())));
        } else if (zoneId != null) {
            DataCenterVO zone = _dcDao.findById(zoneId);
            storageAccessGroups.addAll(List.of(com.cloud.utils.StringUtils.splitCommaSeparatedStrings(zone.getStorageAccessGroups())));
        }

        storageAccessGroups.removeIf(tag -> tag == null || tag.trim().isEmpty());

        return storageAccessGroups.isEmpty()
                ? new String[0]
                : storageAccessGroups.toArray(org.apache.commons.lang.ArrayUtils.EMPTY_STRING_ARRAY);
    }

    private void handleRemoveChildStoragePoolFromDatastoreCluster(Set<String> childDatastoreUUIDs) {

        for (String childDatastoreUUID : childDatastoreUUIDs) {
            StoragePoolVO dataStoreVO = _storagePoolDao.findPoolByUUID(childDatastoreUUID);
            List<VolumeVO> allVolumes = volumeDao.findNonDestroyedVolumesByPoolId(dataStoreVO.getId());
            allVolumes.removeIf(volumeVO -> volumeVO.getInstanceId() == null);
            allVolumes.removeIf(volumeVO -> volumeVO.getState() != Volume.State.Ready);
            for (VolumeVO volume : allVolumes) {
                VMInstanceVO vmInstance = _vmInstanceDao.findById(volume.getInstanceId());
                if (vmInstance == null) {
                    continue;
                }
                long volumeId = volume.getId();
                Long hostId = vmInstance.getHostId();
                if (hostId == null) {
                    hostId = vmInstance.getLastHostId();
                }
                HostVO hostVO = _hostDao.findById(hostId);

                // Prepare for the syncvolumepath command
                DataTO volTO = volFactory.getVolume(volume.getId()).getTO();
                DiskTO disk = new DiskTO(volTO, volume.getDeviceId(), volume.getPath(), volume.getVolumeType());
                Map<String, String> details = new HashMap<>();
                details.put(DiskTO.PROTOCOL_TYPE, Storage.StoragePoolType.DatastoreCluster.toString());
                disk.setDetails(details);

                logger.debug("Attempting to process SyncVolumePathCommand for the volume {} on the host {} with state {}", volume, hostVO, hostVO.getResourceState());
                SyncVolumePathCommand cmd = new SyncVolumePathCommand(disk);
                final Answer answer = _agentMgr.easySend(hostId, cmd);
                // validate answer
                if (answer == null) {
                    throw new CloudRuntimeException(String.format("Unable to get an answer to the SyncVolumePath command for volume %s", volume));
                }
                if (!answer.getResult()) {
                    throw new CloudRuntimeException(String.format("Unable to process SyncVolumePathCommand for the volume %s to the host %s due to %s", volume, hostVO, answer.getDetails()));
                }
                assert (answer instanceof SyncVolumePathAnswer) : String.format("Well, now why won't you actually return the SyncVolumePathAnswer when it's SyncVolumePathCommand? volume=%s Host=%s", volume, hostVO);

                // check for the changed details of volume and update database
                VolumeVO volumeVO = volumeDao.findById(volumeId);
                String datastoreName = answer.getContextParam("datastoreName");
                if (datastoreName != null) {
                    StoragePoolVO storagePoolVO = _storagePoolDao.findByUuid(datastoreName);
                    if (storagePoolVO != null) {
                        volumeVO.setPoolId(storagePoolVO.getId());
                        volumeVO.setPoolType(storagePoolVO.getPoolType());
                    } else {
                        logger.warn("Unable to find datastore {} while updating the new datastore of the volume {}", datastoreName, volumeVO);
                    }
                }

                String volumePath = answer.getContextParam("volumePath");
                if (volumePath != null) {
                    volumeVO.setPath(volumePath);
                }

                String chainInfo = answer.getContextParam("chainInfo");
                if (chainInfo != null) {
                    volumeVO.setChainInfo(chainInfo);
                }

                volumeDao.update(volumeVO.getId(), volumeVO);
            }
            dataStoreVO.setParent(0L);
            _storagePoolDao.update(dataStoreVO.getId(), dataStoreVO);
        }

    }

    @Override
    public void updateStoragePoolHostVOAndBytes(StoragePool pool, long hostId, ModifyStoragePoolAnswer mspAnswer) {
        StoragePoolHostVO poolHost = _storagePoolHostDao.findByPoolHost(pool.getId(), hostId);
        if (poolHost == null) {
            poolHost = new StoragePoolHostVO(pool.getId(), hostId, mspAnswer.getPoolInfo().getLocalPath().replaceAll("//", "/"));
            _storagePoolHostDao.persist(poolHost);
        } else {
            poolHost.setLocalPath(mspAnswer.getPoolInfo().getLocalPath().replaceAll("//", "/"));
        }

        StoragePoolVO poolVO = _storagePoolDao.findById(pool.getId());
        if (!Storage.StoragePoolType.StorPool.equals(poolVO.getPoolType()) && !DataStoreProvider.ONTAP_PLUGIN_NAME.equals(poolVO.getStorageProviderName())) {
            poolVO.setUsedBytes(mspAnswer.getPoolInfo().getCapacityBytes() - mspAnswer.getPoolInfo().getAvailableBytes());
            poolVO.setCapacityBytes(mspAnswer.getPoolInfo().getCapacityBytes());
        }

        _storagePoolDao.update(pool.getId(), poolVO);
    }

    protected class StorageGarbageCollector extends ManagedContextRunnable {

        public StorageGarbageCollector() {
        }

        @Override
        protected void runInContext() {
            try {
                logger.trace("Storage Garbage Collection Thread is running.");

                cleanupStorage();

            } catch (Exception e) {
                logger.error("Caught the following Exception", e);
            }
        }
    }

    @Override
    public void onManagementNodeJoined(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
    }

    @Override
    public void onManagementNodeLeft(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
        for (ManagementServerHost vo : nodeList) {
            if (vo.getMsid() == _serverId) {
                logger.info("Cleaning up storage maintenance jobs associated with Management server: {}", vo);
                List<Long> poolIds = _storagePoolWorkDao.searchForPoolIdsForPendingWorkJobs(vo.getMsid());
                if (!poolIds.isEmpty()) {
                    for (Long poolId : poolIds) {
                        StoragePoolVO pool = _storagePoolDao.findById(poolId);
                        // check if pool is in an inconsistent state
                        if (pool != null && (pool.getStatus().equals(StoragePoolStatus.ErrorInMaintenance) || pool.getStatus().equals(StoragePoolStatus.PrepareForMaintenance)
                                || pool.getStatus().equals(StoragePoolStatus.CancelMaintenance))) {
                            _storagePoolWorkDao.removePendingJobsOnMsRestart(vo.getMsid(), poolId);
                            pool.setStatus(StoragePoolStatus.ErrorInMaintenance);
                            _storagePoolDao.update(poolId, pool);
                        }

                    }
                }
            }
        }
    }

    @Override
    public void onManagementNodeIsolated() {
    }

    @Override
    public CapacityVO getSecondaryStorageUsedStats(Long hostId, Long zoneId) {
        List<Long> hosts = new ArrayList<>();
        if (hostId != null) {
            hosts.add(hostId);
        } else {
            List<DataStore> stores = _dataStoreMgr.getImageStoresByScope(new ZoneScope(zoneId));
            if (stores != null) {
                for (DataStore store : stores) {
                    hosts.add(store.getId());
                }
            }
        }

        CapacityVO capacity = new CapacityVO(hostId, zoneId, null, null, 0, 0, Capacity.CAPACITY_TYPE_SECONDARY_STORAGE);
        for (Long id : hosts) {
            StorageStats stats = ApiDBUtils.getSecondaryStorageStatistics(id);
            if (stats == null) {
                continue;
            }
            capacity.setUsedCapacity(stats.getByteUsed() + capacity.getUsedCapacity());
            capacity.setTotalCapacity(stats.getCapacityBytes() + capacity.getTotalCapacity());
        }

        return capacity;
    }

    private CapacityVO getStoragePoolUsedStatsInternal(Long zoneId, Long podId, Long clusterId, List<Long> poolIds, Long poolId) {
        List<StoragePoolVO> pools = new ArrayList<>();
        if (poolId != null) {
            pools.add(_storagePoolDao.findById(poolId));
        } else {
            SearchCriteria<StoragePoolVO> sc = PoolUsedStatsSearch.create();
            if (zoneId != null) {
                sc.setParameters("dataCenterId", zoneId);
            }
            if (podId != null) {
                sc.setParameters("podId", podId);
            }
            if (clusterId != null) {
                sc.setParameters("clusterId", clusterId);
            }
            if (CollectionUtils.isNotEmpty(poolIds)) {
                sc.setParameters("ids", poolIds.toArray());
            }
            sc.setParameters("parent", 0L);
            pools = _storagePoolDao.search(sc, null);
        }

        CapacityVO capacity = new CapacityVO(poolId, zoneId, podId, clusterId, 0, 0, Capacity.CAPACITY_TYPE_STORAGE);
        for (StoragePoolVO primaryDataStoreVO : pools) {
            StorageStats stats = ApiDBUtils.getStoragePoolStatistics(primaryDataStoreVO.getId());
            if (stats == null) {
                continue;
            }
            capacity.setUsedCapacity(stats.getByteUsed() + capacity.getUsedCapacity());
            capacity.setTotalCapacity(stats.getCapacityBytes() + capacity.getTotalCapacity());
        }
        return capacity;

    }

    @Override
    public CapacityVO getStoragePoolUsedStats(Long poolId, Long clusterId, Long podId, Long zoneId) {
        return getStoragePoolUsedStatsInternal(zoneId, podId, clusterId, null, poolId);
    }

    @Override
    public CapacityVO getStoragePoolUsedStats(Long zoneId, Long podId, Long clusterId, List<Long> poolIds) {
        return getStoragePoolUsedStatsInternal(zoneId, podId, clusterId, poolIds, null);
    }

    @Override
    public PrimaryDataStoreInfo getStoragePool(long id) {
        return (PrimaryDataStoreInfo)_dataStoreMgr.getDataStore(id, DataStoreRole.Primary);
    }

    @Override
    @DB
    public List<VMInstanceVO> listByStoragePool(long storagePoolId) {
        SearchCriteria<VMInstanceVO> sc = StoragePoolSearch.create();
        sc.setJoinParameters("vmVolume", "volumeType", Volume.Type.ROOT);
        sc.setJoinParameters("vmVolume", "poolId", storagePoolId);
        sc.setJoinParameters("vmVolume", "state", Volume.State.Ready);
        return _vmInstanceDao.search(sc, null);
    }

    @Override
    @DB
    public StoragePoolVO findLocalStorageOnHost(long hostId) {
        SearchCriteria<StoragePoolVO> sc = LocalStorageSearch.create();
        sc.setParameters("type", StoragePoolType.Filesystem, StoragePoolType.LVM);
        sc.setJoinParameters("poolHost", "hostId", hostId);
        List<StoragePoolVO> storagePools = _storagePoolDao.search(sc, null);
        if (!storagePools.isEmpty()) {
            return storagePools.get(0);
        } else {
            return null;
        }
    }

    @Override
    public Host findUpAndEnabledHostWithAccessToStoragePools(List<Long> poolIds) {
        List<Long> hostIds = _storagePoolHostDao.findHostsConnectedToPools(poolIds);
        if (hostIds.isEmpty()) {
            return null;
        }
        Collections.shuffle(hostIds);

        for (Long hostId : hostIds) {
            Host host = _hostDao.findById(hostId);
            if (canHostAccessStoragePools(host, poolIds)) {
                return host;
            }
        }

        return null;
    }

    private boolean canHostAccessStoragePools(Host host, List<Long> poolIds) {
        if (poolIds == null || poolIds.isEmpty()) {
            return false;
        }

        for (Long poolId : poolIds) {
            StoragePool pool = _storagePoolDao.findById(poolId);
            if (!canHostAccessStoragePool(host, pool)) {
                return false;
            }
        }

        return true;
    }

    @Override
    @DB
    public List<StoragePoolHostVO> findStoragePoolsConnectedToHost(long hostId) {
        return _storagePoolHostDao.listByHostId(hostId);
    }

    @Override
    public boolean canHostAccessStoragePool(Host host, StoragePool pool) {
        if (host == null || pool == null) {
            return false;
        }

        if (!pool.isManaged()) {
            return true;
        }

        DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(pool.getStorageProviderName());
        DataStoreDriver storeDriver = storeProvider.getDataStoreDriver();

        return (storeDriver instanceof PrimaryDataStoreDriver && ((PrimaryDataStoreDriver)storeDriver).canHostAccessStoragePool(host, pool));
    }

    @Override
    public boolean canHostPrepareStoragePoolAccess(Host host, StoragePool pool) {
        if (host == null || pool == null || !pool.isManaged()) {
            return false;
        }

        DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(pool.getStorageProviderName());
        DataStoreDriver storeDriver = storeProvider.getDataStoreDriver();
        return storeDriver instanceof PrimaryDataStoreDriver && ((PrimaryDataStoreDriver)storeDriver).canHostPrepareStoragePoolAccess(host, pool);
    }

    @Override
    @DB
    public Host getHost(long hostId) {
        return _hostDao.findById(hostId);
    }

    @Override
    public void updateSecondaryStorage(long secStorageId, String newUrl) {
        HostVO secHost = _hostDao.findById(secStorageId);
        if (secHost == null) {
            throw new InvalidParameterValueException("Can not find out the secondary storage id: " + secStorageId);
        }

        if (secHost.getType() != Host.Type.SecondaryStorage) {
            throw new InvalidParameterValueException(String.format("host: %s is not a secondary storage", secHost));
        }

        URI uri;
        try {
            uri = new URI(UriUtils.encodeURIComponent(newUrl));
            if (uri.getScheme() == null) {
                throw new InvalidParameterValueException("uri.scheme is null " + newUrl + ", add nfs:// (or cifs://) as a prefix");
            } else if (uri.getScheme().equalsIgnoreCase("nfs")) {
                if (uri.getHost() == null || uri.getHost().equalsIgnoreCase("") || uri.getPath() == null || uri.getPath().equalsIgnoreCase("")) {
                    throw new InvalidParameterValueException("Your host and/or path is wrong.  Make sure it's of the format nfs://hostname/path");
                }
            } else if (uri.getScheme().equalsIgnoreCase("cifs")) {
                // Don't validate against a URI encoded URI.
                URI cifsUri = new URI(newUrl);
                String warnMsg = UriUtils.getCifsUriParametersProblems(cifsUri);
                if (warnMsg != null) {
                    throw new InvalidParameterValueException(warnMsg);
                }
            }
        } catch (URISyntaxException e) {
            throw new InvalidParameterValueException(newUrl + " is not a valid uri");
        }

        String oldUrl = secHost.getStorageUrl();

        URI oldUri;
        try {
            oldUri = new URI(UriUtils.encodeURIComponent(oldUrl));
            if (!oldUri.getScheme().equalsIgnoreCase(uri.getScheme())) {
                throw new InvalidParameterValueException("can not change old scheme:" + oldUri.getScheme() + " to " + uri.getScheme());
            }
        } catch (URISyntaxException e) {
            logger.debug("Failed to get uri from {}", oldUrl);
        }

        secHost.setStorageUrl(newUrl);
        secHost.setGuid(newUrl);
        secHost.setName(newUrl);
        _hostDao.update(secHost.getId(), secHost);
    }

    @Override
    public HypervisorType getHypervisorTypeFromFormat(ImageFormat format) {

        if (format == null) {
            return HypervisorType.None;
        }

        if (format == ImageFormat.VHD) {
            return HypervisorType.XenServer;
        } else if (format == ImageFormat.OVA) {
            return HypervisorType.VMware;
        } else if (format == ImageFormat.QCOW2) {
            return HypervisorType.KVM;
        } else if (format == ImageFormat.RAW) {
            return HypervisorType.Ovm;
        } else if (format == ImageFormat.VHDX) {
            return HypervisorType.Hyperv;
        } else {
            return HypervisorType.None;
        }
    }

    private boolean checkUsagedSpace(StoragePool pool) {
        // Managed storage does not currently deal with accounting for physically used space (only provisioned space). Just return true if "pool" is managed.
        if (pool.isManaged() && !canPoolProvideStorageStats(pool)) {
            return true;
        }

        long totalSize = pool.getCapacityBytes();
        if (totalSize <= 0) {
            return false;
        }
        long usedSize = getUsedSize(pool);
        double usedPercentage = ((double)usedSize / (double)totalSize);
        double storageUsedThreshold = CapacityManager.StorageCapacityDisableThreshold.valueIn(pool.getId());
        logger.debug("Checking pool {} for storage, totalSize: {}, usedBytes: {}, usedPct: {}, disable threshold: {}", pool, pool.getCapacityBytes(), pool.getUsedBytes(), usedPercentage, storageUsedThreshold);
        if (usedPercentage >= storageUsedThreshold) {
            logger.debug("Insufficient space on pool: {} since its usage percentage: {} has crossed the pool.storage.capacity.disablethreshold: {}", pool, usedPercentage, storageUsedThreshold);
            return false;
        }
        return true;
    }

    private long getUsedSize(StoragePool pool) {
        if (pool.getStorageProviderName().equalsIgnoreCase(DataStoreProvider.DEFAULT_PRIMARY) || canPoolProvideStorageStats(pool)) {
            return (pool.getUsedBytes());
        }

        StatsCollector sc = StatsCollector.getInstance();
        if (sc != null) {
            StorageStats stats = sc.getStoragePoolStats(pool.getId());
            if (stats == null) {
                stats = sc.getStorageStats(pool.getId());
            }
            if (stats != null) {
                return (stats.getByteUsed());
            }
        }

        return 0;
    }

    protected boolean checkIfPoolIopsCapacityNull(StoragePool pool) {
        // Only IOPS-guaranteed primary storage like SolidFire is using/setting IOPS.
        // This check returns true for storage that does not specify IOPS.
        if (pool.getCapacityIops() == null) {
            logger.info("Storage pool {} does not supply IOPS capacity, assuming enough capacity", pool);

            return true;
        }
        return false;
    }

    protected boolean storagePoolHasEnoughIops(long requestedIops, List<Pair<Volume, DiskProfile>> requestedVolumes, StoragePool pool, boolean skipPoolNullIopsCheck) {
        if (!skipPoolNullIopsCheck && checkIfPoolIopsCapacityNull(pool)) {
            return true;
        }
        StoragePoolVO storagePoolVo = _storagePoolDao.findById(pool.getId());
        long currentIops = _capacityMgr.getUsedIops(storagePoolVo);
        long futureIops = currentIops + requestedIops;
        boolean hasEnoughIops = futureIops <= pool.getCapacityIops();
        String hasCapacity = hasEnoughIops ? "has" : "does not have";
        StringBuilder stringBuilder = new StringBuilder();
        stringBuilder.append(String.format("Pool [%s] %s enough IOPS to allocate volumes [%s]", pool, hasCapacity, requestedVolumes));
        if (!hasEnoughIops) {
            stringBuilder.append(String.format(" - Insufficient un-allocated IOPS for storage allocation: " +
                    "capacityIops : %d, usedIops : %d, requestedIops : %d", pool.getCapacityIops(), currentIops, requestedIops));
        }
        logger.debug(stringBuilder.toString());
        return hasEnoughIops;
    }

    @Override
    public boolean storagePoolHasEnoughIops(List<Pair<Volume, DiskProfile>> requestedVolumes, StoragePool pool) {
        if (requestedVolumes == null || requestedVolumes.isEmpty() || pool == null) {
            logger.debug("Cannot check if storage [{}] has enough IOPS to allocate volumes [{}].",
                    pool, requestedVolumes);
            return false;
        }
        if (checkIfPoolIopsCapacityNull(pool)) {
            return true;
        }
        long requestedIops = 0;
        for (Pair<Volume, DiskProfile> volumeDiskProfilePair : requestedVolumes) {
            Volume requestedVolume = volumeDiskProfilePair.first();
            DiskProfile diskProfile = volumeDiskProfilePair.second();
            Long minIops = requestedVolume.getMinIops();
            if (requestedVolume.getDiskOfferingId() != diskProfile.getDiskOfferingId()) {
                minIops = diskProfile.getMinIops();
            }

            if (minIops != null && minIops > 0) {
                requestedIops += minIops;
            }
        }
        return storagePoolHasEnoughIops(requestedIops, requestedVolumes, pool, true);
    }

    @Override
    public boolean storagePoolHasEnoughIops(Long requestedIops, StoragePool pool) {
        if (pool == null) {
            return false;
        }
        if (requestedIops == null || requestedIops == 0) {
            return true;
        }
        return storagePoolHasEnoughIops(requestedIops, new ArrayList<>(), pool, false);
    }

    @Override
    public boolean storagePoolHasEnoughSpace(Long size, StoragePool pool) {
        if (size == null || size == 0) {
            return true;
        }
        final StoragePoolVO poolVO = _storagePoolDao.findById(pool.getId());
        long allocatedSizeWithTemplate = _capacityMgr.getAllocatedPoolCapacity(poolVO, null);
        return checkPoolforSpace(pool, allocatedSizeWithTemplate, size);
    }

    @Override
    public boolean storagePoolHasEnoughSpace(List<Pair<Volume, DiskProfile>> volumeDiskProfilePairs, StoragePool pool) {
        return storagePoolHasEnoughSpace(volumeDiskProfilePairs, pool, null);
    }

    @Override
    public boolean storagePoolHasEnoughSpace(List<Pair<Volume, DiskProfile>> volumeDiskProfilesList, StoragePool pool, Long clusterId) {
        if (CollectionUtils.isEmpty(volumeDiskProfilesList)) {
            logger.debug("Cannot check if pool [{}] has enough space to allocate volumes because the volumes list is empty.", pool);
            return false;
        }

        if (!checkUsagedSpace(pool)) {
            logger.debug("Cannot allocate pool [{}] because there is not enough space in this pool.", pool);
            return false;
        }

        // allocated space includes templates
        logger.debug("Destination pool: {}", pool);
        // allocated space includes templates
        final StoragePoolVO poolVO = _storagePoolDao.findById(pool.getId());
        long allocatedSizeWithTemplate = _capacityMgr.getAllocatedPoolCapacity(poolVO, null);
        long totalAskingSize = 0;

        for (Pair<Volume, DiskProfile> volumeDiskProfilePair : volumeDiskProfilesList) {
            // refreshing the volume from the DB to get latest hv_ss_reserve (hypervisor snapshot reserve) field
            // I could have just assigned this to "volume", but decided to make a new variable for it so that it
            // might be clearer that this "volume" in "volumeDiskProfilesList" still might have an old value for hv_ss_reverse.
            Volume volume = volumeDiskProfilePair.first();
            DiskProfile diskProfile = volumeDiskProfilePair.second();
            VolumeVO volumeVO = volumeDao.findById(volume.getId());

            if (volumeVO.getHypervisorSnapshotReserve() == null) {
                // update the volume's hv_ss_reserve (hypervisor snapshot reserve) from a disk offering (used for managed storage)
                volService.updateHypervisorSnapshotReserveForVolume(getDiskOfferingVO(volumeVO), volumeVO.getId(), getHypervisorType(volumeVO));

                // hv_ss_reserve field might have been updated; refresh from DB to make use of it in getDataObjectSizeIncludingHypervisorSnapshotReserve
                volumeVO = volumeDao.findById(volume.getId());
            }

            // this if statement should resolve to true at most once per execution of the for loop it's contained within (for a root disk that is
            // to leverage a template)
            if (volume.getTemplateId() != null) {
                VMTemplateVO tmpl = _templateDao.findByIdIncludingRemoved(volume.getTemplateId());

                if (tmpl != null && !ImageFormat.ISO.equals(tmpl.getFormat())) {
                    allocatedSizeWithTemplate = _capacityMgr.getAllocatedPoolCapacity(poolVO, tmpl);
                }
            }

            logger.debug("Pool ID for the volume {} is {}", volumeVO, volumeVO.getPoolId());

            // A ready-state volume is already allocated in a pool, so the asking size is zero for it.
            // In case the volume is moving across pools or is not ready yet, the asking size has to be computed.
            if ((volumeVO.getState() != Volume.State.Ready) || (volumeVO.getPoolId() != pool.getId())) {
                totalAskingSize += getDataObjectSizeIncludingHypervisorSnapshotReserve(volumeVO, diskProfile, poolVO);

                totalAskingSize += getAskingSizeForTemplateBasedOnClusterAndStoragePool(volumeVO.getTemplateId(), clusterId, poolVO);
            }
        }

        return checkPoolforSpace(pool, allocatedSizeWithTemplate, totalAskingSize);
    }

    @Override
    public boolean storagePoolHasEnoughSpaceForResize(StoragePool pool, long currentSize, long newSize) {
        if (!checkUsagedSpace(pool)) {
            return false;
        }
        logger.debug("Destination pool: {}", pool);
        long totalAskingSize = newSize - currentSize;

        if (totalAskingSize <= 0) {
            return true;
        } else {
            final StoragePoolVO poolVO = _storagePoolDao.findById(pool.getId());
            final long allocatedSizeWithTemplate = _capacityMgr.getAllocatedPoolCapacity(poolVO, null);
            return checkPoolforSpace(pool, allocatedSizeWithTemplate, totalAskingSize, true);
        }
    }

    protected Answer getCheckDatastorePolicyComplianceAnswer(String storagePolicyId, StoragePool pool) throws StorageUnavailableException {
        if (StringUtils.isEmpty(storagePolicyId)) {
            return null;
        }
        VsphereStoragePolicyVO storagePolicyVO = _vsphereStoragePolicyDao.findById(Long.parseLong(storagePolicyId));
        List<Long> hostIds = getUpHostsInPool(pool.getId());
        Collections.shuffle(hostIds);

        if (CollectionUtils.isEmpty(hostIds)) {
            throw new StorageUnavailableException("Unable to send command to the pool " + pool.getName() + " due to there is no enabled hosts up in this cluster", pool.getId());
        }
        try {
            StorageFilerTO storageFilerTO = new StorageFilerTO(pool);
            CheckDataStoreStoragePolicyComplianceCommand cmd = new CheckDataStoreStoragePolicyComplianceCommand(storagePolicyVO.getPolicyId(), storageFilerTO);
            long targetHostId = _hvGuruMgr.getGuruProcessedCommandTargetHost(hostIds.get(0), cmd);
            return _agentMgr.send(targetHostId, cmd);
        } catch (AgentUnavailableException e) {
            logger.debug("Unable to send storage pool command to {} via {}", pool, hostIds.get(0), e);
            throw new StorageUnavailableException("Unable to send command to the pool ", pool.getId());
        } catch (OperationTimedoutException e) {
            logger.debug("Failed to process storage pool command to {} via {}", pool, hostIds.get(0), e);
            throw new StorageUnavailableException("Failed to process storage command to the pool ", pool.getId());
        }
    }

    @Override
    public boolean isStoragePoolCompliantWithStoragePolicy(long diskOfferingId, StoragePool pool) throws StorageUnavailableException {
        String storagePolicyId = _diskOfferingDetailsDao.getDetail(diskOfferingId, ApiConstants.STORAGE_POLICY);
        Answer answer = getCheckDatastorePolicyComplianceAnswer(storagePolicyId, pool);
        return answer == null || answer.getResult();
    }

    @Override
    public boolean isStoragePoolCompliantWithStoragePolicy(List<Pair<Volume, DiskProfile>> volumes, StoragePool pool) throws StorageUnavailableException {
        if (CollectionUtils.isEmpty(volumes)) {
            return false;
        }
        List<Pair<Volume, Answer>> answers = new ArrayList<>();

        for (Pair<Volume, DiskProfile> volumeDiskProfilePair : volumes) {
            Volume volume = volumeDiskProfilePair.first();
            DiskProfile diskProfile = volumeDiskProfilePair.second();
            String storagePolicyId = _diskOfferingDetailsDao.getDetail(diskProfile.getDiskOfferingId(), ApiConstants.STORAGE_POLICY);
            Answer answer = getCheckDatastorePolicyComplianceAnswer(storagePolicyId, pool);
            if (answer != null) {
                answers.add(new Pair<>(volume, answer));
            }
        }
        // check cummilative result for all volumes
        for (Pair<Volume, Answer> answer : answers) {
            if (!answer.second().getResult()) {
                logger.debug("Storage pool {} is not compliance with storage policy for volume {}", pool, answer.first().getName());
                return false;
            }
        }
        return true;
    }

    protected boolean checkPoolforSpace(StoragePool pool, long allocatedSizeWithTemplate, long totalAskingSize) {
        return checkPoolforSpace(pool, allocatedSizeWithTemplate, totalAskingSize, false);
    }

    protected boolean checkPoolforSpace(StoragePool pool, long allocatedSizeWithTemplate, long totalAskingSize, boolean forVolumeResize) {
        // allocated space includes templates
        StoragePoolVO poolVO = _storagePoolDao.findById(pool.getId());

        long totalOverProvCapacity;

        if (pool.getPoolType().supportsOverProvisioning()) {
            BigDecimal overProvFactor = getStorageOverProvisioningFactor(pool.getId());

            totalOverProvCapacity = overProvFactor.multiply(new BigDecimal(pool.getCapacityBytes())).longValue();

            logger.debug("Found storage pool {} of type {} with overprovisioning factor {}", pool, pool.getPoolType(), overProvFactor);
            logger.debug("Total over provisioned capacity calculated is {} * {}", overProvFactor, toHumanReadableSize(pool.getCapacityBytes()));
        } else {
            totalOverProvCapacity = pool.getCapacityBytes();

            logger.debug("Found storage pool {} of type {}", poolVO, pool.getPoolType());
        }

        logger.debug("Total capacity of the pool {} is {}", poolVO, toHumanReadableSize(totalOverProvCapacity));

        double storageAllocatedThreshold = CapacityManager.StorageAllocatedCapacityDisableThreshold.valueIn(pool.getId());

        if (logger.isDebugEnabled()) {
            logger.debug("Checking pool: {} for storage allocation , maxSize : {}, " +
                    "totalAllocatedSize : {}, askingSize : {}, allocated disable threshold: {}",
                    pool, toHumanReadableSize(totalOverProvCapacity), toHumanReadableSize(allocatedSizeWithTemplate), toHumanReadableSize(totalAskingSize), storageAllocatedThreshold);
        }

        double usedPercentage = (allocatedSizeWithTemplate + totalAskingSize) / (double)(totalOverProvCapacity);

        if (usedPercentage > storageAllocatedThreshold) {
            logger.debug("Insufficient un-allocated capacity on: {} for storage " +
                    "allocation since its allocated percentage: {} has crossed the allocated" +
                    " pool.storage.allocated.capacity.disablethreshold: {}",
                    pool, usedPercentage, storageAllocatedThreshold);
            if (!forVolumeResize) {
                return false;
            }
            if (!AllowVolumeReSizeBeyondAllocation.valueIn(pool.getId())) {
                logger.debug("Skipping the pool {} as {} is false", pool, AllowVolumeReSizeBeyondAllocation.key());
                return false;
            }

            double storageAllocatedThresholdForResize = CapacityManager.StorageAllocatedCapacityDisableThresholdForVolumeSize.valueIn(pool.getId());
            if (usedPercentage > storageAllocatedThresholdForResize) {
                logger.debug("Skipping the pool {} since its allocated percentage: {} has crossed the allocated {}: {}",
                        pool, usedPercentage, CapacityManager.StorageAllocatedCapacityDisableThresholdForVolumeSize.key(), storageAllocatedThresholdForResize);
                return false;
            }
        }

        if (totalOverProvCapacity < (allocatedSizeWithTemplate + totalAskingSize)) {
            if (logger.isDebugEnabled()) {
                logger.debug("Insufficient un-allocated capacity on: {} for storage " +
                        "allocation, not enough storage, maxSize : {}, totalAllocatedSize : {}, " +
                        "askingSize : {}", pool, toHumanReadableSize(totalOverProvCapacity),
                        toHumanReadableSize(allocatedSizeWithTemplate), toHumanReadableSize(totalAskingSize));
            }

            return false;
        }

        return true;
    }

    /**
     * Storage plug-ins for managed storage can be designed in such a way as to store a template on the primary storage once and
     * make use of it via storage-side cloning.
     * This method determines how many more bytes it will need for the template (if the template is already stored on the primary storage,
     * then the answer is 0).
     */
    private long getAskingSizeForTemplateBasedOnClusterAndStoragePool(Long templateId, Long clusterId, StoragePoolVO storagePoolVO) {
        if (templateId == null || clusterId == null || storagePoolVO == null || !storagePoolVO.isManaged()) {
            return 0;
        }

        VMTemplateVO tmpl = _templateDao.findByIdIncludingRemoved(templateId);

        if (tmpl == null || ImageFormat.ISO.equals(tmpl.getFormat())) {
            return 0;
        }

        HypervisorType hypervisorType = tmpl.getHypervisorType();

        // The getSupportsResigning method is applicable for XenServer as a UUID-resigning patch may or may not be installed on those hypervisor hosts.
        if (_clusterDao.getSupportsResigning(clusterId) || HypervisorType.VMware.equals(hypervisorType) || HypervisorType.KVM.equals(hypervisorType)) {
            return getBytesRequiredForTemplate(tmpl, storagePoolVO);
        }

        return 0;
    }

    private long getDataObjectSizeIncludingHypervisorSnapshotReserve(Volume volume, DiskProfile diskProfile, StoragePool pool) {
        DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(pool.getStorageProviderName());
        DataStoreDriver storeDriver = storeProvider.getDataStoreDriver();

        if (storeDriver instanceof PrimaryDataStoreDriver) {
            PrimaryDataStoreDriver primaryStoreDriver = (PrimaryDataStoreDriver)storeDriver;

            VolumeInfo volumeInfo = volFactory.getVolume(volume.getId());
            if (volume.getDiskOfferingId() != diskProfile.getDiskOfferingId()) {
                return diskProfile.getSize();
            }
            return primaryStoreDriver.getDataObjectSizeIncludingHypervisorSnapshotReserve(volumeInfo, pool);
        }

        return volume.getSize();
    }

    private DiskOfferingVO getDiskOfferingVO(Volume volume) {
        Long diskOfferingId = volume.getDiskOfferingId();

        return _diskOfferingDao.findById(diskOfferingId);
    }

    private HypervisorType getHypervisorType(Volume volume) {
        Long instanceId = volume.getInstanceId();

        VMInstanceVO vmInstance = _vmInstanceDao.findById(instanceId);

        if (vmInstance != null) {
            return vmInstance.getHypervisorType();
        }

        return null;
    }

    private long getBytesRequiredForTemplate(VMTemplateVO tmpl, StoragePool pool) {
        if (tmplFactory.isTemplateMarkedForDirectDownload(tmpl.getId())) {
            return tmpl.getSize();
        }

        DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(pool.getStorageProviderName());
        DataStoreDriver storeDriver = storeProvider.getDataStoreDriver();

        if (storeDriver instanceof PrimaryDataStoreDriver) {
            PrimaryDataStoreDriver primaryStoreDriver = (PrimaryDataStoreDriver)storeDriver;

            TemplateInfo templateInfo = tmplFactory.getReadyTemplateOnImageStore(tmpl.getId(), pool.getDataCenterId());

            return primaryStoreDriver.getBytesRequiredForTemplate(templateInfo, pool);
        }

        return tmpl.getSize();
    }

    @Override
    public boolean storagePoolCompatibleWithVolumePool(StoragePool pool, Volume volume) {
        if (pool == null || volume == null) {
            logger.debug("Cannot check if storage pool [{}] is compatible with volume [{}].", pool, volume);
            return false;
        }

        if (volume.getPoolId() == null) {
            logger.debug("Volume [{}] is not allocated to any pool. Cannot check compatibility with pool [{}].",
                    volume, pool);
            return true;
        }

        StoragePool volumePool = _storagePoolDao.findById(volume.getPoolId());
        if (volumePool == null) {
            logger.debug("Pool [{}] used by volume [{}] does not exist. Cannot check compatibility.", pool, volume);
            return true;
        }

        if (volume.getState() == Volume.State.Ready) {
            if (volumePool.getPoolType() == Storage.StoragePoolType.PowerFlex && pool.getPoolType() != Storage.StoragePoolType.PowerFlex) {
                logger.debug("Pool [{}] with type [{}] does not match volume [{}] pool type [{}].",
                        pool, pool.getPoolType(), volume, volumePool.getPoolType());
                return false;
            } else if (volumePool.getPoolType() != Storage.StoragePoolType.PowerFlex && pool.getPoolType() == Storage.StoragePoolType.PowerFlex) {
                logger.debug("Pool [{}] with type [{}] does not match volume [{}] pool type [{}].",
                        pool, pool.getPoolType(), volume, volumePool.getPoolType());
                return false;
            }
        } else {
            logger.debug("Cannot check compatibility of pool [{}] because volume [{}] is not in [{}] state.", pool, volume, Volume.State.Ready);
            return false;
        }
        logger.debug("Pool [{}] is compatible with volume [{}].", pool, volume);
        return true;
    }

    @Override
    public void createCapacityEntry(long poolId) {
        StoragePoolVO storage = _storagePoolDao.findById(poolId);
        createCapacityEntry(storage, Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED, 0);
    }

    @Override
    public synchronized boolean registerHostListener(String providerName, HypervisorHostListener listener) {
        hostListeners.put(providerName, listener);
        return true;
    }

    @Override
    public String getName() {
        return null;
    }

    private String getValidTemplateName(Long zoneId, HypervisorType hType) {
        String templateName = null;
        if (hType.equals(HypervisorType.XenServer)) {
            templateName = VirtualNetworkApplianceManager.RouterTemplateXen.valueIn(zoneId);
        } else if (hType.equals(HypervisorType.KVM)) {
            templateName = VirtualNetworkApplianceManager.RouterTemplateKvm.valueIn(zoneId);
        } else if (hType.equals(HypervisorType.VMware)) {
            templateName = VirtualNetworkApplianceManager.RouterTemplateVmware.valueIn(zoneId);
        } else if (hType.equals(HypervisorType.Hyperv)) {
            templateName = VirtualNetworkApplianceManager.RouterTemplateHyperV.valueIn(zoneId);
        } else if (hType.equals(HypervisorType.LXC)) {
            templateName = VirtualNetworkApplianceManager.RouterTemplateLxc.valueIn(zoneId);
        }
        return templateName;
    }
    @Override
    public ImageStore discoverImageStore(String name, String url, String providerName, Long zoneId, Map<String, String> details) throws IllegalArgumentException, InvalidParameterValueException {
        DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(providerName);

        if (storeProvider == null) {
            storeProvider = _dataStoreProviderMgr.getDefaultImageDataStoreProvider();
            if (storeProvider == null) {
                throw new InvalidParameterValueException("can't find image store provider: " + providerName);
            }
            providerName = storeProvider.getName(); // ignored passed provider name and use default image store provider name
        }

        ScopeType scopeType = ScopeType.ZONE;
        if (zoneId == null) {
            scopeType = ScopeType.REGION;
        }

        if (name == null) {
            name = url;
        }

        ImageStoreVO imageStore = _imageStoreDao.findByName(name);
        if (imageStore != null) {
            throw new InvalidParameterValueException("The image store with name " + name + " already exists, try creating with another name");
        }

        // check if scope is supported by store provider
        if (!((ImageStoreProvider)storeProvider).isScopeSupported(scopeType)) {
            throw new InvalidParameterValueException("Image store provider " + providerName + " does not support scope " + scopeType);
        }

        // check if we have already image stores from other different providers,
        // we currently are not supporting image stores from different
        // providers co-existing
        List<ImageStoreVO> imageStores = _imageStoreDao.listImageStores();
        for (ImageStoreVO store : imageStores) {
            if (!store.getProviderName().equalsIgnoreCase(providerName)) {
                throw new InvalidParameterValueException("You can only add new image stores from the same provider " + store.getProviderName() + " already added");
            }
        }

        if (zoneId != null) {
            // Check if the zone exists in the system
            DataCenterVO zone = _dcDao.findById(zoneId);
            if (zone == null) {
                throw new InvalidParameterValueException("Can't find zone by id " + zoneId);
            }

            Account account = CallContext.current().getCallingAccount();
            if (Grouping.AllocationState.Disabled == zone.getAllocationState() && !_accountMgr.isRootAdmin(account.getId())) {
                PermissionDeniedException ex = new PermissionDeniedException("Cannot perform this operation, Zone with specified id is currently disabled");
                ex.addProxyObject(zone.getUuid(), "dcId");
                throw ex;
            }
        }

        Map<String, Object> params = new HashMap<>();
        params.put("zoneId", zoneId);
        params.put("url", url);
        params.put("name", name);
        params.put("details", details);
        params.put("scope", scopeType);
        params.put("providerName", storeProvider.getName());
        params.put("role", DataStoreRole.Image);

        DataStoreLifeCycle lifeCycle = storeProvider.getDataStoreLifeCycle();

        DataStore store;
        try {
            store = lifeCycle.initialize(params);
        } catch (Exception e) {
            logger.debug("Failed to add data store: {}", e.getMessage(), e);
            throw new CloudRuntimeException("Failed to add data store: " + e.getMessage(), e);
        }

        if (((ImageStoreProvider)storeProvider).needDownloadSysTemplate()) {
            // trigger system vm template download
            _imageSrv.downloadBootstrapSysTemplate(store);
        } else {
            // populate template_store_ref table
            _imageSrv.addSystemVMTemplatesToSecondary(store);
            _imageSrv.handleTemplateSync(store);
            registerSystemVmTemplateOnFirstNfsStore(zoneId, providerName, url, store);
        }

        // associate builtin template with zones associated with this image store
        associateCrosszoneTemplatesToZone(zoneId);

        // duplicate cache store records to region wide storage
        if (scopeType == ScopeType.REGION) {
            duplicateCacheStoreRecordsToRegionStore(store.getId());
        }

        return (ImageStore)_dataStoreMgr.getDataStore(store.getId(), DataStoreRole.Image);
    }

    protected void registerSystemVmTemplateForHypervisorArch(final HypervisorType hypervisorType,
                 final CPU.CPUArch arch, final Long zoneId, final String url, final DataStore store,
                 final SystemVmTemplateRegistration systemVmTemplateRegistration, final String filePath,
                 final Pair<String, Long> storeUrlAndId, final String nfsVersion) {
        if (HypervisorType.Simulator.equals(hypervisorType)) {
            return;
        }
        String templateName = getValidTemplateName(zoneId, hypervisorType);
        VMTemplateVO registeredTemplate = systemVmTemplateRegistration.getRegisteredTemplate(templateName,
                hypervisorType, arch, url);
        TemplateDataStoreVO templateDataStoreVO = null;
        if (registeredTemplate != null) {
            templateDataStoreVO = _templateStoreDao.findByStoreTemplate(store.getId(), registeredTemplate.getId());
            if (templateDataStoreVO != null) {
                try {
                    if (systemVmTemplateRegistration.validateIfSeeded(templateDataStoreVO, url,
                            templateDataStoreVO.getInstallPath(), nfsVersion)) {
                        return;
                    }
                } catch (Exception e) {
                    logger.error("Failed to validated if template is seeded", e);
                }
            }
        }
        SystemVmTemplateRegistration.mountStore(storeUrlAndId.first(), filePath, nfsVersion);
        if (registeredTemplate != null) {
            systemVmTemplateRegistration.validateAndAddTemplateToStore(registeredTemplate, templateDataStoreVO, zoneId,
                    storeUrlAndId.second(), filePath);
        } else {
            systemVmTemplateRegistration.validateAndRegisterNewTemplate(hypervisorType, arch, templateName, zoneId,
                    storeUrlAndId.second(), filePath);
        }
    }

    private void registerSystemVmTemplateOnFirstNfsStore(Long zoneId, String providerName, String url, DataStore store) {
        if (zoneId == null || !DataStoreProvider.NFS_IMAGE.equals(providerName)) {
            logger.debug("Skipping system VM template registration as either zoneId is null or {} " +
                    "provider is not NFS", store);
            return;
        }
        Transaction.execute(new TransactionCallbackNoReturn() {
            @Override
            public void doInTransactionWithoutResult(final TransactionStatus status) {
                List<ImageStoreVO> stores = _imageStoreDao.listAllStoresInZoneExceptId(zoneId, providerName,
                        DataStoreRole.Image, store.getId());
                if (CollectionUtils.isEmpty(stores)) {
                    List<Pair<HypervisorType, CPU.CPUArch>> hypervisorArchTypes =
                            _clusterDao.listDistinctHypervisorsAndArchExcludingExternalType(zoneId);
                    TransactionLegacy txn = TransactionLegacy.open("AutomaticTemplateRegister");
                    SystemVmTemplateRegistration systemVmTemplateRegistration = new SystemVmTemplateRegistration();
                    String filePath = null;
                    try {
                        filePath = Files.createTempDirectory(SystemVmTemplateRegistration.TEMPORARY_SECONDARY_STORE)
                                .toString();
                        Pair<String, Long> storeUrlAndId = new Pair<>(url, store.getId());
                        String nfsVersion = imageStoreDetailsUtil.getNfsVersion(store.getId());
                        for (Pair<HypervisorType, CPU.CPUArch> hypervisorArchType : hypervisorArchTypes) {
                            try {
                                registerSystemVmTemplateForHypervisorArch(hypervisorArchType.first(),
                                        hypervisorArchType.second(), zoneId, url, store,
                                        systemVmTemplateRegistration, filePath, storeUrlAndId, nfsVersion);
                            } catch (CloudRuntimeException e) {
                                SystemVmTemplateRegistration.unmountStore(filePath);
                                logger.error("Failed to register system VM template for hypervisor: {} {}",
                                        hypervisorArchType.first().name(), hypervisorArchType.second().name(), e);
                            }
                        }
                    } catch (Exception e) {
                        logger.error("Failed to register systemVM template(s) due to: ", e);
                    } finally {
                        SystemVmTemplateRegistration.unmountStore(filePath);
                        txn.close();
                    }
                }
            }
        });
    }
    @Override
    public ImageStore migrateToObjectStore(String name, String url, String providerName, Map<String, String> details) throws DiscoveryException, InvalidParameterValueException {
        // check if current cloud is ready to migrate, we only support cloud with only NFS secondary storages
        List<ImageStoreVO> imgStores = _imageStoreDao.listImageStores();
        List<ImageStoreVO> nfsStores = new ArrayList<>();
        if (CollectionUtils.isNotEmpty(imgStores)) {
            for (ImageStoreVO store : imgStores) {
                if (!store.getProviderName().equals(DataStoreProvider.NFS_IMAGE)) {
                    throw new InvalidParameterValueException("We only support migrate NFS secondary storage to use object store!");
                } else {
                    nfsStores.add(store);
                }
            }
        }
        // convert all NFS secondary storage to staging store
        if (CollectionUtils.isNotEmpty(nfsStores)) {
            for (ImageStoreVO store : nfsStores) {
                long storeId = store.getId();

                _accountMgr.checkAccessAndSpecifyAuthority(CallContext.current().getCallingAccount(), store.getDataCenterId());

                DataStoreProvider provider = _dataStoreProviderMgr.getDataStoreProvider(store.getProviderName());
                DataStoreLifeCycle lifeCycle = provider.getDataStoreLifeCycle();
                DataStore secStore = _dataStoreMgr.getDataStore(storeId, DataStoreRole.Image);
                lifeCycle.migrateToObjectStore(secStore);
                // update store_role in template_store_ref and snapshot_store_ref to ImageCache
                _templateStoreDao.updateStoreRoleToCachce(storeId);
                _snapshotStoreDao.updateStoreRoleToCache(storeId);
            }
        }
        // add object store
        return discoverImageStore(name, url, providerName, null, details);
    }

    @Override
    public ImageStore updateImageStore(UpdateImageStoreCmd cmd) {
        return updateImageStoreStatus(cmd.getId(), cmd.getName(), cmd.getReadonly(), cmd.getCapacityBytes());
    }

    @Override
    @ActionEvent(eventType = EventTypes.EVENT_UPDATE_IMAGE_STORE_ACCESS_STATE,
            eventDescription = "image store access updated")
    public ImageStore updateImageStoreStatus(Long id, String name, Boolean readonly, Long capacityBytes) {
        // Input validation
        ImageStoreVO imageStoreVO = _imageStoreDao.findById(id);
        if (imageStoreVO == null) {
            throw new IllegalArgumentException("Unable to find image store with ID: " + id);
        }
        if (com.cloud.utils.StringUtils.isNotBlank(name)) {
            imageStoreVO.setName(name);
        }
        if (capacityBytes != null) {
            imageStoreVO.setTotalSize(capacityBytes);
        }
        if (readonly != null) {
            imageStoreVO.setReadonly(readonly);
        }
        _imageStoreDao.update(id, imageStoreVO);
        return imageStoreVO;
    }

    @Override
    public ImageStore updateImageStoreStatus(Long id, Boolean readonly) {
        return updateImageStoreStatus(id, null, readonly, null);
    }

    /**
     * @param poolId - Storage pool id for pool to update.
     * @param failOnChecks - If true, throw an error if pool type and state checks fail.
     */
    @Override
    public void updateStorageCapabilities(Long poolId, boolean failOnChecks) {
        StoragePoolVO pool = _storagePoolDao.findById(poolId);

        if (pool == null) {
            throw new CloudRuntimeException("Primary storage not found for id: " + poolId);
        }

        // Only checking NFS for now - required for disk provisioning type support for vmware.
        if (pool.getPoolType() != StoragePoolType.NetworkFilesystem) {
            if (failOnChecks) {
                throw new CloudRuntimeException("Storage capabilities update only supported on NFS storage mounted.");
            }
            return;
        }

        if (pool.getStatus() != StoragePoolStatus.Initialized && pool.getStatus() != StoragePoolStatus.Up) {
            if (failOnChecks){
                throw new CloudRuntimeException("Primary storage is not in the right state to update capabilities");
            }
            return;
        }

        HypervisorType hypervisor = pool.getHypervisor();

        if (hypervisor == null){
            if (pool.getClusterId() != null) {
                ClusterVO cluster = _clusterDao.findById(pool.getClusterId());
                hypervisor = cluster.getHypervisorType();
            }
        }

        if (!HypervisorType.VMware.equals(hypervisor)) {
            if (failOnChecks) {
                throw new CloudRuntimeException("Storage capabilities update only supported on VMWare.");
            }
            return;
        }

        // find the host
        List<Long> poolIds = new ArrayList<>();
        poolIds.add(pool.getId());
        List<Long> hosts = _storagePoolHostDao.findHostsConnectedToPools(poolIds);
        if (!hosts.isEmpty()) {
            GetStoragePoolCapabilitiesCommand cmd = new GetStoragePoolCapabilitiesCommand();
            cmd.setPool(new StorageFilerTO(pool));
            GetStoragePoolCapabilitiesAnswer answer = (GetStoragePoolCapabilitiesAnswer) _agentMgr.easySend(hosts.get(0), cmd);
            if (answer == null) {
                String msg = "Unable to get storage capabilities from storage pool: " + pool.getName();
                logger.error(msg);
                if (failOnChecks) {
                    throw new CloudRuntimeException(msg);
                }
            } else if (answer.getPoolDetails() != null && answer.getPoolDetails().containsKey(Storage.Capability.HARDWARE_ACCELERATION.toString())) {
                StoragePoolDetailVO hardwareAccelerationSupported = _storagePoolDetailsDao.findDetail(pool.getId(), Storage.Capability.HARDWARE_ACCELERATION.toString());
                if (hardwareAccelerationSupported == null) {
                    StoragePoolDetailVO storagePoolDetailVO = new StoragePoolDetailVO(pool.getId(), Storage.Capability.HARDWARE_ACCELERATION.toString(), answer.getPoolDetails().get(Storage.Capability.HARDWARE_ACCELERATION.toString()), false);
                    _storagePoolDetailsDao.persist(storagePoolDetailVO);
                } else {
                    hardwareAccelerationSupported.setValue(answer.getPoolDetails().get(Storage.Capability.HARDWARE_ACCELERATION.toString()));
                    _storagePoolDetailsDao.update(hardwareAccelerationSupported.getId(), hardwareAccelerationSupported);
                }
            } else if (!answer.getResult()) {
                logger.error("Failed to update storage pool capabilities: {}", answer.getDetails());
                if (failOnChecks) {
                    throw new CloudRuntimeException(answer.getDetails());
                }
            }
        }
    }

    private void duplicateCacheStoreRecordsToRegionStore(long storeId) {
        _templateStoreDao.duplicateCacheRecordsOnRegionStore(storeId);
        _snapshotStoreDao.duplicateCacheRecordsOnRegionStore(storeId);
        _volumeStoreDao.duplicateCacheRecordsOnRegionStore(storeId);
    }

    private void associateCrosszoneTemplatesToZone(Long zoneId) {
        List<Long> dcIds = new ArrayList<>();
        if (zoneId != null) {
            dcIds.add(zoneId);
        } else {
            List<DataCenterVO> dcs = _dcDao.listAll();
            if (dcs != null) {
                for (DataCenterVO dc : dcs) {
                    dcIds.add(dc.getId());
                }
            }
        }

        for (Long dcId : dcIds) {
            _imageSrv.associateCrossZoneTemplatesToZone(dcId);
        }
    }

    @Override
    public boolean deleteImageStore(DeleteImageStoreCmd cmd) {
        final long storeId = cmd.getId();
        // Verify that image store exists
        ImageStoreVO store = _imageStoreDao.findById(storeId);
        if (store == null) {
            throw new InvalidParameterValueException("Image store with id " + storeId + " doesn't exist");
        }
        _accountMgr.checkAccessAndSpecifyAuthority(CallContext.current().getCallingAccount(), store.getDataCenterId());

        // Verify that there are no live snapshot, template, volume on the image
        // store to be deleted
        List<SnapshotDataStoreVO> snapshots = _snapshotStoreDao.listByStoreId(storeId, DataStoreRole.Image);
        if (CollectionUtils.isNotEmpty(snapshots)) {
            throw new InvalidParameterValueException("Cannot delete image store with active snapshots backup!");
        }
        List<VolumeDataStoreVO> volumes = _volumeStoreDao.listByStoreId(storeId);
        if (CollectionUtils.isNotEmpty(volumes)) {
            throw new InvalidParameterValueException("Cannot delete image store with active volumes backup!");
        }

        // search if there are user templates stored on this image store, excluding system, builtin templates
        List<TemplateJoinVO> templates = _templateViewDao.listActiveTemplates(storeId);
        if (CollectionUtils.isNotEmpty(templates)) {
            throw new InvalidParameterValueException("Cannot delete image store with active Templates backup!");
        }

        // ready to delete
        Transaction.execute(new TransactionCallbackNoReturn() {
            @Override
            public void doInTransactionWithoutResult(TransactionStatus status) {
                // first delete from image_store_details table, we need to do that since
                // we are not actually deleting record from main
                // image_data_store table, so delete cascade will not work
                _imageStoreDetailsDao.deleteDetails(storeId);
                _snapshotStoreDao.deletePrimaryRecordsForStore(storeId, DataStoreRole.Image);
                _volumeStoreDao.deletePrimaryRecordsForStore(storeId);
                _templateStoreDao.deletePrimaryRecordsForStore(storeId);
                annotationDao.removeByEntityType(AnnotationService.EntityType.SECONDARY_STORAGE.name(), store.getUuid());
                _imageStoreDao.remove(storeId);
            }
        });

        return true;
    }

    @Override
    public ImageStore createSecondaryStagingStore(CreateSecondaryStagingStoreCmd cmd) {
        String providerName = cmd.getProviderName();
        DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(providerName);

        if (storeProvider == null) {
            storeProvider = _dataStoreProviderMgr.getDefaultCacheDataStoreProvider();
            if (storeProvider == null) {
                throw new InvalidParameterValueException("can't find cache store provider: " + providerName);
            }
        }

        Long dcId = cmd.getZoneId();

        ScopeType scopeType = null;
        String scope = cmd.getScope();
        if (scope != null) {
            try {
                scopeType = Enum.valueOf(ScopeType.class, scope.toUpperCase());

            } catch (Exception e) {
                throw new InvalidParameterValueException("invalid scope for cache store " + scope);
            }

            if (scopeType != ScopeType.ZONE) {
                throw new InvalidParameterValueException("Only zone wide cache storage is supported");
            }
        }

        if (scopeType == ScopeType.ZONE && dcId == null) {
            throw new InvalidParameterValueException("zone id can't be null, if scope is zone");
        }

        // Check if the zone exists in the system
        DataCenterVO zone = _dcDao.findById(dcId);
        if (zone == null) {
            throw new InvalidParameterValueException("Can't find zone by id " + dcId);
        }

        Account account = CallContext.current().getCallingAccount();
        if (Grouping.AllocationState.Disabled == zone.getAllocationState() && !_accountMgr.isRootAdmin(account.getId())) {
            PermissionDeniedException ex = new PermissionDeniedException("Cannot perform this operation, Zone with specified id is currently disabled");
            ex.addProxyObject(zone.getUuid(), "dcId");
            throw ex;
        }

        Map<String, Object> params = new HashMap<>();
        params.put("zoneId", dcId);
        params.put("url", cmd.getUrl());
        params.put("name", cmd.getUrl());
        params.put("details", cmd.getDetails());
        params.put("scope", scopeType);
        params.put("providerName", storeProvider.getName());
        params.put("role", DataStoreRole.ImageCache);

        DataStoreLifeCycle lifeCycle = storeProvider.getDataStoreLifeCycle();
        DataStore store;
        try {
            store = lifeCycle.initialize(params);
        } catch (Exception e) {
            logger.debug("Failed to add data store: {}", e.getMessage(), e);
            throw new CloudRuntimeException("Failed to add data store: " + e.getMessage(), e);
        }

        return (ImageStore)_dataStoreMgr.getDataStore(store.getId(), DataStoreRole.ImageCache);
    }

    @Override
    public boolean deleteSecondaryStagingStore(DeleteSecondaryStagingStoreCmd cmd) {
        final long storeId = cmd.getId();
        // Verify that cache store exists
        ImageStoreVO store = _imageStoreDao.findById(storeId);
        if (store == null) {
            throw new InvalidParameterValueException("Cache store with id " + storeId + " doesn't exist");
        }
        _accountMgr.checkAccessAndSpecifyAuthority(CallContext.current().getCallingAccount(), store.getDataCenterId());

        // Verify that there are no live snapshot, template, volume on the cache
        // store that is currently referenced
        List<SnapshotDataStoreVO> snapshots = _snapshotStoreDao.listActiveOnCache(storeId);
        if (CollectionUtils.isNotEmpty(snapshots)) {
            throw new InvalidParameterValueException("Cannot delete cache store with staging snapshots currently in use!");
        }
        List<VolumeDataStoreVO> volumes = _volumeStoreDao.listActiveOnCache(storeId);
        if (CollectionUtils.isNotEmpty(volumes)) {
            throw new InvalidParameterValueException("Cannot delete cache store with staging Volumes currently in use!");
        }

        List<TemplateDataStoreVO> templates = _templateStoreDao.listActiveOnCache(storeId);
        if (CollectionUtils.isNotEmpty(templates)) {
            throw new InvalidParameterValueException("Cannot delete cache store with staging Templates currently in use!");
        }

        // ready to delete
        Transaction.execute(new TransactionCallbackNoReturn() {
            @Override
            public void doInTransactionWithoutResult(TransactionStatus status) {
                // first delete from image_store_details table, we need to do that since
                // we are not actually deleting record from main
                // image_data_store table, so delete cascade will not work
                _imageStoreDetailsDao.deleteDetails(storeId);
                _snapshotStoreDao.deletePrimaryRecordsForStore(storeId, DataStoreRole.ImageCache);
                _volumeStoreDao.deletePrimaryRecordsForStore(storeId);
                _templateStoreDao.deletePrimaryRecordsForStore(storeId);
                _imageStoreDao.remove(storeId);
            }
        });

        return true;
    }

    protected class DownloadURLGarbageCollector implements Runnable {

        public DownloadURLGarbageCollector() {
        }

        @Override
        public void run() {
            try {
                logger.trace("Download URL Garbage Collection Thread is running.");

                cleanupDownloadUrls();

            } catch (Exception e) {
                logger.error("Caught the following Exception", e);
            }
        }
    }

    @Override
    public void cleanupDownloadUrls() {

        // Cleanup expired volume URLs
        List<VolumeDataStoreVO> volumesOnImageStoreList = _volumeStoreDao.listVolumeDownloadUrls();
        HashSet<Long> expiredVolumeIds = new HashSet<>();
        HashSet<Long> activeVolumeIds = new HashSet<>();
        for (VolumeDataStoreVO volumeOnImageStore : volumesOnImageStoreList) {

            long volumeId = volumeOnImageStore.getVolumeId();
            VolumeVO volume = volumeDao.findById(volumeId);
            try {
                long downloadUrlCurrentAgeInSecs = DateUtil.getTimeDifference(DateUtil.now(), volumeOnImageStore.getExtractUrlCreated());
                if (downloadUrlCurrentAgeInSecs < _downloadUrlExpirationInterval) {  // URL hasnt expired yet
                    activeVolumeIds.add(volumeId);
                    continue;
                }
                expiredVolumeIds.add(volumeId);
                logger.debug("Removing download url {} for volume {}", volumeOnImageStore.getExtractUrl(), volume);

                // Remove it from image store
                ImageStoreEntity secStore = (ImageStoreEntity)_dataStoreMgr.getDataStore(volumeOnImageStore.getDataStoreId(), DataStoreRole.Image);
                secStore.deleteExtractUrl(volumeOnImageStore.getInstallPath(), volumeOnImageStore.getExtractUrl(), Upload.Type.VOLUME);

                // Now expunge it from DB since this entry was created only for download purpose
                _volumeStoreDao.expunge(volumeOnImageStore.getId());
            } catch (Throwable th) {
                logger.warn("Caught exception while deleting download url {} for volume {}", volumeOnImageStore.getExtractUrl(), volume, th);
            }
        }
        for (Long volumeId : expiredVolumeIds) {
            if (activeVolumeIds.contains(volumeId)) {
                continue;
            }
            Volume volume = volumeDao.findById(volumeId);
            if (volume != null && volume.getState() == Volume.State.Expunged) {
                volumeDao.remove(volumeId);
            }
        }

        // Cleanup expired template URLs
        List<TemplateDataStoreVO> templatesOnImageStoreList = _templateStoreDao.listTemplateDownloadUrls();
        for (TemplateDataStoreVO templateOnImageStore : templatesOnImageStoreList) {
            VMTemplateVO template = _templateDao.findById(templateOnImageStore.getId());
            try {
                long downloadUrlCurrentAgeInSecs = DateUtil.getTimeDifference(DateUtil.now(), templateOnImageStore.getExtractUrlCreated());
                if (downloadUrlCurrentAgeInSecs < _downloadUrlExpirationInterval) {  // URL hasnt expired yet
                    continue;
                }

                logger.debug("Removing download url {} for template {}", templateOnImageStore.getExtractUrl(), template);

                // Remove it from image store
                ImageStoreEntity secStore = (ImageStoreEntity)_dataStoreMgr.getDataStore(templateOnImageStore.getDataStoreId(), DataStoreRole.Image);
                secStore.deleteExtractUrl(templateOnImageStore.getInstallPath(), templateOnImageStore.getExtractUrl(), Upload.Type.TEMPLATE);

                // Now remove download details from DB.
                templateOnImageStore.setExtractUrl(null);
                templateOnImageStore.setExtractUrlCreated(null);
                _templateStoreDao.update(templateOnImageStore.getId(), templateOnImageStore);
            } catch (Throwable th) {
                logger.warn("caught exception while deleting download url {} for template {}", templateOnImageStore.getExtractUrl(), template, th);
            }
        }

        Date date = DateUtils.addSeconds(new Date(), -1 * _downloadUrlExpirationInterval);
        List<ImageStoreObjectDownloadVO> imageStoreObjectDownloadList = _imageStoreObjectDownloadDao.listToExpire(date);
        for (ImageStoreObjectDownloadVO imageStoreObjectDownloadVO : imageStoreObjectDownloadList) {
            try {
                ImageStoreEntity secStore = (ImageStoreEntity)_dataStoreMgr.getDataStore(imageStoreObjectDownloadVO.getStoreId(), DataStoreRole.Image);
                secStore.deleteExtractUrl(imageStoreObjectDownloadVO.getPath(), imageStoreObjectDownloadVO.getDownloadUrl(), null);
                _imageStoreObjectDownloadDao.expunge(imageStoreObjectDownloadVO.getId());
            } catch (Throwable th) {
                logger.warn("caught exception while deleting download url {} for object {}", imageStoreObjectDownloadVO.getDownloadUrl(), imageStoreObjectDownloadVO, th);
            }
        }

        List <SnapshotDataStoreVO> snapshotDataStoreVos = _snapshotStoreDao.listExtractedSnapshotsBeforeDate(DateUtils.addSeconds(DateUtil.now(), -_downloadUrlExpirationInterval));

        for (SnapshotDataStoreVO snapshotDataStoreVo : snapshotDataStoreVos) {
            ImageStoreEntity secStore = (ImageStoreEntity)_dataStoreMgr.getDataStore(snapshotDataStoreVo.getDataStoreId(), DataStoreRole.Image);
            secStore.deleteExtractUrl(snapshotDataStoreVo.getInstallPath(), snapshotDataStoreVo.getExtractUrl(), Upload.Type.SNAPSHOT);

            snapshotDataStoreVo.setExtractUrl(null);
            snapshotDataStoreVo.setExtractUrlCreated(null);
            _snapshotStoreDao.update(snapshotDataStoreVo.getId(), snapshotDataStoreVo);
        }
    }

    // get bytesReadRate from service_offering, disk_offering and vm.disk.throttling.bytes_read_rate
    @Override
    public Long getDiskBytesReadRate(final ServiceOffering offering, final DiskOffering diskOffering) {
        if ((diskOffering != null) && (diskOffering.getBytesReadRate() != null) && (diskOffering.getBytesReadRate() > 0)) {
            return diskOffering.getBytesReadRate();
        } else if ((diskOffering != null) && (diskOffering.getBytesReadRate() != null) && (diskOffering.getBytesReadRate() > 0)) {
            return diskOffering.getBytesReadRate();
        } else {
            Long bytesReadRate = Long.parseLong(_configDao.getValue(Config.VmDiskThrottlingBytesReadRate.key()));
            if ((bytesReadRate > 0) && ((offering == null) || (!offering.isSystemUse()))) {
                return bytesReadRate;
            }
        }
        return 0L;
    }

    // get bytesWriteRate from service_offering, disk_offering and vm.disk.throttling.bytes_write_rate
    @Override
    public Long getDiskBytesWriteRate(final ServiceOffering offering, final DiskOffering diskOffering) {
        if ((diskOffering != null) && (diskOffering.getBytesWriteRate() != null) && (diskOffering.getBytesWriteRate() > 0)) {
            return diskOffering.getBytesWriteRate();
        } else {
            Long bytesWriteRate = Long.parseLong(_configDao.getValue(Config.VmDiskThrottlingBytesWriteRate.key()));
            if ((bytesWriteRate > 0) && ((offering == null) || (!offering.isSystemUse()))) {
                return bytesWriteRate;
            }
        }
        return 0L;
    }

    // get iopsReadRate from service_offering, disk_offering and vm.disk.throttling.iops_read_rate
    @Override
    public Long getDiskIopsReadRate(final ServiceOffering offering, final DiskOffering diskOffering) {
        if ((diskOffering != null) && (diskOffering.getIopsReadRate() != null) && (diskOffering.getIopsReadRate() > 0)) {
            return diskOffering.getIopsReadRate();
        } else {
            Long iopsReadRate = Long.parseLong(_configDao.getValue(Config.VmDiskThrottlingIopsReadRate.key()));
            if ((iopsReadRate > 0) && ((offering == null) || (!offering.isSystemUse()))) {
                return iopsReadRate;
            }
        }
        return 0L;
    }

    // get iopsWriteRate from service_offering, disk_offering and vm.disk.throttling.iops_write_rate
    @Override
    public Long getDiskIopsWriteRate(final ServiceOffering offering, final DiskOffering diskOffering) {
        if ((diskOffering != null) && (diskOffering.getIopsWriteRate() != null) && (diskOffering.getIopsWriteRate() > 0)) {
            return diskOffering.getIopsWriteRate();
        } else {
            Long iopsWriteRate = Long.parseLong(_configDao.getValue(Config.VmDiskThrottlingIopsWriteRate.key()));
            if ((iopsWriteRate > 0) && ((offering == null) || (!offering.isSystemUse()))) {
                return iopsWriteRate;
            }
        }
        return 0L;
    }

    @Override
    public String getConfigComponentName() {
        return StorageManager.class.getSimpleName();
    }

    @Override
    public ConfigKey<?>[] getConfigKeys() {
        return new ConfigKey<?>[]{
                StorageCleanupInterval,
                StorageCleanupDelay,
                StorageCleanupEnabled,
                TemplateCleanupEnabled,
                KvmStorageOfflineMigrationWait,
                KvmStorageOnlineMigrationWait,
                KvmAutoConvergence,
                MaxNumberOfManagedClusteredFileSystems,
                STORAGE_POOL_DISK_WAIT,
                STORAGE_POOL_CLIENT_TIMEOUT,
                STORAGE_POOL_CLIENT_MAX_CONNECTIONS,
                STORAGE_POOL_CONNECTED_CLIENTS_LIMIT,
                STORAGE_POOL_IO_POLICY,
                PRIMARY_STORAGE_DOWNLOAD_WAIT,
                SecStorageMaxMigrateSessions,
                MaxDataMigrationWaitTime,
                DiskProvisioningStrictness,
                PreferredStoragePool,
                SecStorageVMAutoScaleDown,
                MountDisabledStoragePool,
                VmwareCreateCloneFull,
                XenserverCreateCloneFull,
                VmwareAllowParallelExecution,
                DataStoreDownloadFollowRedirects,
                AllowVolumeReSizeBeyondAllocation,
                StoragePoolHostConnectWorkers,
                ObjectStorageCapacityThreshold,
                COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES,
                AgentMaxDataMigrationWaitTime
        };
    }

    @Override
    public void setDiskProfileThrottling(DiskProfile dskCh, final ServiceOffering offering, final DiskOffering diskOffering) {
        dskCh.setBytesReadRate(getDiskBytesReadRate(offering, diskOffering));
        dskCh.setBytesWriteRate(getDiskBytesWriteRate(offering, diskOffering));
        dskCh.setIopsReadRate(getDiskIopsReadRate(offering, diskOffering));
        dskCh.setIopsWriteRate(getDiskIopsWriteRate(offering, diskOffering));
    }

    @Override
    public DiskTO getDiskWithThrottling(final DataTO volTO, final Volume.Type volumeType, final long deviceId, final String path, final long offeringId, final long diskOfferingId) {
        DiskTO disk;
        if (volTO instanceof VolumeObjectTO) {
            VolumeObjectTO volumeTO = (VolumeObjectTO)volTO;
            ServiceOffering offering = _entityMgr.findById(ServiceOffering.class, offeringId);
            DiskOffering diskOffering = _entityMgr.findById(DiskOffering.class, diskOfferingId);
            if (volumeType == Volume.Type.ROOT) {
                setVolumeObjectTOThrottling(volumeTO, offering, diskOffering);
            } else {
                setVolumeObjectTOThrottling(volumeTO, null, diskOffering);
            }
            disk = new DiskTO(volumeTO, deviceId, path, volumeType);
        } else {
            disk = new DiskTO(volTO, deviceId, path, volumeType);
        }
        return disk;
    }

    @Override
    public boolean isStoragePoolDatastoreClusterParent(StoragePool pool) {
        List<StoragePoolVO> childStoragePools = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(pool.getId());
        return CollectionUtils.isNotEmpty(childStoragePools);
    }

    private void setVolumeObjectTOThrottling(VolumeObjectTO volumeTO, final ServiceOffering offering, final DiskOffering diskOffering) {
        volumeTO.setBytesReadRate(getDiskBytesReadRate(offering, diskOffering));
        volumeTO.setBytesWriteRate(getDiskBytesWriteRate(offering, diskOffering));
        volumeTO.setIopsReadRate(getDiskIopsReadRate(offering, diskOffering));
        volumeTO.setIopsWriteRate(getDiskIopsWriteRate(offering, diskOffering));
    }

    @Override
    @ActionEvent(eventType = EventTypes.EVENT_OBJECT_STORE_CREATE, eventDescription = "creating object storage")
    public ObjectStore discoverObjectStore(String name, String url, Long size, String providerName, Map<String, String> details)
            throws IllegalArgumentException, InvalidParameterValueException {
        DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(providerName);

        if (storeProvider == null) {
            throw new InvalidParameterValueException("can't find object store provider: " + providerName);
        }

        // Check Unique object store name
        ObjectStoreVO objectStore = _objectStoreDao.findByName(name);
        if (objectStore != null) {
            throw new InvalidParameterValueException("The object store with name " + name + " already exists, try creating with another name");
        }

        try {
            UriUtils.validateUrl(url);
        } catch (InvalidParameterValueException e) {
            throw new InvalidParameterValueException(url + " is not a valid URL:" + e.getMessage());
        }

        // Check Unique object store url
        ObjectStoreVO objectStoreUrl = _objectStoreDao.findByUrl(url);
        if (objectStoreUrl != null) {
            throw new InvalidParameterValueException("The object store with url " + url + " already exists");
        }


        Map<String, Object> params = new HashMap<>();
        params.put("url", url);
        params.put("name", name);
        params.put("size", size == null ? 0L : size);
        params.put("providerName", storeProvider.getName());
        params.put("role", DataStoreRole.Object);
        params.put("details", details);

        DataStoreLifeCycle lifeCycle = storeProvider.getDataStoreLifeCycle();

        DataStore store;
        try {
            store = lifeCycle.initialize(params);
        } catch (Exception e) {
            logger.debug("Failed to add object store: {}", e.getMessage(), e);
            throw new CloudRuntimeException("Failed to add object store: " + e.getMessage(), e);
        }

        return (ObjectStore)_dataStoreMgr.getDataStore(store.getId(), DataStoreRole.Object);
    }

    @Override
    @ActionEvent(eventType = EventTypes.EVENT_OBJECT_STORE_DELETE, eventDescription = "deleting object storage")
    public boolean deleteObjectStore(DeleteObjectStoragePoolCmd cmd) {
        final long storeId = cmd.getId();
        // Verify that object store exists
        ObjectStoreVO store = _objectStoreDao.findById(storeId);
        if (store == null) {
            throw new InvalidParameterValueException("Object store with id " + storeId + " doesn't exist");
        }

        // Verify that there are no buckets in the store
        List<BucketVO> buckets = _bucketDao.listByObjectStoreId(storeId);
        if(CollectionUtils.isNotEmpty(buckets)) {
            throw new InvalidParameterValueException("Cannot delete object store with buckets");
        }

        // ready to delete
        Transaction.execute(new TransactionCallbackNoReturn() {
            @Override
            public void doInTransactionWithoutResult(TransactionStatus status) {
                _objectStoreDetailsDao.deleteDetails(storeId);
                _objectStoreDao.remove(storeId);
            }
        });
        logger.debug("Successfully deleted object store: {}", store);
        return true;
    }

    @Override
    @ActionEvent(eventType = EventTypes.EVENT_OBJECT_STORE_UPDATE, eventDescription = "update object storage")
    public ObjectStore updateObjectStore(Long id, UpdateObjectStoragePoolCmd cmd) {

        // Input validation
        ObjectStoreVO objectStoreVO = _objectStoreDao.findById(id);
        if (objectStoreVO == null) {
            throw new IllegalArgumentException("Unable to find object store with ID: " + id);
        }

        if(cmd.getUrl() != null ) {
            String url = cmd.getUrl();
            try {
                // Check URL
                UriUtils.validateUrl(url);
            } catch (final Exception e) {
                throw new InvalidParameterValueException(url + " is not a valid URL");
            }
            ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object);
            String oldUrl = objectStoreVO.getUrl();
            objectStoreVO.setUrl(url);
            _objectStoreDao.update(id, objectStoreVO);
            //Update URL and check access
            try {
                objectStore.listBuckets();
            } catch (Exception e) {
                //Revert to old URL on failure
                objectStoreVO.setUrl(oldUrl);
                _objectStoreDao.update(id, objectStoreVO);
                throw new IllegalArgumentException("Unable to access Object Storage with URL: " + cmd.getUrl());
            }
        }

        if(cmd.getName() != null ) {
            objectStoreVO.setName(cmd.getName());
        }
        if (cmd.getSize() != null) {
            objectStoreVO.setTotalSize(cmd.getSize() * ResourceType.bytesToGiB);
        }
        _objectStoreDao.update(id, objectStoreVO);
        logger.debug("Successfully updated object store: {}", objectStoreVO);
        return objectStoreVO;
    }

    @Override
    public CapacityVO getObjectStorageUsedStats(Long zoneId) {
        List<ObjectStoreVO> objectStores = _objectStoreDao.listObjectStores();
        Long allocated = 0L;
        Long total = 0L;
        for (ObjectStoreVO objectStore: objectStores) {
            if (objectStore.getAllocatedSize() != null) {
                allocated += objectStore.getAllocatedSize();
            }
            if (objectStore.getTotalSize() != null) {
                total += objectStore.getTotalSize();
            }
        }
        return new CapacityVO(null, zoneId, null, null, allocated, total, Capacity.CAPACITY_TYPE_OBJECT_STORAGE);
    }
}