deal with excessive attempts on failing snapshots

api/src/main/java/com/cloud/event/EventTypes.java

@@ -370,6 +370,7 @@ public class EventTypes {
     public static final String EVENT_SNAPSHOT_DELETE = "SNAPSHOT.DELETE";
     public static final String EVENT_SNAPSHOT_REVERT = "SNAPSHOT.REVERT";
     public static final String EVENT_SNAPSHOT_EXTRACT = "SNAPSHOT.EXTRACT";
+    public static final String EVENT_SNAPSHOT_SKIPPED = "SNAPSHOT.SKIPPED";
     public static final String EVENT_SNAPSHOT_POLICY_CREATE = "SNAPSHOTPOLICY.CREATE";
     public static final String EVENT_SNAPSHOT_POLICY_UPDATE = "SNAPSHOTPOLICY.UPDATE";
     public static final String EVENT_SNAPSHOT_POLICY_DELETE = "SNAPSHOTPOLICY.DELETE";
@@ -1074,6 +1075,7 @@ public class EventTypes {
 
         // Snapshots
         entityEventDetails.put(EVENT_SNAPSHOT_CREATE, Snapshot.class);
+        entityEventDetails.put(EVENT_SNAPSHOT_SKIPPED, Snapshot.class);
         entityEventDetails.put(EVENT_SNAPSHOT_DELETE, Snapshot.class);
         entityEventDetails.put(EVENT_SNAPSHOT_EXTRACT, Snapshot.class);
         entityEventDetails.put(EVENT_SNAPSHOT_ON_PRIMARY, Snapshot.class);

engine/schema/src/main/java/com/cloud/event/dao/EventDao.java

@@ -35,4 +35,10 @@ public interface EventDao extends GenericDao<EventVO, Long> {
 
     public void archiveEvents(List<EventVO> events);
 
+    /**
+     * Returns the most recent events of the given type for a resource, newest first.
+     * Used to derive how many consecutive attempts have failed for that resource without a dedicated counter column.
+     */
+    List<EventVO> listLatestEventsByResource(long resourceId, String resourceType, String type, int limit);
+
 }

engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java

@@ -38,6 +38,7 @@
     protected final SearchBuilder<EventVO> CompletedEventSearch;
     protected final SearchBuilder<EventVO> ToArchiveOrDeleteEventSearch;
     protected final SearchBuilder<EventVO> ArchiveByIdsSearch;
+    protected final SearchBuilder<EventVO> LatestEventsByResourceSearch;
 
     public EventDaoImpl() {
         CompletedEventSearch = createSearchBuilder();
@@ -46,6 +47,12 @@
         CompletedEventSearch.and("archived", CompletedEventSearch.entity().getArchived(), Op.EQ);
         CompletedEventSearch.done();
 
+        LatestEventsByResourceSearch = createSearchBuilder();
+        LatestEventsByResourceSearch.and("resourceId", LatestEventsByResourceSearch.entity().getResourceId(), Op.EQ);
+        LatestEventsByResourceSearch.and("resourceType", LatestEventsByResourceSearch.entity().getResourceType(), Op.EQ);
+        LatestEventsByResourceSearch.and("type", LatestEventsByResourceSearch.entity().getType(), Op.EQ);
+        LatestEventsByResourceSearch.done();
+
         ToArchiveOrDeleteEventSearch = createSearchBuilder();
         ToArchiveOrDeleteEventSearch.and("id", ToArchiveOrDeleteEventSearch.entity().getId(), Op.IN);
         ToArchiveOrDeleteEventSearch.and("type", ToArchiveOrDeleteEventSearch.entity().getType(), Op.EQ);
@@ -105,6 +112,16 @@
         return search(sc, null);
     }
 
+    @Override
+    public List<EventVO> listLatestEventsByResource(long resourceId, String resourceType, String type, int limit) {
+        SearchCriteria<EventVO> sc = LatestEventsByResourceSearch.create();
+        sc.setParameters("resourceId", resourceId);
+        sc.setParameters("resourceType", resourceType);
+        sc.setParameters("type", type);
+        Filter filter = new Filter(EventVO.class, "createDate", false, 0L, (long) limit);
+        return listBy(sc, filter);
+    }
+
     @Override
     public void archiveEvents(List<EventVO> events) {
         if (CollectionUtils.isEmpty(events)) {

server/src/main/java/com/cloud/storage/snapshot/SnapshotManager.java

@@ -16,6 +16,8 @@
 // under the License.
 package com.cloud.storage.snapshot;
 
+import java.util.List;
+
 import com.cloud.user.Account;
 import com.cloud.hypervisor.Hypervisor;
 import com.cloud.storage.StoragePool;
@@ -53,6 +55,18 @@ public interface SnapshotManager extends Configurable {
     public static final ConfigKey<Integer> BackupRetryInterval = new ConfigKey<Integer>(Integer.class, "backup.retry.interval", "Advanced", "300",
             "Time in seconds between retries in backing up snapshot to secondary", false, ConfigKey.Scope.Global, null);
 
+    ConfigKey<Integer> SnapshotRecurringMaxFailures = new ConfigKey<>(Integer.class, "snapshot.recurring.max.failures", "Snapshots", "3",
+            "Maximum number of consecutive failed attempts allowed for a recurring snapshot before it is left until its next regularly scheduled run and a failure event is logged. Set to 0 to retry indefinitely.",
+            true, List.of(ConfigKey.Scope.Account, ConfigKey.Scope.Domain, ConfigKey.Scope.Zone, ConfigKey.Scope.Global), null);
+
+    ConfigKey<Integer> SnapshotRecurringRetryInterval = new ConfigKey<>(Integer.class, "snapshot.recurring.retry.interval", "Snapshots", "300",
+            "Time in seconds to wait before retrying a failed recurring snapshot attempt.",
+            true, List.of(ConfigKey.Scope.Account, ConfigKey.Scope.Domain, ConfigKey.Scope.Zone, ConfigKey.Scope.Global), null);
+
+    ConfigKey<Boolean> SnapshotSkipIfVmNotRunning = new ConfigKey<>(Boolean.class, "snapshot.skip.if.vm.not.running", "Snapshots", "false",
+            "For recurring snapshots, skip taking a new snapshot of a volume (and log a skipped event instead) when the VM it is attached to has not been running since the last snapshot was taken, since nothing on the volume could have changed.",
+            true, List.of(ConfigKey.Scope.Account, ConfigKey.Scope.Domain, ConfigKey.Scope.Zone, ConfigKey.Scope.Global), null);
+
     public static final ConfigKey<Boolean> VmStorageSnapshotKvm = new ConfigKey<>(Boolean.class, "kvm.vmstoragesnapshot.enabled", "Snapshots", "true", "For live snapshot of virtual machine instance on KVM hypervisor without memory. Requires qemu version 1.6+ (on NFS or Local file system) and qemu-guest-agent installed on guest VM", true, ConfigKey.Scope.Global, null);
 
     ConfigKey<Boolean> KVMSnapshotEnabled = new ConfigKey<>(Boolean.class, "kvm.snapshot.enabled", "Snapshots", "true", "Whether volume snapshot is enabled on running instances " +

server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java

@@ -310,7 +310,8 @@
     @Override
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] {BackupRetryAttempts, BackupRetryInterval, SnapshotHourlyMax, SnapshotDailyMax, SnapshotMonthlyMax, SnapshotWeeklyMax, usageSnapshotSelection,
-                SnapshotInfo.BackupSnapshotAfterTakingSnapshot, VmStorageSnapshotKvm, kvmIncrementalSnapshot, snapshotDeltaMax, snapshotShowChainSize, UseStorageReplication, KVMSnapshotEnabled};
+                SnapshotInfo.BackupSnapshotAfterTakingSnapshot, VmStorageSnapshotKvm, kvmIncrementalSnapshot, snapshotDeltaMax, snapshotShowChainSize, UseStorageReplication, KVMSnapshotEnabled,
+                SnapshotRecurringMaxFailures, SnapshotRecurringRetryInterval, SnapshotSkipIfVmNotRunning};
     }
 
     @Override
@@ -1167,7 +1168,7 @@
     @Override
     @DB
     @ActionEvent(eventType = EventTypes.EVENT_SNAPSHOT_POLICY_CREATE, eventDescription = "creating snapshot policy")
     public SnapshotPolicyVO createPolicy(CreateSnapshotPolicyCmd cmd, Account policyOwner) {
         Long volumeId = cmd.getVolumeId();
         boolean display = cmd.isDisplay();
         VolumeVO volume = _volsDao.findById(cmd.getVolumeId());
@@ -1621,7 +1622,7 @@
 
     @Override
     @DB
     public SnapshotInfo takeSnapshot(VolumeInfo volume) throws ResourceAllocationException {
         CreateSnapshotPayload payload = (CreateSnapshotPayload)volume.getpayload();
 
         Long snapshotId = payload.getSnapshotId();