// Copyright (C) 2024, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal in the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Server.Serialization.Interface; using System; using System.Collections.Generic; using System.Text; using System.Linq; using System.Threading; using Duplicati.Library.Utility; using Duplicati.Library.RestAPI; // TODO: Rewrite this class. // It should just signal what new backups to run, and not mix with the worker thread. namespace Duplicati.Server { /// /// This class handles scheduled runs of backups /// public class Scheduler { private static readonly string LOGTAG = Duplicati.Library.Logging.Log.LogTagFromType(); /// /// The thread that runs the scheduler /// private Thread m_thread; /// /// A termination flag /// private volatile bool m_terminate; /// /// The worker thread that is invoked to do work /// private WorkerThread m_worker; /// /// The wait event /// private AutoResetEvent m_event; /// /// The data synchronization lock /// private readonly object m_lock = new object(); /// /// An event that is raised when the schedule changes /// public event EventHandler NewSchedule; /// /// The currently scheduled items /// private KeyValuePair[] m_schedule; /// /// List of update tasks, used to set the timestamp on the schedule once completed /// private Dictionary> m_updateTasks; /// /// Constructs a new scheduler /// public Scheduler() { } /// /// Initializes scheduler /// /// The worker thread public void Init(WorkerThread worker) { m_thread = new Thread(new ThreadStart(Runner)); m_worker = worker; m_worker.CompletedWork += OnCompleted; m_worker.StartingWork += OnStartingWork; m_schedule = new KeyValuePair[0]; m_terminate = false; m_event = new AutoResetEvent(false); m_updateTasks = new Dictionary>(); m_thread.IsBackground = true; m_thread.Name = "TaskScheduler"; m_thread.Start(); } public IList> GetSchedulerQueueIds() { return (from n in WorkerQueue where n.Backup != null select new Tuple(n.TaskID, n.Backup.ID)).ToList(); } /// /// Forces the scheduler to re-evaluate the order. /// Call this method if something changes /// public void Reschedule() { m_event.Set(); } /// /// A snapshot copy of the current schedule list /// public List> Schedule { get { lock (m_lock) return m_schedule.ToList(); } } /// /// A snapshot copy of the current worker queue, that is items that are scheduled, but waiting for execution /// public List WorkerQueue { get { return (from t in m_worker.CurrentTasks where t != null select t).ToList(); } } /// /// Terminates the thread. Any items still in queue will be removed /// /// True if the call should block until the thread has exited, false otherwise public void Terminate(bool wait) { m_terminate = true; m_event.Set(); if (wait) { try { m_thread.Join(); } catch { } } } /// /// Returns the next valid date, given the start and the interval /// /// The base time /// The first allowed date /// The repetition interval /// The days the backup is allowed to run /// The next valid date, or throws an exception if no such date can be found public static DateTime GetNextValidTime(DateTime basetime, DateTime firstdate, string repetition, DayOfWeek[] allowedDays) { var res = basetime; var i = 50000; while (res < firstdate && i-- > 0) res = Timeparser.ParseTimeInterval(repetition, res); // If we arrived somewhere after the first allowed date if (res >= firstdate) { var ts = Timeparser.ParseTimeSpan(repetition); if (ts.TotalDays >= 1) { // We jump in days, so we pick the first valid day after firstdate for (var n = 0; n < 8; n++) if (IsDateAllowed(res, allowedDays)) break; else res = res.AddDays(1); } else { // We jump less than a day, so we keep adding the repetition until // we hit a valid day i = 50000; while (!IsDateAllowed(res, allowedDays) && i-- > 0) res = Timeparser.ParseTimeInterval(repetition, res); } } if (!IsDateAllowed(res, allowedDays) || res < firstdate) { StringBuilder sb = new StringBuilder(); if (allowedDays != null) foreach (DayOfWeek w in allowedDays) { if (sb.Length != 0) sb.Append(", "); sb.Append(w.ToString()); } throw new Exception(Strings.Scheduler.InvalidTimeSetupError(basetime, repetition, sb.ToString())); } return res; } private void OnCompleted(WorkerThread worker, Runner.IRunnerData task) { Tuple t = null; lock (m_lock) { if (task != null && m_updateTasks.TryGetValue(task, out t)) m_updateTasks.Remove(task); } if (t != null) { t.Item1.Time = t.Item2; t.Item1.LastRun = t.Item3; FIXMEGlobal.DataConnection.AddOrUpdateSchedule(t.Item1); } } private void OnStartingWork(WorkerThread worker, Runner.IRunnerData task) { if (task is null) { return; } lock (m_lock) { if (m_updateTasks.TryGetValue(task, out Tuple scheduleInfo)) { // Item2 is the scheduled start time (Time in the Schedule table). // Item3 is the actual start time (LastRun in the Schedule table). m_updateTasks[task] = Tuple.Create(scheduleInfo.Item1, scheduleInfo.Item2, DateTime.UtcNow); } } } /// /// The actual scheduling procedure /// private void Runner() { var scheduled = new Dictionary>(); while (!m_terminate) { //TODO: As this is executed repeatedly we should cache it // to avoid frequent db lookups //Determine schedule list var lst = FIXMEGlobal.DataConnection.Schedules; foreach (var sc in lst) { if (!string.IsNullOrEmpty(sc.Repeat)) { KeyValuePair startkey; DateTime last = new DateTime(0, DateTimeKind.Utc); DateTime start; var scticks = sc.Time.Ticks; if (!scheduled.TryGetValue(sc.ID, out startkey) || startkey.Key != scticks) { start = new DateTime(scticks, DateTimeKind.Utc); last = sc.LastRun; } else { start = startkey.Value; } try { // Recover from timedrift issues by overriding the dates if the last run date is in the future. if (last > DateTime.UtcNow) { start = DateTime.UtcNow; last = DateTime.UtcNow; } start = GetNextValidTime(start, last, sc.Repeat, sc.AllowedDays); } catch (Exception ex) { FIXMEGlobal.DataConnection.LogError(sc.ID.ToString(), "Scheduler failed to find next date", ex); } //If time is exceeded, run it now if (start <= DateTime.UtcNow) { var jobsToRun = new List(); //TODO: Cache this to avoid frequent lookups foreach (var id in FIXMEGlobal.DataConnection.GetBackupIDsForTags(sc.Tags).Distinct() .Select(x => x.ToString())) { //See if it is already queued var tmplst = from n in m_worker.CurrentTasks where n.Operation == Duplicati.Server.Serialization.DuplicatiOperation.Backup select n.Backup; var tastTemp = m_worker.CurrentTask; if (tastTemp != null && tastTemp.Operation == Duplicati.Server.Serialization.DuplicatiOperation.Backup) tmplst = tmplst.Union(new[] { tastTemp.Backup }); //If it is not already in queue, put it there if (!tmplst.Any(x => x.ID == id)) { var entry = FIXMEGlobal.DataConnection.GetBackup(id); if (entry != null) { Dictionary options = Duplicati.Server.Runner.GetCommonOptions(); Duplicati.Server.Runner.ApplyOptions(entry, options); if ((new Duplicati.Library.Main.Options(options)).DisableOnBattery && (Duplicati.Library.Utility.Power.PowerSupply.GetSource() == Duplicati.Library.Utility.Power.PowerSupply.Source.Battery)) { Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "BackupDisabledOnBattery", "Scheduled backup disabled while on battery power."); } else { jobsToRun.Add(Server.Runner.CreateTask( Duplicati.Server.Serialization.DuplicatiOperation.Backup, entry)); } } } } // Calculate next time, by finding the first entry later than now try { start = GetNextValidTime(start, new DateTime( Math.Max(DateTime.UtcNow.AddSeconds(1).Ticks, start.AddSeconds(1).Ticks), DateTimeKind.Utc), sc.Repeat, sc.AllowedDays); } catch (Exception ex) { FIXMEGlobal.DataConnection.LogError(sc.ID.ToString(), "Scheduler failed to find next date", ex); continue; } Server.Runner.IRunnerData lastJob = jobsToRun.LastOrDefault(); if (lastJob != null) { lock (m_lock) { // The actual last run time will be updated when the StartingWork event is raised. m_updateTasks[lastJob] = new Tuple(sc, start, DateTime.UtcNow); } } foreach (var job in jobsToRun) m_worker.AddTask(job); if (start < DateTime.UtcNow) { //TODO: Report this somehow continue; } } scheduled[sc.ID] = new KeyValuePair(scticks, start); } } var existing = lst.ToDictionary(x => x.ID); //Sort them, lock as we assign the m_schedule variable lock (m_lock) m_schedule = (from n in scheduled where existing.ContainsKey(n.Key) orderby n.Value.Value select new KeyValuePair(n.Value.Value, existing[n.Key])).ToArray(); // Remove unused entries foreach (var c in (from n in scheduled where !existing.ContainsKey(n.Key) select n.Key).ToArray()) scheduled.Remove(c); //Raise event if needed // TODO: This triggers a new data event and a reconnect with long-poll if (NewSchedule != null) NewSchedule(this, null); int waittime = 0; //Figure out a sensible amount of time to sleep the thread if (scheduled.Count > 0) { //When is the next run scheduled? TimeSpan nextrun = scheduled.Values.Min((x) => x.Value) - DateTime.UtcNow; if (nextrun.TotalMilliseconds < 0) continue; //Don't sleep for more than 5 minutes waittime = (int)Math.Min(nextrun.TotalMilliseconds, 60 * 1000 * 5); } else { // TODO: This should be handled with events, instead of one wakeup per minute //No tasks, check back later waittime = 60 * 1000; } //Waiting on the event, enables a wakeup call from termination // never use waittime = 0 m_event.WaitOne(Math.Max(100, waittime), false); } } /// /// Returns true if the time is at an allowed weekday, false otherwise /// /// The time to evaluate /// The allowed days /// True if the backup is allowed to run, false otherwise private static bool IsDateAllowed(DateTime time, DayOfWeek[] allowedDays) { var localTime = time.ToLocalTime(); if (allowedDays == null || allowedDays.Length == 0) return true; else return Array.IndexOf(allowedDays, localTime.DayOfWeek) >= 0; } } }