Broad performance changes (draft - early comments welcome)

src/Context.cpp

@@ -738,12 +738,11 @@ int Context::run() {
 #endif
       << ' ' << Datetime().toISO()
 
-      << " init:" << time_init_us << " load:" << time_load_us
-      << " gc:" << (time_gc_us > 0 ? time_gc_us - time_load_us : time_gc_us)
+      << " init:" << time_init_us << " load:" << time_load_us << " gc:" << time_gc_us
       << " filter:" << time_filter_us << " commit:" << time_commit_us << " sort:" << time_sort_us
       << " render:" << time_render_us << " hooks:" << time_hooks_us << " other:"
-      << time_total_us - time_init_us - time_gc_us - time_filter_us - time_commit_us -
-             time_sort_us - time_render_us - time_hooks_us
+      << time_total_us - time_init_us - time_load_us - time_gc_us - time_filter_us -
+             time_commit_us - time_sort_us - time_render_us - time_hooks_us
       << " total:" << time_total_us << '\n';
     debug(s.str());
   }
@@ -1126,6 +1125,10 @@ void Context::staticInitialization() {
   for (auto& var : config.all())
     if (var.substr(0, 13) == "urgency.user." || var.substr(0, 12) == "urgency.uda.")
       Task::coefficients[var] = config.getReal(var);
+
+  // Pre-parse the coefficient keys, so urgency_c() doesn't have to re-parse for
+  // each task.
+  Task::setUrgencyCoefficients();
 }
 
 ////////////////////////////////////////////////////////////////////////////////

src/Filter.cpp

@@ -40,180 +40,193 @@
 // Take an input set of tasks and filter into a subset.
 void Filter::subset(const std::vector<Task>& input, std::vector<Task>& output) {
   Timer timer;
+  const auto load_before = Context::getContext().time_load_us;
   _startCount = (int)input.size();
 
   Context::getContext().cli2.prepareFilter();
 
-  std::vector<std::pair<std::string, Lexer::Type>> precompiled;
-  for (auto& a : Context::getContext().cli2._args)
-    if (a.hasTag("FILTER")) precompiled.emplace_back(a.getToken(), a._lextype);
-
-  if (precompiled.size()) {
-    Eval eval;
-    eval.addSource(domSource);
-
-    // Debug output from Eval during compilation is useful.  During evaluation
-    // it is mostly noise.
-    eval.debug(Context::getContext().config.getInteger("debug.parser") >= 3 ? true : false);
-    eval.compileExpression(precompiled);
-
-    for (auto& task : input) {
-      // Set up context for any DOM references.
-      auto currentTask = Context::getContext().withCurrentTask(&task);
-
-      Variant var;
-      eval.evaluateCompiledExpression(var);
-      if (var.get_bool()) output.push_back(task);
-    }
-
-    eval.debug(false);
-  } else
-    output = input;
+  filter_to_tasks(input, output);
 
   _endCount = (int)output.size();
   Context::getContext().debug(
       format("Filtered {1} tasks --> {2} tasks [list subset]", _startCount, _endCount));
-  Context::getContext().time_filter_us += timer.total_us();
+  Context::getContext().time_filter_us +=
+      timer.total_us() - (Context::getContext().time_load_us - load_before);
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 // Take the set of all tasks and filter into a subset.
 void Filter::subset(std::vector<Task>& output) {
   Timer timer;
+  const auto load_before = Context::getContext().time_load_us;
   Context::getContext().cli2.prepareFilter();
 
   std::vector<std::pair<std::string, Lexer::Type>> precompiled;
   for (auto& a : Context::getContext().cli2._args)
     if (a.hasTag("FILTER")) precompiled.emplace_back(a.getToken(), a._lextype);
 
-  // Shortcut indicates that only pending.data needs to be loaded.
+  // Shortcut indicates that only tasks in the working set are loaded.
   bool shortcut = false;
 
   if (precompiled.size()) {
-    Timer timer_pending;
-    auto pending = Context::getContext().tdb2.pending_tasks();
-    Context::getContext().time_filter_us -= timer_pending.total_us();
+    const auto& pending = Context::getContext().tdb2.pending_tasks();
     _startCount = (int)pending.size();
 
-    Eval eval;
-    eval.addSource(domSource);
-
-    // Debug output from Eval during compilation is useful.  During evaluation
-    // it is mostly noise.
-    eval.debug(Context::getContext().config.getInteger("debug.parser") >= 3 ? true : false);
-    eval.compileExpression(precompiled);
-
     output.clear();
-    for (auto& task : pending) {
-      // Set up context for any DOM references.
-      auto currentTask = Context::getContext().withCurrentTask(&task);
 
-      Variant var;
-      eval.evaluateCompiledExpression(var);
-      if (var.get_bool()) output.push_back(task);
-    }
+    filter_to_tasks(pending, output);
 
     shortcut = pendingOnly();
     if (!shortcut) {
-      Timer timer_completed;
-      auto completed = Context::getContext().tdb2.completed_tasks();
-      Context::getContext().time_filter_us -= timer_completed.total_us();
+      const auto& completed = Context::getContext().tdb2.completed_tasks();
       _startCount += (int)completed.size();
 
-      for (auto& task : completed) {
-        // Set up context for any DOM references.
-        auto currentTask = Context::getContext().withCurrentTask(&task);
-
-        Variant var;
-        eval.evaluateCompiledExpression(var);
-        if (var.get_bool()) output.push_back(task);
-      }
+      filter_to_tasks(completed, output);
     }
-
-    eval.debug(false);
   } else {
     safety();
 
-    Timer pending_completed;
     output = Context::getContext().tdb2.all_tasks();
-    Context::getContext().time_filter_us -= pending_completed.total_us();
+    _startCount = (int)output.size();
   }
 
   _endCount = (int)output.size();
   Context::getContext().debug(format("Filtered {1} tasks --> {2} tasks [{3}]", _startCount,
                                      _endCount, (shortcut ? "pending only" : "all tasks")));
-  Context::getContext().time_filter_us += timer.total_us();
+  Context::getContext().time_filter_us +=
+      timer.total_us() - (Context::getContext().time_load_us - load_before);
 }
 
-////////////////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////////////
 bool Filter::hasFilter() const {
   for (const auto& a : Context::getContext().cli2._args)
     if (a.hasTag("FILTER")) return true;
 
   return false;
 }
 
-////////////////////////////////////////////////////////////////////////////////
-// If the filter contains no 'or', 'xor' or 'not' operators, and only includes
-// status values 'pending', 'waiting' or 'recurring', then the filter is
-// guaranteed to only need data from pending.data.
-bool Filter::pendingOnly() const {
-  // When GC is off, there are no shortcuts.
-  if (!Context::getContext().config.getBoolean("gc")) return false;
+/////////////////////////////////////////////////////////////////////////////////
+// Evaluates a pre-parsed filter against a set of tasks and stores their indices
+// from the vector. The filter is parsed with prepareFilter(), but this
+// function does not call that or safety() itself - callers are expected to do so.
+void Filter::filter_to_indices(const std::vector<Task>& pending, std::vector<int>& indices) const {
+  std::vector<std::pair<std::string, Lexer::Type>> precompiled;
+  for (auto& a : Context::getContext().cli2._args)
+    if (a.hasTag("FILTER")) precompiled.emplace_back(a.getToken(), a._lextype);
 
-  // To skip loading completed.data, there should be:
-  // - 'status' in filter
-  // - no 'completed'
-  // - no 'deleted'
-  // - no 'xor'
-  // - no 'or'
-  int countStatus = 0;
-  int countPending = 0;
-  int countWaiting = 0;
-  int countRecurring = 0;
-  int countId = (int)Context::getContext().cli2._id_ranges.size();
-  int countUUID = (int)Context::getContext().cli2._uuid_list.size();
-  int countOr = 0;
-  int countXor = 0;
-  int countNot = 0;
-  bool pendingTag = false;
-  bool activeTag = false;
-
-  for (const auto& a : Context::getContext().cli2._args) {
-    if (a.hasTag("FILTER")) {
-      std::string raw = a.attribute("raw");
-      std::string canonical = a.attribute("canonical");
-
-      if (a._lextype == Lexer::Type::op && raw == "or") ++countOr;
-      if (a._lextype == Lexer::Type::op && raw == "xor") ++countXor;
-      if (a._lextype == Lexer::Type::op && raw == "not") ++countNot;
-      if (a._lextype == Lexer::Type::dom && canonical == "status") ++countStatus;
-      if (raw == "pending") ++countPending;
-      if (raw == "waiting") ++countWaiting;
-      if (raw == "recurring") ++countRecurring;
+  if (precompiled.empty()) {
+    indices.reserve(pending.size());
+    for (int i = 0; i < (int)pending.size(); ++i) indices.push_back(i);
+  } else {
+    Eval eval;
+    eval.addSource(domSource);
+    eval.debug(Context::getContext().config.getInteger("debug.parser") >= 3);
+    eval.compileExpression(precompiled);
+    for (int i = 0; i < (int)pending.size(); ++i) {
+      auto currentTask = Context::getContext().withCurrentTask(&pending[i]);
+      Variant var;
+      eval.evaluateCompiledExpression(var);
+      if (var.get_bool()) indices.push_back(i);
     }
+    eval.debug(false);
   }
+}
 
-  for (const auto& word : Context::getContext().cli2._original_args) {
-    if (word.attribute("raw") == "+PENDING") pendingTag = true;
-    if (word.attribute("raw") == "+ACTIVE") activeTag = true;
+////////////////////////////////////////////////////////////////////////////////
+// Like filter_to_indices, but copies matched tasks into the output.
+void Filter::filter_to_tasks(const std::vector<Task>& input, std::vector<Task>& output) const {
+  std::vector<std::pair<std::string, Lexer::Type>> precompiled;
+  for (auto& a : Context::getContext().cli2._args)
+    if (a.hasTag("FILTER")) precompiled.emplace_back(a.getToken(), a._lextype);
+
+  if (precompiled.empty()) {
+    output = input;
+  } else {
+    Eval eval;
+    eval.addSource(domSource);
+    eval.debug(Context::getContext().config.getInteger("debug.parser") >= 3);
+    eval.compileExpression(precompiled);
+    for (auto& task : input) {
+      auto currentTask = Context::getContext().withCurrentTask(&task);
+      Variant var;
+      eval.evaluateCompiledExpression(var);
+      if (var.get_bool()) output.push_back(task);
+    }
+    eval.debug(false);
   }
+}
 
-  if (countUUID) return false;
+////////////////////////////////////////////////////////////////////////////////
+// Recognizes pending only constraints that allow us to use the shortcut.
 
-  if (countOr || countXor || countNot) return false;
+bool Filter::pendingOnly() const {
+  if (!Context::getContext().config.getBoolean("gc")) return false;
 
-  if (pendingTag || activeTag) return true;
+  const auto& cli = Context::getContext().cli2;
+  if (!cli._uuid_list.empty()) return false;
 
-  if (countStatus) {
-    if (!countPending && !countWaiting && !countRecurring) return false;
+  std::vector<const A2*> filter_args;
+  for (const auto& arg : cli._args) {
+    if (!arg.hasTag("FILTER")) continue;
 
-    return true;
+    filter_args.push_back(&arg);
   }
 
-  if (countId) return true;
+  const auto requires_pending = [&](const auto& self, size_t begin, size_t end) -> bool {
+    if (begin == end) return false;
+    int depth = 0;
+    size_t first_close = end;
+    std::vector<size_t> conjunctions;
+    for (size_t i = begin; i < end; ++i) {
+      const auto& arg = *filter_args[i];
+      if (arg._lextype != Lexer::Type::op) continue;
+      const auto& op = arg.attribute("raw");
+      if (op == "(") {
+        ++depth;
+      } else if (op == ")") {
+        if (--depth < 0) return false;
+        if (depth == 0 && first_close == end) first_close = i;
+      } else if (depth == 0) {
+        if (op == "or" || op == "xor") return false;
+        if (op == "and") conjunctions.push_back(i);
+      }
+    }
 
-  return false;
+    if (depth != 0) return false;
+    if (filter_args[begin]->_lextype == Lexer::Type::op &&
+        filter_args[begin]->attribute("raw") == "(" && first_close == end - 1)
+      return self(self, begin + 1, end - 1);
+
+    if (!conjunctions.empty()) {
+      bool required = false;
+      for (auto boundary : conjunctions) {
+        required |= self(self, begin, boundary);
+        begin = boundary + 1;
+      }
+      return self(self, begin, end) || required;
+    }
+
+    if (end - begin != 3) return false;
+    const auto& left = *filter_args[begin];
+    const auto& op = *filter_args[begin + 1];
+    const auto& right = *filter_args[begin + 2];
+    if (left._lextype != Lexer::Type::dom || op._lextype != Lexer::Type::op) return false;
+    const auto& operation = op.attribute("raw");
+    const auto& value = right.attribute("raw");
+
+    if (left.attribute("raw") == "id" && right._lextype == Lexer::Type::number &&
+        (operation == "=" || operation == "==" || operation == ">=") &&
+        value.find_first_not_of("0123456789") == std::string::npos &&
+        value.find_first_not_of('0') != std::string::npos)
+      return true;
+    if (right._lextype != Lexer::Type::string) return false;
+    return (left.attribute("canonical") == "status" && (operation == "=" || operation == "==") &&
+            (value == "pending" || value == "waiting" || value == "recurring")) ||
+           (left.attribute("raw") == "tags" && operation == "_hastag_" &&
+            (value == "PENDING" || value == "ACTIVE" || value == "READY" || value == "WAITING"));
+  };
+
+  return requires_pending(requires_pending, 0, filter_args.size());
 }
 
 ////////////////////////////////////////////////////////////////////////////////

src/Filter.h

@@ -40,6 +40,13 @@ class Filter {
   void subset(std::vector<Task>&);
   bool hasFilter() const;
   bool pendingOnly() const;
+
+  // These evaluate already-prepared filters (ie. caller calls
+  // cli2.prepareFilter() and safety()). filter_to_indices() stores matching
+  // indices, filter_to_tasks() copies task objects.
+  void filter_to_indices(const std::vector<Task>&, std::vector<int>&) const;
+  void filter_to_tasks(const std::vector<Task>&, std::vector<Task>&) const;
+
   void safety() const;
   void disableSafety();
 

src/Hooks.cpp

@@ -30,6 +30,7 @@
 #include <Hooks.h>
 
 #include <algorithm>
+#include <utility>
 // If <iostream> is included, put it after <stdio.h>, because it includes
 // <stdio.h>, and therefore would ignore the _WITH_GETLINE.
 #ifdef FREEBSD
@@ -313,7 +314,9 @@ void Hooks::onModify(Task& before, Task& after) const {
       }
     }
 
-    after = Task(input[1]);
+    Task updated(input[1]);
+    updated.copyTransientState(after);
+    after = std::move(updated);
   }
 
   Context::getContext().time_hooks_us += timer.total_us();
@@ -322,6 +325,9 @@ void Hooks::onModify(Task& before, Task& after) const {
 ////////////////////////////////////////////////////////////////////////////////
 std::vector<std::string> Hooks::list() const { return _scripts; }
 
+////////////////////////////////////////////////////////////////////////////////
+bool Hooks::hasOnModify() const { return _enabled && !scripts("on-modify").empty(); }
+
 ////////////////////////////////////////////////////////////////////////////////
 std::vector<std::string> Hooks::scripts(const std::string& event) const {
   std::vector<std::string> matching;

src/Hooks.h

@@ -41,6 +41,7 @@ class Hooks {
   void onExit() const;
   void onAdd(Task&) const;
   void onModify(Task&, Task&) const;
+  bool hasOnModify() const;
   std::vector<std::string> list() const;
 
  private: