[traffic] Feature-complete cache persistence, including decoded coloring

Signed-off-by: mvglasow <michael -at- vonglasow.com>

map/traffic_manager.cpp

@@ -13,7 +13,6 @@
 #include "platform/platform.hpp"
 
 #include "traffxml/traff_model_xml.hpp"
-#include "traffxml/traff_storage.hpp"
 
 using namespace std::chrono;
 
@@ -43,6 +42,16 @@ auto constexpr kDrapeUpdateInterval = seconds(10);
  * Interval at which the traffic observer gets traffic updates while messages are being processed.
  */
 auto constexpr kObserverUpdateInterval = minutes(1);
+
+/**
+ * Interval at which the message cache file is updated while messages are being processed.
+ */
+auto constexpr kStorageUpdateInterval = minutes(1);
+
+/**
+ * File name at which traffic data is persisted.
+ */
+auto constexpr kTrafficXMLFileName = "traffic.xml";
 } // namespace
 
 TrafficManager::CacheEntry::CacheEntry()
@@ -125,14 +134,28 @@ void TrafficManager::SetStateListener(TrafficStateChangedFn const & onStateChang
 
 void TrafficManager::SetEnabled(bool enabled)
 {
+  /*
+   * Whether to notify interested parties that traffic data has been updated.
+   * This is based on the return value of RestoreCache().
+   */
+  bool notifyUpdate = false;
   {
     std::lock_guard<std::mutex> lock(m_mutex);
     if (enabled == IsEnabled())
        return;
-    if (enabled && !m_traffDecoder)
+    if (enabled)
+    {
+      if (!m_traffDecoder)
         // deferred decoder initialization (requires maps to be loaded)
         m_traffDecoder = make_unique<traffxml::DefaultTraffDecoder>(m_dataSource, m_countryInfoGetterFn,
                                                                     m_countryParentNameGetterFn, m_messageCache);
+      if (!m_storage && !IsTestMode())
+      {
+        m_storage = make_unique<traffxml::LocalStorage>(kTrafficXMLFileName);
+        notifyUpdate = RestoreCache();
+        m_lastStorageUpdate = steady_clock::now();
+      }
+    }
     ChangeState(enabled ? TrafficState::Enabled : TrafficState::Disabled);
   }
 
@@ -140,6 +163,10 @@ void TrafficManager::SetEnabled(bool enabled)
 
   if (enabled)
   {
+    if (notifyUpdate)
+      OnTrafficDataUpdate();
+    else
+      // TODO After sorting out invalidation, figure out if we need that here.
       Invalidate();
     m_canSetMode = false;
   }
@@ -227,6 +254,28 @@ void TrafficManager::OnRecoverSurface()
   Resume();
 }
 
+/*
+ * TODO Revisit invalidation logic.
+ * We currently invalidate when enabling and resuming, and when a new MWM file is downloaded
+ * (behavior inherited from MapsWithMe).
+ * Traffic data in MapsWithMe was a set of pre-decoded messages per MWM; the whole set would get
+ * re-fetched periodically. Invalidation meant discarding and re-fetching all traffic data.
+ * This logic is different for TraFF:
+ * - Messages expire individually or get replaced by updates, thus there is hardly ever a reason
+ *   to discard messages.
+ * - Messages are decoded into segments in the app. Discarding decoded segments may be needed on
+ *   a per-message basis for the following reasons:
+ *   - the message is replaced by a new one and the location or traffic situation has changed
+ *     (this is dealt with as part of the message update process)
+ *   - one of the underlying MWMs has been updated to a new version
+ *   - a new MWM has been added, and a message location that previously could not be decoded
+ *     completely now can
+ * The sensible equivalent in TraFF would be to discard and re-generate decoded locations, and
+ * possibly poll for updates. Discarding and re-generating decoded locations could be done
+ * selectively:
+ * - compare map versions of decoded segments to current map version
+ * - figure out when a new map has been added, and which segments are affected by it
+ */
 void TrafficManager::Invalidate()
 {
   if (!IsEnabled())
@@ -369,6 +418,56 @@ bool TrafficManager::IsSubscribed()
   return !m_subscriptionId.empty();
 }
 
+bool TrafficManager::RestoreCache()
+{
+  ASSERT(m_storage, ("m_storage cannot be null"));
+  pugi::xml_document document;
+  if (!m_storage->Load(document))
+  {
+    LOG(LWARNING, ("Failed to reload cache from storage"));
+    return false;
+  }
+
+  traffxml::TraffFeed feedIn;
+  traffxml::TraffFeed feedOut;
+  bool hasDecoded = false;
+  bool hasUndecoded = false;
+  if (traffxml::ParseTraff(document, std::nullopt /* dataSource */, feedIn))
+  {
+    while (!feedIn.empty())
+    {
+      traffxml::TraffMessage message;
+      std::swap(message, feedIn.front());
+      feedIn.erase(feedIn.begin());
+
+      if (!message.IsExpired(traffxml::IsoTime::Now()))
+      {
+        if (!message.m_decoded.empty())
+        {
+          hasDecoded = true;
+          // store message in cache
+          m_messageCache.insert_or_assign(message.m_id, message);
+        }
+        else
+        {
+          hasUndecoded = true;
+          // message needs decoding, prepare to enqueue
+          feedOut.push_back(message);
+        }
+      }
+    }
+    if (!feedOut.empty())
+      m_feedQueue.insert(m_feedQueue.begin(), feedOut);
+    // update notification is caller’s responsibility
+    return hasDecoded && !hasUndecoded;
+  }
+  else
+  {
+    LOG(LWARNING, ("An error occurred parsing the cache file"));
+  }
+  return false;
+}
+
 // TODO make this work with multiple sources (e.g. Android)
 // TODO deal with subscriptions rejected by the server (delete, resubscribe)
 bool TrafficManager::Poll()
@@ -385,7 +484,7 @@ bool TrafficManager::Poll()
 
   std::setlocale(LC_ALL, "en_US.UTF-8");
   traffxml::TraffFeed feed;
-  if (traffxml::ParseTraff(document, feed))
+  if (traffxml::ParseTraff(document, std::nullopt /* dataSource */, feed))
   {
     {
       std::lock_guard<std::mutex> lock(m_mutex);
@@ -784,13 +883,34 @@ void TrafficManager::OnTrafficDataUpdate()
   // Whether to notify the observer of the update.
   bool notifyObserver = (feedQueueEmpty);
 
+  // Whether to update the cache file.
+  bool updateStorage = (feedQueueEmpty);
+
   if (!feedQueueEmpty)
   {
     auto const currentTime = steady_clock::now();
     auto const drapeAge = currentTime - m_lastDrapeUpdate;
     auto const observerAge = currentTime - m_lastObserverUpdate;
+    auto const storageAge = currentTime - m_lastStorageUpdate;
     notifyDrape = (drapeAge >= kDrapeUpdateInterval);
     notifyObserver = (observerAge >= kObserverUpdateInterval);
+    updateStorage = (storageAge >= kStorageUpdateInterval);
+  }
+
+  if (!m_storage || IsTestMode())
+    updateStorage = false;
+
+  if (updateStorage)
+  {
+    std::lock_guard<std::mutex> lock(m_mutex);
+
+    pugi::xml_document document;
+
+    traffxml::GenerateTraff(m_messageCache, document);
+    if (!m_storage->Save(document))
+      LOG(LWARNING, ("Storing message cache to file failed."));
+
+    m_lastStorageUpdate = steady_clock::now();
   }
 
   if (!notifyDrape && !notifyObserver)

map/traffic_manager.hpp

@@ -13,6 +13,7 @@
 
 #include "traffxml/traff_decoder.hpp"
 #include "traffxml/traff_model.hpp"
+#include "traffxml/traff_storage.hpp"
 
 #include "geometry/point2d.hpp"
 #include "geometry/polyline2d.hpp"
@@ -356,6 +357,22 @@ private:
    */
   bool IsSubscribed();
 
+  /**
+   * @brief Restores the message cache from file storage.
+   *
+   * @note The caller must lock `m_mutex` prior to calling this function, as it makes unprotected
+   * changes to shared data structures.
+   *
+   * @note The return value indicates whether actions related to a traffic update should be taken,
+   * such as notifying the routing and drape engine. It is true if at least one message with a
+   * decoded location was read, and no messages without decoded locations. If messages without a
+   * decoded location were read, the return value is false, as the location decoding will trigger
+   * updates by itself. If errors occurred and no messages are read, the return value is also false.
+   *
+   * @return True if a traffic update needs to be sent, false if not
+   */
+  bool RestoreCache();
+
   /**
    * @brief Polls the traffic service for updates.
    *
@@ -667,6 +684,11 @@ private:
    */
   std::chrono::time_point<std::chrono::steady_clock> m_lastObserverUpdate;
 
+  /**
+   * @brief When the cache file was last updated.
+   */
+  std::chrono::time_point<std::chrono::steady_clock> m_lastStorageUpdate;
+
   /**
    * @brief Whether active MWMs have changed since the last request.
    */
@@ -706,6 +728,13 @@ private:
    */
   std::map<std::string, traffxml::TraffMessage> m_messageCache;
 
+  /**
+   * @brief The storage instance.
+   *
+   * Used to persist the TraFF message cache between sessions.
+   */
+  std::unique_ptr<traffxml::LocalStorage> m_storage;
+
   /**
    * @brief The TraFF decoder instance.
    *

traffxml/traff_assessment_tool/mainwindow.cpp

@@ -367,7 +367,7 @@ void MainWindow::OnOpenTrafficSample()
   std::setlocale(LC_ALL, "en_US.UTF-8");
   traffxml::TraffFeed feed;
   traffxml::TraffFeed shiftedFeed;
-  if (traffxml::ParseTraff(document, feed))
+  if (traffxml::ParseTraff(document, std::nullopt /* dataSource */, feed))
   {
     for (auto message : feed)
     {

traffxml/traff_model_xml.cpp

@@ -122,6 +122,42 @@ const boost::bimap<std::string, EventType> kEventTypeMap = MakeBimap<std::string
   // TODO Security*, Transport*, Weather* (not in enum yet)
 });
 
+const boost::bimap<std::string, traffic::SpeedGroup> kSpeedGroupMap = MakeBimap<std::string, traffic::SpeedGroup>({
+  {"G0", traffic::SpeedGroup::G0},
+  {"G1", traffic::SpeedGroup::G1},
+  {"G2", traffic::SpeedGroup::G2},
+  {"G3", traffic::SpeedGroup::G3},
+  {"G4", traffic::SpeedGroup::G4},
+  {"G5", traffic::SpeedGroup::G5},
+  {"TEMP_BLOCK", traffic::SpeedGroup::TempBlock},
+  {"UNKNOWN", traffic::SpeedGroup::Unknown}
+});
+
+/**
+ * @brief Retrieves an integer value from an attribute.
+ *
+ * @param attribute The XML attribute to retrieve.
+ * @param value The variable which will receive the value, must be of an integer type
+ * @return `true` on success, `false` if the attribute is not set or does not contain an integer value.
+ */
+template <typename Value>
+bool IntegerFromXml(pugi::xml_attribute const & attribute, Value & value)
+{
+  if (attribute.empty())
+    return false;
+  try
+  {
+    value = static_cast<Value>(is_signed<Value>::value
+                               ? std::stoll(attribute.as_string())
+                               : std::stoull(attribute.as_string()));
+    return true;
+  }
+  catch (std::invalid_argument const& ex)
+  {
+    return false;
+  }
+}
+
 /**
  * @brief Retrieves an integer value from an attribute.
  *
@@ -581,8 +617,6 @@ void LocationToXml(TraffLocation const & location, pugi::xml_node & node)
     PointToXml(location.m_notVia.value(), "not_via", node);
   if (location.m_to)
     PointToXml(location.m_to.value(), "to", node);
-
-  // TODO decoded segments
 }
 
 /**
@@ -734,13 +768,180 @@ bool EventsFromXml(pugi::xml_node const & node, std::vector<TraffEvent> & events
   return result;
 }
 
+/**
+ * @brief Retrieves a coloring segment (segment with speed group) from XML
+ * @param node The `segment` node
+ * @param coloring The coloring to which the segment will be added.
+ * @return true if each segment was parsed successfully, false if errors occurred (in this case,
+ * the decoded coloring for this message should be discarded and regenerated from scratch)
+ */
+bool SegmentFromXml(pugi::xml_node const & node,
+               std::map<traffic::TrafficInfo::RoadSegmentId, traffic::SpeedGroup> & coloring)
+{
+  uint32_t fid;
+  uint16_t idx;
+  uint8_t dir;
+  if (IntegerFromXml(node.attribute("fid"), fid)
+      && IntegerFromXml(node.attribute("idx"), idx)
+      && IntegerFromXml(node.attribute("dir"), dir))
+  {
+    traffic::TrafficInfo::RoadSegmentId segment(fid, idx, dir);
+    traffic::SpeedGroup sg = traffic::SpeedGroup::Unknown;
+    if (EnumFromXml(node.attribute("speed_group"), sg, kSpeedGroupMap))
+      coloring[segment] = sg;
+    else
+    {
+      LOG(LWARNING, ("missing or invalid speed group for", segment, "(aborting)"));
+      return false;
+    }
+  }
+  else
+  {
+    LOG(LWARNING, ("segment with incomplete information (fid, idx, dir), aborting"));
+    return false;
+  }
+  return true;
+}
+
+/**
+ * @brief Retrieves coloring for a single MWM from XML.
+ *
+ * This function returns false if errors occurred during decoding (due to invalid data), or if the
+ * data version to which the segments refer does not coincide with the currently used version of the
+ * corresponding MWM. In this case, the entire coloring for this message should be discarded and the
+ * message should be decoded from scratch.
+ *
+ * @todo Errors in segments are currently not considered, i.e. this function may return true even if
+ * one or more segments have errors.
+ *
+ * @param node The `coloring` node.
+ * @param dataSource The data source for coloring.
+ * @param decoded Receives the decoded global coloring.
+ * @return whether the decoded segments can be used, see description
+ */
+bool ColoringFromXml(pugi::xml_node const & node, DataSource const & dataSource,
+                     MultiMwmColoring & decoded)
+{
+  std::string countryName;
+  if (!StringFromXml(node.attribute("country_name"), countryName))
+  {
+    LOG(LWARNING, ("coloring element without coutry_name attribute, skipping"));
+    return false;
+  }
+  auto const & mwmId = dataSource.GetMwmIdByCountryFile(platform::CountryFile(countryName));
+  if (!mwmId.IsAlive())
+  {
+    LOG(LWARNING, ("Can’t get MWM ID for country", countryName, "(skipping)"));
+    return false;
+  }
+
+  uint64_t version = 0;
+  if (!IntegerFromXml(node.attribute("version"), version))
+  {
+    LOG(LWARNING, ("Can’t get version for country", countryName, "(skipping)"));
+    return false;
+  }
+  else if (version != mwmId.GetInfo()->GetVersion())
+  {
+    LOG(LINFO, ("XML data for country", countryName, "has version", version, "while MWM has", mwmId.GetInfo()->GetVersion(), "(skipping)"));
+    return false;
+  }
+
+  auto const segmentNodes = node.select_nodes("./segment");
+
+  if (segmentNodes.empty())
+    return true;
+
+  std::map<traffic::TrafficInfo::RoadSegmentId, traffic::SpeedGroup> coloring;
+
+  for (auto const & segmentXpathNode : segmentNodes)
+  {
+    auto const & segmentNode = segmentXpathNode.node();
+    if (!SegmentFromXml(segmentNode, coloring))
+      return false;
+  }
+
+  if (!coloring.empty())
+    decoded[mwmId] = coloring;
+
+  return true;
+}
+
+/**
+ * @brief Stores coloring for an indidual MWM in an XML node.
+ *
+ * The vaues of `mwmId` will be added to `node` as attributes. The segments and their traffic group
+ * will be added to `node` as child nodes.
+ *
+ * @param mwmId
+ * @param coloring
+ * @param node The `coloring` node to store the coloring in.
+ */
+void ColoringToXml(MwmSet::MwmId const & mwmId,
+                   std::map<traffic::TrafficInfo::RoadSegmentId, traffic::SpeedGroup> const & coloring,
+                   pugi::xml_node node)
+{
+  node.append_attribute("country_name").set_value(mwmId.GetInfo()->GetCountryName());
+  node.append_attribute("version").set_value(mwmId.GetInfo()->GetVersion());
+  for (auto & [segId, sg] : coloring)
+  {
+    auto segNode = node.append_child("segment");
+    segNode.append_attribute("fid").set_value(segId.GetFid());
+    segNode.append_attribute("idx").set_value(segId.GetIdx());
+    segNode.append_attribute("dir").set_value(segId.GetDir());
+    EnumToXml(sg, "speed_group", segNode, kSpeedGroupMap);
+  }
+}
+
+/**
+ * @brief Retrieves global coloring from XML.
+ *
+ * If the MWM version does not match for at least one MWM, no coloring is decoded (`decoded` is
+ * empty after this function returns) and the message needs to be decoded from scratch.
+ *
+ * @param node The `mwm_coloring` node.
+ * @param dataSource The data source for coloring, see `ParseTraff()`.
+ * @param decoded Receives the decoded global coloring.
+ */
+void AllMwmColoringFromXml(pugi::xml_node const & node,
+                           std::optional<std::reference_wrapper<const DataSource>> dataSource,
+                           MultiMwmColoring & decoded)
+{
+  if (!node)
+    return;
+
+  if (!dataSource)
+  {
+    LOG(LWARNING, ("Message has mwm_coloring but it cannot be parsed as no data source was specified"));
+    return;
+  }
+
+  auto const coloringNodes = node.select_nodes("./coloring");
+
+  if (coloringNodes.empty())
+    return;
+
+  for (auto const & coloringXpathNode : coloringNodes)
+  {
+    auto const & coloringNode = coloringXpathNode.node();
+    if (!ColoringFromXml(coloringNode, dataSource->get(), decoded))
+    {
+      decoded.clear();
+      return;
+    }
+  }
+}
+
 /**
  * @brief Retrieves a TraFF message from an XML element.
  * @param node The XML element to retrieve (`message`).
+ * @param dataSource The data source for coloring, see `ParseTraff()`.
  * @param message Receives the message.
  * @return `true` on success, `false` if the node does not exist or does not contain valid message data.
  */
-bool MessageFromXml(pugi::xml_node const & node, TraffMessage & message)
+bool MessageFromXml(pugi::xml_node const & node,
+                    std::optional<std::reference_wrapper<const DataSource>> dataSource,
+                    TraffMessage & message)
 {
   if (!StringFromXml(node.attribute("id"), message.m_id))
   {
@@ -779,7 +980,9 @@ bool MessageFromXml(pugi::xml_node const & node, TraffMessage & message)
   if (!message.m_cancellation)
   {
     message.m_location.emplace();
-    if (!LocationFromXml(node.child("location"), message.m_location.value()))
+    if (LocationFromXml(node.child("location"), message.m_location.value()))
+      AllMwmColoringFromXml(node.child("mwm_coloring"), dataSource, message.m_decoded);
+    else
     {
       message.m_location.reset();
       LOG(LWARNING, ("Message", message.m_id, "has no location but is not a cancellation message"));
@@ -842,9 +1045,21 @@ void MessageToXml(TraffMessage const & message, pugi::xml_node node)
       EventToXml(event, eventNode);
     }
   }
+
+  if (!message.m_decoded.empty())
+  {
+    auto allMwmColoringNode = node.append_child("mwm_coloring");
+    for (auto & [mwmId, coloring] : message.m_decoded)
+    {
+      auto coloringNode = allMwmColoringNode.append_child("coloring");
+      ColoringToXml(mwmId, coloring, coloringNode);
+    }
+  }
 }
 
-bool ParseTraff(pugi::xml_document const & document, TraffFeed & feed)
+bool ParseTraff(pugi::xml_document const & document,
+                std::optional<std::reference_wrapper<const DataSource>> dataSource,
+                TraffFeed & feed)
 {
   bool result = false;
 
@@ -859,7 +1074,7 @@ bool ParseTraff(pugi::xml_document const & document, TraffFeed & feed)
   {
     auto const messageNode = xpathNode.node();
     TraffMessage message;
-    if (MessageFromXml(messageNode, message))
+    if (MessageFromXml(messageNode, dataSource, message))
     {
       feed.push_back(message);
       result = true;

traffxml/traff_model_xml.hpp

@@ -4,6 +4,9 @@
 
 #include "geometry/rect2d.hpp"
 
+#include "indexer/data_source.hpp"
+
+#include <functional>
 #include <string>
 #include <vector>
 
@@ -34,14 +37,25 @@ namespace traffxml
  * Parsing the feed will report failure if all its messages fail to parse, but not if it has no
  * messages.
  *
- * @note Custom elements and attributes which are not part of the TraFF specification are currently
- * ignored. Future versions may process certain custom elements.
+ * In addition to the TraFF specification, we also use a custom extension, `mwm_coloring`, which is
+ * a child of `message` and holds decoded traffic coloring. In order to parse it, `dataSource` must
+ * be specified. If `dataSource` is `nullopt`, coloring will be ignored. It is recommended to pass
+ * `dataSource` if, and only if, parsing an XML stream that is expected to contain traffic coloring.
+ *
+ * @note To pass a reference to the framework data source (assuming the `framework` is the framework
+ * instance), use `std::cref(framework.GetDataSource())`.
+ *
+ * @note Custom elements and attributes which are not part of the TraFF specification, other than
+ * `mwm_coloring`, are ignored.
  *
  * @param document The XML document from which to retrieve the messages.
+ * @param dataSource The data source for coloring, see description.
  * @param feed Receives the TraFF feed.
  * @return `true` on success, `false` on failure.
  */
-bool ParseTraff(pugi::xml_document const & document, TraffFeed & feed);
+bool ParseTraff(pugi::xml_document const & document,
+                std::optional<std::reference_wrapper<const DataSource>> dataSource,
+                TraffFeed & feed);
 
 /**
  * @brief Generates XML from a TraFF feed.

traffxml/traff_storage.hpp

@@ -45,7 +45,7 @@ public:
    * interpreted relative to the platform-specific path; absolute paths are not supported as some
    * platforms restrict applications’ access to files outside their designated path.
    */
-  LocalStorage(std::string & fileName)
+  LocalStorage(std::string const & fileName)
     : m_fileName(fileName)
   {}