Trigger e2e tests after building server debs, rename field for storin…

…g statentries, add dedicated registry for Flow classes.

appveyor/linux/appveyor.yml

@@ -1,3 +1,7 @@
+branches:
+  only:
+    - master
+
 image: ubuntu
 
 environment:

appveyor/linux/run_e2e_tests.sh

@@ -8,6 +8,15 @@
 
 set -ex
 
+readonly FLAKY_TESTS_ARR=(\
+  TestClientInterrogate.runTest \
+  TestFileFinderOSHomedir.runTest \
+  TestCheckRunner.runTest \
+  TestRecursiveListDirectoryLinuxDarwin.runTest \
+)
+# Convert array to string (comma-separated).
+readonly FLAKY_TESTS="$(IFS=,;echo "${FLAKY_TESTS_ARR[*]}")"
+
 function fatal() {
   >&2 echo "Error: ${1}"
   exit 1
@@ -25,7 +34,12 @@ echo -e "Logging.engines: stderr,file\nLogging.verbose: True\nClient.poll_max: 5
 
 systemctl restart grr
 
-grr_end_to_end_tests --api_password "${GRR_ADMIN_PASS}" --client_id "${CLIENT_ID}" --flow_timeout_secs 60 --verbose 2>&1 | tee e2e.log
+grr_end_to_end_tests --verbose \
+  --api_password "${GRR_ADMIN_PASS}" \
+  --client_id "${CLIENT_ID}" \
+  --flow_timeout_secs 60 \
+  --blacklisted_tests "${FLAKY_TESTS}" \
+  2>&1 | tee e2e.log
 
 if [[ ! -z "$(cat e2e.log | grep -F '[ FAIL ]')" ]]; then
   fatal 'End-to-end tests failed.'

grr/lib/rdfvalues/cronjobs.py

@@ -2,13 +2,13 @@
 """RDFValues for cronjobs."""
 
 from grr.lib import rdfvalue
+from grr.lib import registry
 from grr.lib.rdfvalues import flows as rdf_flows
 from grr.lib.rdfvalues import protodict as rdf_protodict
 from grr.lib.rdfvalues import structs
 from grr_response_proto import flows_pb2
 from grr_response_proto import jobs_pb2
 from grr_response_proto import sysinfo_pb2
-from grr.server.grr_response_server import flow
 
 
 class CronTabEntry(structs.RDFProtoStruct):
@@ -38,10 +38,8 @@ class CreateCronJobFlowArgs(structs.RDFProtoStruct):
 
   def GetFlowArgsClass(self):
     if self.flow_runner_args.flow_name:
-      flow_cls = flow.GRRFlow.classes.get(self.flow_runner_args.flow_name)
-      if flow_cls is None:
-        raise ValueError("Flow '%s' not known by this implementation." %
-                         self.flow_runner_args.flow_name)
+      flow_cls = registry.FlowRegistry.FlowClassByName(
+          self.flow_runner_args.flow_name)
 
       # The required protobuf for this class is in args_type.
       return flow_cls.args_type

grr/lib/rdfvalues/objects.py

@@ -255,6 +255,7 @@ class PathInfo(structs.RDFProtoStruct):
   rdf_deps = [
       rdfvalue.RDFDatetime,
       rdf_client.StatEntry,
+      rdf_crypto.Hash,
   ]
 
   def __init__(self, *args, **kwargs):
@@ -397,8 +398,8 @@ def UpdateFrom(self, src):
     if src.HasField("stat_entry"):
       self.stat_entry = src.stat_entry
 
-    self.last_path_history_timestamp = max(self.last_path_history_timestamp,
-                                           src.last_path_history_timestamp)
+    self.last_stat_entry_timestamp = max(self.last_stat_entry_timestamp,
+                                         src.last_stat_entry_timestamp)
     self.directory |= src.directory
 
 

grr/lib/rdfvalues/objects_test.py

@@ -381,40 +381,40 @@ def testUpdateFromDirectory(self):
   def testMergePathInfoLastUpdate(self):
     components = ["usr", "local", "bin"]
     dest = objects.PathInfo(components=components)
-    self.assertIsNone(dest.last_path_history_timestamp)
+    self.assertIsNone(dest.last_stat_entry_timestamp)
 
     dest.UpdateFrom(
         objects.PathInfo(
             components=components,
-            last_path_history_timestamp=rdfvalue.RDFDatetime.FromHumanReadable(
+            last_stat_entry_timestamp=rdfvalue.RDFDatetime.FromHumanReadable(
                 "2017-01-01")))
-    self.assertEqual(dest.last_path_history_timestamp,
+    self.assertEqual(dest.last_stat_entry_timestamp,
                      rdfvalue.RDFDatetime.FromHumanReadable("2017-01-01"))
 
-    # Merging in a record without last_path_history_timestamp shouldn't change
+    # Merging in a record without last_stat_entry_timestamp shouldn't change
     # it.
     dest.UpdateFrom(objects.PathInfo(components=components))
-    self.assertEqual(dest.last_path_history_timestamp,
+    self.assertEqual(dest.last_stat_entry_timestamp,
                      rdfvalue.RDFDatetime.FromHumanReadable("2017-01-01"))
 
-    # Merging in a record with an earlier last_path_history_timestamp shouldn't
+    # Merging in a record with an earlier last_stat_entry_timestamp shouldn't
     # change it.
     dest.UpdateFrom(
         objects.PathInfo(
             components=components,
-            last_path_history_timestamp=rdfvalue.RDFDatetime.FromHumanReadable(
+            last_stat_entry_timestamp=rdfvalue.RDFDatetime.FromHumanReadable(
                 "2016-01-01")))
-    self.assertEqual(dest.last_path_history_timestamp,
+    self.assertEqual(dest.last_stat_entry_timestamp,
                      rdfvalue.RDFDatetime.FromHumanReadable("2017-01-01"))
 
-    # Merging in a record with a later last_path_history_timestamp should change
+    # Merging in a record with a later last_stat_entry_timestamp should change
     # it.
     dest.UpdateFrom(
         objects.PathInfo(
             components=components,
-            last_path_history_timestamp=rdfvalue.RDFDatetime.FromHumanReadable(
+            last_stat_entry_timestamp=rdfvalue.RDFDatetime.FromHumanReadable(
                 "2018-01-01")))
-    self.assertEqual(dest.last_path_history_timestamp,
+    self.assertEqual(dest.last_stat_entry_timestamp,
                      rdfvalue.RDFDatetime.FromHumanReadable("2018-01-01"))
 
 

grr/lib/registry.py

@@ -99,6 +99,26 @@ def __init__(cls, name, bases, env_dict):
       EventRegistry.EVENT_NAME_MAP.setdefault(ev, set()).add(cls)
 
 
+class FlowRegistry(MetaclassRegistry):
+  """A dedicated registry that only contains flows."""
+
+  FLOW_REGISTRY = {}
+
+  def __init__(cls, name, bases, env_dict):
+    MetaclassRegistry.__init__(cls, name, bases, env_dict)
+
+    cls.FLOW_REGISTRY[name] = cls
+
+  @classmethod
+  def FlowClassByName(mcs, flow_name):
+    flow_cls = mcs.FLOW_REGISTRY.get(flow_name)
+    if flow_cls is None:
+      raise ValueError(
+          "Flow '%s' not known by this implementation." % flow_name)
+
+    return flow_cls
+
+
 # Utility functions
 class HookRegistry(object):
   """An initializer that can be extended by plugins.

grr/proto/grr_response_proto/objects.proto

@@ -165,6 +165,7 @@ message ApprovalGrant {
     }];
 }
 
+// Next field id: 9.
 message PathInfo {
   enum PathType {
     UNSET = 0;
@@ -175,10 +176,15 @@ message PathInfo {
   }
   optional PathType path_type = 1;
   repeated string components = 2;
-  optional uint64 last_path_history_timestamp = 3
+  optional uint64 timestamp = 8 [(sem_type) = { type: "RDFDatetime" }];
+  optional uint64 last_stat_entry_timestamp = 3
       [(sem_type) = { type: "RDFDatetime" }];
+  optional uint64 last_hash_entry_timestamp = 7 [(sem_type) = {
+    type: "RDFDatetime",
+  }];
   optional bool directory = 4;
   optional StatEntry stat_entry = 5;
+  optional Hash hash_entry = 6;
 }
 
 message ObjectReference {

grr/server/grr_response_server/aff4_objects/aff4_grr.py

@@ -479,7 +479,7 @@ def _RunActions(self, rule, client_id):
             logging.info("Foreman: Starting hunt %s on client %s.",
                          action.hunt_id, client_id)
 
-            flow_cls = flow.GRRFlow.classes[action.hunt_name]
+            flow_cls = registry.FlowRegistry.FlowClassByName(action.hunt_name)
             flow_cls.StartClients(action.hunt_id, [client_id])
             actions_count += 1
         else:

grr/server/grr_response_server/aff4_objects/cronjobs.py

@@ -275,8 +275,8 @@ def ScheduleSystemCronFlows(names=None, token=None):
   errors = []
   for name in config.CONFIG["Cron.disabled_system_jobs"]:
     try:
-      cls = flow.GRRFlow.classes[name]
-    except KeyError:
+      cls = registry.FlowRegistry.FlowClassByName(name)
+    except ValueError:
       errors.append("No such flow: %s." % name)
       continue
 
@@ -285,10 +285,10 @@ def ScheduleSystemCronFlows(names=None, token=None):
                     "a flow inherited from SystemCronFlow: %s" % name)
 
   if names is None:
-    names = flow.GRRFlow.classes.keys()
+    names = registry.FlowRegistry.FLOW_REGISTRY.keys()
 
   for name in names:
-    cls = flow.GRRFlow.classes[name]
+    cls = registry.FlowRegistry.FlowClassByName(name)
 
     if aff4.issubclass(cls, SystemCronFlow):
       cron_args = rdf_cronjobs.CreateCronJobFlowArgs(periodicity=cls.frequency)

grr/server/grr_response_server/bin/fleetspeak_frontend.py

@@ -68,7 +68,6 @@ def _ProcessGrrMessage(self, fs_msg):
     msg.auth_state = rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED
 
     self.frontend.EnrolFleetspeakClient(client_id=grr_id)
-    self.frontend.RecordFleetspeakClientPing(client_id=grr_id)
     self.frontend.ReceiveMessages(client_id=grr_id, messages=[msg])
 
   def _ProcessMessageList(self, fs_msg):
@@ -84,7 +83,6 @@ def _ProcessMessageList(self, fs_msg):
       msg.auth_state = rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED
 
     self.frontend.EnrolFleetspeakClient(client_id=grr_id)
-    self.frontend.RecordFleetspeakClientPing(client_id=grr_id)
     self.frontend.ReceiveMessages(client_id=grr_id, messages=msg_list.job)
 
 

grr/server/grr_response_server/bin/fleetspeak_frontend_test.py

@@ -204,52 +204,6 @@ def testReceiveMessageListFleetspeak(self):
       stored_message.timestamp = None
       self.assertRDFValuesEqual(stored_message, want_message)
 
-  def testPingIsRecorded(self):
-    service_name = "GRR"
-    fake_service_client = _FakeGRPCServiceClient(service_name)
-
-    fleetspeak_connector.Reset()
-    fleetspeak_connector.Init(service_client=fake_service_client)
-
-    fsd = fs_frontend_tool.GRRFSServer()
-
-    grr_client_nr = 0xab
-    grr_client = self.SetupTestClientObject(grr_client_nr)
-    self.SetupClient(grr_client_nr)
-
-    messages = [
-        rdf_flows.GrrMessage(
-            request_id=1,
-            response_id=1,
-            session_id="F:123456",
-            payload=rdfvalue.RDFInteger(1))
-    ]
-
-    fs_client_id = "\x10\x00\x00\x00\x00\x00\x00\xab"
-    # fs_client_id should be equivalent to grr_client_id_urn
-    self.assertEqual(fs_client_id,
-                     fleetspeak_utils.GRRIDToFleetspeakID(grr_client.client_id))
-
-    message_list = rdf_flows.PackedMessageList()
-    communicator.Communicator.EncodeMessageList(
-        rdf_flows.MessageList(job=messages), message_list)
-
-    fs_message = fs_common_pb2.Message(
-        message_type="MessageList",
-        source=fs_common_pb2.Address(
-            client_id=fs_client_id, service_name=service_name))
-    fs_message.data.Pack(message_list.AsPrimitiveProto())
-
-    fake_time = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(42)
-    with test_lib.FakeTime(fake_time):
-      fsd.Process(fs_message, None)
-
-    md = data_store.REL_DB.ReadClientMetadata(grr_client.client_id)
-    self.assertEqual(md.ping, fake_time)
-
-    with aff4.FACTORY.Open(grr_client.client_id) as client:
-      self.assertEqual(client.Get(client.Schema.PING), fake_time)
-
 
 @db_test_lib.DualDBTest
 class ListProcessesFleetspeakTest(flow_test_lib.FlowTestsBaseclass):

grr/server/grr_response_server/databases/mem.py

@@ -24,14 +24,19 @@ def __init__(self, path_type, components):
     self._path_info = objects.PathInfo(
         path_type=path_type, components=components)
 
-    self._stat_entries = []
+    self._stat_entries = {}
+    self._hash_entries = {}
     self._children = set()
 
   def AddPathHistory(self, path_info):
     """Extends the path record history and updates existing information."""
     self.AddPathInfo(path_info)
-    self._stat_entries.append((rdfvalue.RDFDatetime.Now(),
-                               path_info.stat_entry))
+
+    timestamp = rdfvalue.RDFDatetime.Now()
+    if path_info.HasField("stat_entry"):
+      self._stat_entries[timestamp] = path_info.stat_entry
+    if path_info.HasField("hash_entry"):
+      self._hash_entries[timestamp] = path_info.hash_entry
 
   def AddPathInfo(self, path_info):
     """Updates existing path information of the path record."""
@@ -44,6 +49,7 @@ def AddPathInfo(self, path_info):
       raise ValueError(
           message % (self._path_info.components, path_info.components))
 
+    self._path_info.timestamp = rdfvalue.RDFDatetime.Now()
     self._path_info.directory |= path_info.directory
 
   def AddChild(self, path_info):
@@ -68,31 +74,45 @@ def GetPathInfo(self, timestamp=None):
     Returns:
       A `rdf_objects.PathInfo` instance.
     """
-    if timestamp is None:
-      timestamp = rdfvalue.RDFDatetime.Now()
-
-    # We want to find last stat entry that has timestamp greater (or equal) to
-    # the given one.
-    result_stat_entry = None
-    for stat_entry_timestamp, stat_entry in self._stat_entries:
-      if timestamp < stat_entry_timestamp:
-        break
+    result = self._path_info.Copy()
 
-      result_stat_entry = stat_entry
+    stat_entry_timestamp = self._LastEntryTimestamp(self._stat_entries,
+                                                    timestamp)
+    result.last_stat_entry_timestamp = stat_entry_timestamp
+    result.stat_entry = self._stat_entries.get(stat_entry_timestamp)
 
-    try:
-      last_path_history_timestamp, _ = self._stat_entries[-1]
-    except IndexError:
-      last_path_history_timestamp = None
+    hash_entry_timestamp = self._LastEntryTimestamp(self._hash_entries,
+                                                    timestamp)
+    result.last_hash_entry_timestamp = hash_entry_timestamp
+    result.hash_entry = self._hash_entries.get(hash_entry_timestamp)
 
-    result = self._path_info.Copy()
-    result.stat_entry = result_stat_entry
-    result.last_path_history_timestamp = last_path_history_timestamp
     return result
 
   def GetChildren(self):
     return set(self._children)
 
+  @staticmethod
+  def _LastEntryTimestamp(collection, upper_bound_timestamp):
+    """Searches for greatest timestamp lower than the specified one.
+
+    Args:
+      collection: A dictionary from timestamps to some items.
+      upper_bound_timestamp: An upper bound for timestamp to be returned.
+
+    Returns:
+      Greatest timestamp that is lower than the specified one. If no such value
+      exists, `None` is returned.
+    """
+    if upper_bound_timestamp is None:
+      upper_bound_timestamp = rdfvalue.RDFDatetime.Now()
+
+    upper_bound = lambda key: key <= upper_bound_timestamp
+
+    try:
+      return max(filter(upper_bound, collection.keys()))
+    except ValueError:  # Thrown if `max` input (result of filtering) is empty.
+      return None
+
 
 class InMemoryDB(db.Database):
   """An in memory database implementation used for testing."""