A number of changes.

* Timeline feature work has started.
* Self-update fixed for Debian.
* UpdateClient flow refactored to remove AFF4 traces.
* Self-update tests added.
* Chipsec dependency correctly listed.
* Protobuf dependency updated.
* Bumped version to 3.4.0.1

debian/rules

@@ -62,6 +62,7 @@ override_dh_virtualenv:
 	dh_virtualenv --python python3 \
 		--use-system-packages \
 		--extra-pip-arg "--ignore-installed" \
+		--extra-pip-arg "--no-cache-dir" \
 		--extra-pip-arg "--no-index" \
 		--extra-pip-arg "--find-links=${LOCAL_DEB_PYINDEX}" \
 		--skip-install \

grr/client/grr_response_client/client_actions/linux/linux.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# Lint as: python3
 """Linux specific actions."""
 from __future__ import absolute_import
 from __future__ import division
@@ -17,7 +18,6 @@
 from future.utils import iteritems
 
 from grr_response_client import actions
-from grr_response_client import client_utils_common
 from grr_response_client.client_actions import standard
 from grr_response_core.lib import rdfvalue
 from grr_response_core.lib import utils
@@ -352,21 +352,32 @@ def ProcessFile(self, path, args):
       raise ValueError("Unknown suffix for file %s." % path)
 
   def _InstallDeb(self, path, args):
-    cmd = "/usr/bin/dpkg"
-    cmd_args = ["-i", path]
-    time_limit = args.time_limit
-
-    client_utils_common.Execute(
-        cmd,
-        cmd_args,
-        time_limit=time_limit,
-        bypass_whitelist=True,
-        daemon=True)
-
-    # The installer will run in the background and kill the main process
-    # so we just wait. If something goes wrong, the nanny will restart the
-    # service after a short while and the client will come back to life.
-    time.sleep(1000)
+    pid = os.fork()
+    if pid == 0:
+      # This is the child that will become the installer process.
+
+      # We call os.setsid here to become the session leader of this new session
+      # and the process group leader of the new process group so we don't get
+      # killed when the main process exits.
+      try:
+        os.setsid()
+      except OSError:
+        # This only works if the process is running as root.
+        pass
+
+      env = os.environ.copy()
+      env.pop("LD_LIBRARY_PATH", None)
+      env.pop("PYTHON_PATH", None)
+
+      cmd = "/usr/bin/dpkg"
+      cmd_args = [cmd, "-i", path]
+
+      os.execve(cmd, cmd_args, env)
+    else:
+      # The installer will run in the background and kill the main process
+      # so we just wait. If something goes wrong, the nanny will restart the
+      # service after a short while and the client will come back to life.
+      time.sleep(1000)
 
   def _InstallRpm(self, path):
     """Client update for rpm based distros.

grr/client/grr_response_client/client_actions/osquery.py

@@ -237,7 +237,6 @@ def Query(args):
     TimeoutError: If a call to the osquery executable times out.
     Error: If anything else goes wrong with the subprocess call.
   """
-  query = args.query.encode("utf-8")
   timeout = args.timeout_millis / 1000  # `subprocess.run` uses seconds.
   # TODO: pytype is not aware of the backport.
   # pytype: disable=module-attr
@@ -254,7 +253,7 @@ def Query(args):
         "--logger_min_status=3",  # Disable status logs.
         "--logger_min_stderr=2",  # Only ERROR-level logs to stderr.
         "--json",  # Set output format to JSON.
-        query,
+        args.query,
     ]
     proc = subprocess.run(
         command,

grr/client/grr_response_client/client_actions/osquery_test.py

@@ -10,6 +10,7 @@
 import hashlib
 import io
 import os
+import platform
 import socket
 import time
 
@@ -104,6 +105,9 @@ def testFile(self):
           ])
       self.assertEqual(list(table.Column("size")), ["3", "6", "4"])
 
+  # TODO(hanuszczak): https://github.com/osquery/osquery/issues/4150
+  @skip.If(platform.system() == "Windows",
+           "osquery ignores files with unicode characters.")
   def testFileUnicode(self):
     with temp.AutoTempFilePath(prefix="zółć", suffix="💰") as filepath:
       with io.open(filepath, "wb") as filedesc:
@@ -156,12 +160,14 @@ def testSystemInfo(self):
     results = _Query("SELECT hostname FROM system_info;")
     self.assertLen(results, 1)
 
-    # Apparently osquery returns FQDN in "hostname" column.
-    hostname = socket.getfqdn()
-
     table = results[0].table
     self.assertLen(table.rows, 1)
-    self.assertEqual(list(table.Column("hostname")), [hostname])
+
+    # osquery sometimes returns FQDN and sometimes real hostname as the result
+    # and it is unclear what determines this. This is why instead of precise
+    # equality we test for either of them.
+    hostname = list(table.Column("hostname"))[0]
+    self.assertIn(hostname, [socket.gethostname(), socket.getfqdn()])
 
   def testMultipleResults(self):
     with temp.AutoTempDirPath(remove_non_empty=True) as dirpath:

grr/client/grr_response_client/client_actions/timeline.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+"""A module with a client action for timeline collection."""
+from __future__ import absolute_import
+from __future__ import division
+
+from __future__ import unicode_literals
+
+import hashlib
+import os
+import stat as stat_mode
+
+from typing import Iterator
+
+from grr_response_client import actions
+from grr_response_core.lib import rdfvalue
+from grr_response_core.lib.rdfvalues import protodict as rdf_protodict
+from grr_response_core.lib.rdfvalues import timeline as rdf_timeline
+
+
+class Timeline(actions.ActionPlugin):
+  """A client action for timeline collection."""
+
+  in_rdfvalue = rdf_timeline.TimelineArgs
+  out_rdfvalues = [rdf_timeline.TimelineResult]
+
+  _TRANSFER_STORE_ID = rdfvalue.SessionID(flow_name="TransferStore")
+
+  def Run(self, args):
+    """Executes the client action."""
+    result = rdf_timeline.TimelineResult()
+
+    entries = Walk(args.root)
+    for entry_batch in rdf_timeline.TimelineEntry.SerializeStream(entries):
+      entry_batch_blob = rdf_protodict.DataBlob(data=entry_batch)
+      self.SendReply(entry_batch_blob, session_id=self._TRANSFER_STORE_ID)
+
+      entry_batch_blob_id = hashlib.sha256(entry_batch).digest()
+      result.entry_batch_blob_ids.append(entry_batch_blob_id)
+
+      self.Progress()
+
+    self.SendReply(result)
+
+
+def Walk(root):
+  """Walks the filesystem collecting stat information.
+
+  This method will recursively descend to all sub-folders and sub-sub-folders
+  and so on. It will stop the recursion at device boundaries and will not follow
+  any symlinks (to avoid cycles and virtual filesystems that may be potentially
+  infinite).
+
+  Args:
+    root: A path to the root folder at which the recursion should start.
+
+  Returns:
+    An iterator over timeline entries with stat information about each file.
+  """
+  try:
+    dev = os.lstat(root).st_dev
+  except OSError:
+    return iter([])
+
+  def Recurse(path):
+    """Performs the recursive walk over the file hierarchy."""
+    try:
+      stat = os.lstat(path)
+    except OSError:
+      return
+
+    yield rdf_timeline.TimelineEntry.FromStat(path, stat)
+
+    # We want to recurse only to folders on the same device.
+    if not stat_mode.S_ISDIR(stat.st_mode) or stat.st_dev != dev:
+      return
+
+    try:
+      childnames = os.listdir(path)
+    except OSError:
+      childnames = []
+
+    # TODO(hanuszczak): Implement more efficient auto-batcher instead of having
+    # multi-level iterators.
+    for childname in childnames:
+      for entry in Recurse(os.path.join(path, childname)):
+        yield entry
+
+  return Recurse(root)

grr/client/grr_response_client/client_actions/timeline_test.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python
+from __future__ import absolute_import
+from __future__ import division
+
+from __future__ import unicode_literals
+
+import hashlib
+import io
+import os
+import platform
+import random
+import stat as stat_mode
+
+from absl.testing import absltest
+from typing import Text
+
+from grr_response_client.client_actions import timeline
+from grr_response_core.lib.rdfvalues import timeline as rdf_timeline
+from grr_response_core.lib.util import temp
+from grr.test_lib import client_test_lib
+from grr.test_lib import skip
+from grr.test_lib import testing_startup
+
+
+# TODO(hanuszczak): `GRRBaseTest` is terrible, try to avoid it in any new code.
+class TimelineTest(client_test_lib.EmptyActionTest):
+
+  @classmethod
+  def setUpClass(cls):
+    super(TimelineTest, cls).setUpClass()
+    testing_startup.TestInit()
+
+  def testRun(self):
+    with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath:
+      for idx in range(64):
+        temp_filepath = os.path.join(temp_dirpath, "foo{}".format(idx))
+        _Touch(temp_filepath, content=os.urandom(random.randint(0, 1024)))
+
+      args = rdf_timeline.TimelineArgs()
+      args.root = temp_dirpath.encode("utf-8")
+
+      results = self.RunAction(timeline.Timeline, args)
+
+      self.assertNotEmpty(results)
+      self.assertNotEmpty(results[-1].entry_batch_blob_ids)
+
+      blob_ids = results[-1].entry_batch_blob_ids
+      for blob in results[:-1]:
+        self.assertIn(hashlib.sha256(blob.data).digest(), blob_ids)
+
+
+class WalkTest(absltest.TestCase):
+
+  def testSingleFile(self):
+    with temp.AutoTempDirPath(remove_non_empty=True) as dirpath:
+      filepath = os.path.join(dirpath, "foo")
+      _Touch(filepath, content=b"foobar")
+
+      entries = list(timeline.Walk(dirpath.encode("utf-8")))
+      self.assertLen(entries, 2)
+
+      self.assertTrue(stat_mode.S_ISDIR(entries[0].mode))
+      self.assertEqual(entries[0].path, dirpath.encode("utf-8"))
+
+      self.assertTrue(stat_mode.S_ISREG(entries[1].mode))
+      self.assertEqual(entries[1].path, filepath.encode("utf-8"))
+      self.assertEqual(entries[1].size, 6)
+
+  def testMultipleFiles(self):
+    with temp.AutoTempDirPath(remove_non_empty=True) as dirpath:
+      foo_filepath = os.path.join(dirpath, "foo")
+      bar_filepath = os.path.join(dirpath, "bar")
+      baz_filepath = os.path.join(dirpath, "baz")
+
+      _Touch(foo_filepath)
+      _Touch(bar_filepath)
+      _Touch(baz_filepath)
+
+      entries = list(timeline.Walk(dirpath.encode("utf-8")))
+      self.assertLen(entries, 4)
+
+      paths = [_.path for _ in entries[1:]]
+      self.assertIn(foo_filepath.encode("utf-8"), paths)
+      self.assertIn(bar_filepath.encode("utf-8"), paths)
+      self.assertIn(baz_filepath.encode("utf-8"), paths)
+
+  def testNestedDirectories(self):
+    with temp.AutoTempDirPath(remove_non_empty=True) as root_dirpath:
+      foobar_dirpath = os.path.join(root_dirpath, "foo", "bar")
+      os.makedirs(foobar_dirpath)
+
+      foobaz_dirpath = os.path.join(root_dirpath, "foo", "baz")
+      os.makedirs(foobaz_dirpath)
+
+      quuxnorfthud_dirpath = os.path.join(root_dirpath, "quux", "norf", "thud")
+      os.makedirs(quuxnorfthud_dirpath)
+
+      entries = list(timeline.Walk(root_dirpath.encode("utf-8")))
+      self.assertLen(entries, 7)
+
+      paths = [_.path.decode("utf-8") for _ in entries]
+      self.assertCountEqual(paths, [
+          os.path.join(root_dirpath),
+          os.path.join(root_dirpath, "foo"),
+          os.path.join(root_dirpath, "foo", "bar"),
+          os.path.join(root_dirpath, "foo", "baz"),
+          os.path.join(root_dirpath, "quux"),
+          os.path.join(root_dirpath, "quux", "norf"),
+          os.path.join(root_dirpath, "quux", "norf", "thud"),
+      ])
+
+      for entry in entries:
+        self.assertTrue(stat_mode.S_ISDIR(entry.mode))
+
+  @skip.If(
+      platform.system() == "Windows",
+      reason="Symlinks are not supported on Windows.")
+  def testSymlinks(self):
+    with temp.AutoTempDirPath(remove_non_empty=True) as root_dirpath:
+      sub_dirpath = os.path.join(root_dirpath, "foo", "bar", "baz")
+      link_path = os.path.join(sub_dirpath, "quux")
+
+      # This creates a cycle, walker should be able to cope with that.
+      os.makedirs(sub_dirpath)
+      os.symlink(root_dirpath, os.path.join(sub_dirpath, link_path))
+
+      entries = list(timeline.Walk(root_dirpath.encode("utf-8")))
+      self.assertLen(entries, 5)
+
+      paths = [_.path.decode("utf-8") for _ in entries]
+      self.assertEqual(paths, [
+          os.path.join(root_dirpath),
+          os.path.join(root_dirpath, "foo"),
+          os.path.join(root_dirpath, "foo", "bar"),
+          os.path.join(root_dirpath, "foo", "bar", "baz"),
+          os.path.join(root_dirpath, "foo", "bar", "baz", "quux")
+      ])
+
+      for entry in entries[:-1]:
+        self.assertTrue(stat_mode.S_ISDIR(entry.mode))
+      self.assertTrue(stat_mode.S_ISLNK(entries[-1].mode))
+
+  def testIncorrectPath(self):
+    not_existing_path = os.path.join("some", "not", "existing", "path")
+
+    entries = list(timeline.Walk(not_existing_path.encode("utf-8")))
+    self.assertEmpty(entries)
+
+
+def _Touch(filepath, content = b""):
+  with io.open(filepath, mode="wb") as filedesc:
+    filedesc.write(content)
+
+
+if __name__ == "__main__":
+  absltest.main()

grr/client/setup.py

@@ -91,9 +91,7 @@ def make_release_tree(self, base_dir, files):
 )
 
 if platform.system() == "Linux":
-  # TODO(user): change 'extras_require' to 'install_requires' if/when
-  # chipsec driver-less PIP package shows up on PyPI.
-  setup_args["extras_require"]["chipsec"] = ["chipsec==1.4.3"]
+  setup_args["install_requires"].append("chipsec==1.4.4")
 
 if platform.system() != "Windows":
   setup_args["install_requires"].append("xattr==0.9.6")

grr/client_builder/grr_response_client_builder/build_helpers.py

@@ -174,7 +174,8 @@ def BuildWithPyInstaller(context=None):
     except IOError:
       logging.error("Unable to create file: %s", file_path)
 
-  version_ini = version.VersionPath()
+  version_ini = config.CONFIG.Get(
+      "ClientBuilder.version_ini_path", default=version.VersionPath())
   shutil.copy(version_ini, os.path.join(output_dir, "version.ini"))
 
   with io.open(os.path.join(output_dir, "build.yaml"), "wb") as fd: