A number of changes:

* Angular project boilerplate for the next-gen UI submitted.
* A number of parser issues fixed.
* Python and JS dependencies bumped.
* FromDatastoreValue/ToDatastoreValue and related methods renamed -> FromWireFormat/ToWireFormat.
* Self-contained e2e tests rewritten to use fork/exec instead of simply
forking.

Dockerfile

@@ -50,7 +50,7 @@ RUN pip install --upgrade --no-cache-dir pip virtualenv && \
     virtualenv --system-site-packages $GRR_VENV
 
 RUN $GRR_VENV/bin/pip install --upgrade --no-cache-dir wheel six setuptools nodeenv && \
-    $GRR_VENV/bin/nodeenv -p --prebuilt --node=10.12.0 && \
+    $GRR_VENV/bin/nodeenv -p --prebuilt --node=12.11.1 && \
     echo '{ "allow_root": true }' > /root/.bowerrc
 
 # Copy the GRR code over.

api_client/python/setup.py

@@ -73,12 +73,12 @@ def make_release_tree(self, base_dir, files):
         "console_scripts": ["grr_api_shell = grr_api_client.api_shell:main",]
     },
     install_requires=[
-        "future==0.17.0",
+        "future==0.17.1",
         "grr_response_proto==%s" % VERSION.get("Version", "packagedepends"),
-        "cryptography==2.4.2",
+        "cryptography==2.7",
         "ipython==%s" % ("5.0.0" if sys.version_info < (3, 0) else "7.2.0"),
-        "requests==2.21.0",
-        "Werkzeug==0.15.3",
+        "requests==2.22.0",
+        "Werkzeug==0.16.0",
     ],
     data=["version.ini"])
 

colab/setup.py

@@ -68,7 +68,7 @@ def make_release_tree(self, base_dir, files):
         "sdist": Sdist,
     },
     install_requires=[
-        "future==0.17.0",
+        "future==0.17.1",
         "grr_api_client==%s" % VERSION.get("Version", "packagedepends"),
         "grr_response_proto==%s" % VERSION.get("Version", "packagedepends"),
         "humanize==0.5.1",

grr/client/grr_response_client/client_actions/memory.py

@@ -88,6 +88,8 @@ def _ScanRegion(self, rules, chunks, deadline):
       if not chunk.data:
         break
 
+      self.Progress()
+
       time_left = (deadline - rdfvalue.RDFDatetime.Now()).ToInt(
           rdfvalue.SECONDS)
 
@@ -403,6 +405,7 @@ def DumpProcess(self, psutil_process, args):
 
       with tempfiles.TemporaryDirectory(cleanup=False) as tmp_dir:
         for region in regions:
+          self.Progress()
           pathspec = self._SaveRegionToDirectory(psutil_process, process,
                                                  region, tmp_dir, streamer)
           if pathspec is not None:

grr/client/grr_response_client/vfs.py

@@ -220,7 +220,7 @@ def VFSMultiOpen(pathspecs, progress_callback=None):
   precondition.AssertIterableType(pathspecs, rdf_paths.PathSpec)
 
   vfs_open = functools.partial(VFSOpen, progress_callback=progress_callback)
-  return context.MultiContext(map(vfs_open, pathspecs))
+  return context.MultiContext(list(map(vfs_open, pathspecs)))
 
 
 def ReadVFS(pathspec, offset, length, progress_callback=None):

grr/client/setup.py

@@ -84,11 +84,11 @@ def make_release_tree(self, base_dir, files):
     packages=find_packages(),
     include_package_data=True,
     install_requires=[
-        "absl-py==0.6.1",
+        "absl-py==0.8.0",
         "grr-response-core==%s" % VERSION.get("Version", "packagedepends"),
         # TODO: This is a backport of Python 3.2+ API, should be
         # removed once support for Python 2 is dropped.
-        "subprocess32==3.5.3",
+        "subprocess32==3.5.4",
         "pyinstaller==3.5",
     ],
     extras_require={
@@ -108,6 +108,6 @@ def make_release_tree(self, base_dir, files):
   setup_args["install_requires"].append("chipsec=={}".format(chipsec_version))
 
 if platform.system() != "Windows":
-  setup_args["install_requires"].append("xattr==0.9.2")
+  setup_args["install_requires"].append("xattr==0.9.6")
 
 setup(**setup_args)

grr/client_builder/setup.py

@@ -75,7 +75,6 @@ def make_release_tree(self, base_dir, files):
     include_package_data=True,
     python_requires=">=2.7.11",
     install_requires=[
-        "absl-py==0.6.1",
         "distro==1.4.0",
         "grr-response-client==%s" % VERSION.get("Version", "packagedepends"),
         "grr-response-core==%s" % VERSION.get("Version", "packagedepends"),

grr/core/grr_response_core/lib/config_lib_test.py

@@ -8,9 +8,7 @@
 import io
 import ntpath
 import os
-import platform
 import stat
-import unittest
 
 from absl import app
 from absl.testing import flagsaver
@@ -1040,19 +1038,16 @@ def testRenameOnWritebackFailure(self):
     self.assertRaises(AttributeError, conf.SetWriteBack, writeback_file)
     self.assertTrue(os.path.isfile(writeback_file + ".bak"))
 
-  # TODO: Fix test, which fails on Windows due to file locking.
-  @unittest.skipIf(platform.system() == "Windows",
-                   "Fails due to Windows file locking issues.")
   def testNoRenameOfReadProtectedFile(self):
     """Don't rename config files we don't have permission to read."""
-    conf = config.CONFIG.MakeNewConfig()
     writeback_file = os.path.join(self.temp_dir, "writeback.yaml")
-    with io.open(writeback_file, "w") as f:
-      f.write("...")
-      f.close()
+    with io.open(writeback_file, mode="w", encoding="utf-8") as f:
+      f.write("")
 
     # Remove all permissions except user write.
     os.chmod(writeback_file, stat.S_IWUSR)
+
+    conf = config.CONFIG.MakeNewConfig()
     conf.SetWriteBack(writeback_file)
 
     # File is still in the same place

grr/core/grr_response_core/lib/parsers/chrome_history.py

@@ -2,17 +2,22 @@
 """Parser for Google chrome/chromium History files."""
 from __future__ import absolute_import
 from __future__ import division
+
 from __future__ import print_function
 from __future__ import unicode_literals
 
-import itertools
+import logging
 
 from future.moves.urllib import parse as urlparse
 from past.builtins import long
+import sqlite3
+from typing import IO
+from typing import Iterator
+from typing import Tuple
 
 from grr_response_core.lib import parsers
-from grr_response_core.lib.parsers import sqlite_file
 from grr_response_core.lib.rdfvalues import webhistory as rdf_webhistory
+from grr_response_core.lib.util import sqlite
 
 
 class ChromeHistoryParser(parsers.SingleFileParser):
@@ -25,8 +30,8 @@ def ParseFile(self, knowledge_base, pathspec, filedesc):
     del knowledge_base  # Unused.
 
     # TODO(user): Convert this to use the far more intelligent plaso parser.
-    chrome = ChromeParser(filedesc)
-    for timestamp, entry_type, url, data1, _, _ in chrome.Parse():
+    chrome = ChromeParser()
+    for timestamp, entry_type, url, data1, _, _ in chrome.Parse(filedesc):
       if entry_type == "CHROME_DOWNLOAD":
         yield rdf_webhistory.BrowserHistoryItem(
             url=url,
@@ -45,22 +50,19 @@ def ParseFile(self, knowledge_base, pathspec, filedesc):
             title=data1)
 
 
-class ChromeParser(sqlite_file.SQLiteFile):
-  """Class for handling the parsing of a Chrome history file.
-
-    Use as:
-      c = ChromeParser(open('History'))
-      for hist in c.Parse():
-        print hist
-
-  Returns results in chronological order
-  """
+# TODO(hanuszczak): This shouldn't be a class, just a `Parse` function taking
+# history file as input.
+class ChromeParser(object):
+  """Class for handling the parsing of a Chrome history file."""
   VISITS_QUERY = ("SELECT visits.visit_time, urls.url, urls.title, "
                   "urls.typed_count "
                   "FROM urls, visits "
                   "WHERE urls.id = visits.url "
                   "ORDER BY visits.visit_time ASC;")
 
+  # TODO(hanuszczak): Do we need to maintain code that is supposed to work with
+  # Chrome history format from 2013?
+
   # We use DESC here so we can pop off the end of the list and interleave with
   # visits to maintain time order.
   DOWNLOADS_QUERY = ("SELECT downloads.start_time, downloads.url, "
@@ -91,29 +93,44 @@ def ConvertTimestamp(self, timestamp):
       timestamp *= 1000000
     return timestamp
 
-  def Parse(self):
+  # TODO(hanuszczak): This function should return a well-structured data instead
+  # of a tuple of 6 elements (of which 2 of them are never used).
+  def Parse(self, filedesc):  # pylint: disable=g-bare-generic
     """Iterator returning a list for each entry in history.
 
     We store all the download events in an array (choosing this over visits
     since there are likely to be less of them). We later interleave them with
     visit events to get an overall correct time order.
 
+    Args:
+      filedesc: A file-like object
+
     Yields:
       a list of attributes for each entry
     """
-    # Query for old style and newstyle downloads storage.
-    query_iter = itertools.chain(
-        self.Query(self.DOWNLOADS_QUERY), self.Query(self.DOWNLOADS_QUERY_2))
-
-    results = []
-    for timestamp, url, path, received_bytes, total_bytes in query_iter:
-      timestamp = self.ConvertTimestamp(timestamp)
-      results.append((timestamp, "CHROME_DOWNLOAD", url, path, received_bytes,
-                      total_bytes))
-
-    for timestamp, url, title, typed_count in self.Query(self.VISITS_QUERY):
-      timestamp = self.ConvertTimestamp(timestamp)
-      results.append((timestamp, "CHROME_VISIT", url, title, typed_count, ""))
+    with sqlite.IOConnection(filedesc) as conn:
+      # Query for old style and newstyle downloads storage.
+      rows = []
+
+      try:
+        rows.extend(conn.Query(self.DOWNLOADS_QUERY))
+      except sqlite3.Error as error:
+        logging.warning("Chrome history database error: %s", error)
+
+      try:
+        rows.extend(conn.Query(self.DOWNLOADS_QUERY_2))
+      except sqlite3.Error as error:
+        logging.warning("Chrome history database error: %s", error)
+
+      results = []
+      for timestamp, url, path, received_bytes, total_bytes in rows:
+        timestamp = self.ConvertTimestamp(timestamp)
+        results.append((timestamp, "CHROME_DOWNLOAD", url, path, received_bytes,
+                        total_bytes))
+
+      for timestamp, url, title, typed_count in conn.Query(self.VISITS_QUERY):
+        timestamp = self.ConvertTimestamp(timestamp)
+        results.append((timestamp, "CHROME_VISIT", url, title, typed_count, ""))
 
     results.sort(key=lambda it: it[0])
     for it in results:

grr/core/grr_response_core/lib/parsers/chrome_history_test.py

@@ -7,6 +7,7 @@
 from __future__ import unicode_literals
 
 import datetime
+import io
 import os
 
 from absl import app
@@ -22,8 +23,9 @@ class ChromeHistoryTest(test_lib.GRRBaseTest):
   def testBasicParsing(self):
     """Test we can parse a standard file."""
     history_file = os.path.join(self.base_path, "parser_test", "History2")
-    history = chrome_history.ChromeParser(open(history_file, "rb"))
-    entries = [x for x in history.Parse()]
+    with io.open(history_file, mode="rb") as history_filedesc:
+      history = chrome_history.ChromeParser()
+      entries = list(history.Parse(history_filedesc))
 
     try:
       dt1 = datetime.datetime(1970, 1, 1)
@@ -53,8 +55,9 @@ def testBasicParsing(self):
   def testTimeOrderingDownload(self):
     """Test we can correctly time order downloads and visits."""
     history_file = os.path.join(self.base_path, "parser_test", "History3")
-    history = chrome_history.ChromeParser(open(history_file, "rb"))
-    entries = [x for x in history.Parse()]
+    with io.open(history_file, mode="rb") as history_filedesc:
+      history = chrome_history.ChromeParser()
+      entries = list(history.Parse(history_filedesc))
 
     # Check that our results are properly time ordered
     time_results = [x[0] for x in entries]
@@ -64,8 +67,9 @@ def testTimeOrderingDownload(self):
   def testBasicParsingOldFormat(self):
     """Test we can parse a standard file."""
     history_file = os.path.join(self.base_path, "parser_test", "History")
-    history = chrome_history.ChromeParser(open(history_file, "rb"))
-    entries = [x for x in history.Parse()]
+    with io.open(history_file, mode="rb") as history_filedesc:
+      history = chrome_history.ChromeParser()
+      entries = list(history.Parse(history_filedesc))
 
     try:
       dt1 = datetime.datetime(1970, 1, 1)

grr/core/grr_response_core/lib/parsers/firefox3_history.py

@@ -2,15 +2,19 @@
 """Parser for Mozilla Firefox3 3 History files."""
 from __future__ import absolute_import
 from __future__ import division
+
 from __future__ import print_function
 from __future__ import unicode_literals
 
 from future.moves.urllib import parse as urlparse
 from past.builtins import long
+from typing import IO
+from typing import Iterator
+from typing import Tuple
 
 from grr_response_core.lib import parsers
-from grr_response_core.lib.parsers import sqlite_file
 from grr_response_core.lib.rdfvalues import webhistory as rdf_webhistory
+from grr_response_core.lib.util import sqlite
 
 
 class FirefoxHistoryParser(parsers.SingleFileParser):
@@ -23,8 +27,8 @@ def ParseFile(self, knowledge_base, pathspec, filedesc):
     del knowledge_base  # Unused.
 
     # TODO(user): Convert this to use the far more intelligent plaso parser.
-    ff = Firefox3History(filedesc)
-    for timestamp, unused_entry_type, url, title in ff.Parse():
+    ff = Firefox3History()
+    for timestamp, unused_entry_type, url, title in ff.Parse(filedesc):
       yield rdf_webhistory.BrowserHistoryItem(
           url=url,
           domain=urlparse.urlparse(url).netloc,
@@ -34,26 +38,22 @@ def ParseFile(self, knowledge_base, pathspec, filedesc):
           title=title)
 
 
-class Firefox3History(sqlite_file.SQLiteFile):
-  """Class for handling the parsing of a Firefox 3 history file (places.sqlite).
-
-    Use as:
-      c = Firefox3History(open('places.sqlite'))
-      for hist in c.Parse():
-        print hist
+# TODO(hanuszczak): This should not be a class.
+class Firefox3History(object):
+  """Class for handling the parsing of a Firefox 3 history file."""
 
-  Returns results in chronological order
-  """
   VISITS_QUERY = ("SELECT moz_historyvisits.visit_date, moz_places.url,"
                   " moz_places.title "
                   "FROM moz_places, moz_historyvisits "
                   "WHERE moz_places.id = moz_historyvisits.place_id "
                   "ORDER BY moz_historyvisits.visit_date ASC;")
 
-  def Parse(self):
+  # TODO(hanuszczak): This should return well-structured data.
+  def Parse(self, filedesc):  # pylint: disable=g-bare-generic
     """Iterator returning dict for each entry in history."""
-    for timestamp, url, title in self.Query(self.VISITS_QUERY):
-      if not isinstance(timestamp, (long, int)):
-        timestamp = 0
+    with sqlite.IOConnection(filedesc) as conn:
+      for timestamp, url, title in conn.Query(self.VISITS_QUERY):
+        if not isinstance(timestamp, (long, int)):
+          timestamp = 0
 
-      yield [timestamp, "FIREFOX3_VISIT", url, title]
+        yield timestamp, "FIREFOX3_VISIT", url, title

grr/core/grr_response_core/lib/parsers/firefox3_history_test.py

@@ -7,6 +7,7 @@
 from __future__ import unicode_literals
 
 import datetime
+import io
 import os
 
 from absl import app
@@ -26,9 +27,10 @@ class Firefox3HistoryTest(test_lib.GRRBaseTest):
   def testBasicParsing(self):
     """Test we can parse a standard file."""
     history_file = os.path.join(self.base_path, "places.sqlite")
-    history = firefox3_history.Firefox3History(open(history_file, "rb"))
-    # Parse returns (timestamp, dtype, url, title)
-    entries = [x for x in history.Parse()]
+    with io.open(history_file, mode="rb") as history_filedesc:
+      history = firefox3_history.Firefox3History()
+      # Parse returns (timestamp, dtype, url, title)
+      entries = [x for x in history.Parse(history_filedesc)]
 
     self.assertLen(entries, 1)
 
@@ -45,8 +47,9 @@ def testBasicParsing(self):
   def testNewHistoryFile(self):
     """Tests reading of history files written by recent versions of Firefox."""
     history_file = os.path.join(self.base_path, "new_places.sqlite")
-    history = firefox3_history.Firefox3History(open(history_file, "rb"))
-    entries = [x for x in history.Parse()]
+    with io.open(history_file, mode="rb") as history_filedesc:
+      history = firefox3_history.Firefox3History()
+      entries = [x for x in history.Parse(history_filedesc)]
 
     self.assertLen(entries, 3)
     self.assertEqual(entries[1][3],