cat/play: support directories

@TarantoolBot document
Title: Multiple directories with WAL files can be processed
by tt cat and tt play commands

This patch adds support of multiple directories with WAL files
to the tt cat and tt play commands.

```shell
$ tt cat /path/to/file.snap /path/to/file.xlog /path/to/dir/
$ tt play <URI> /path/to/file.snap /path/to/file.xlog /path/to/dir/
```

Closes #1020
Closes #1027

CHANGELOG.md

@@ -40,6 +40,7 @@ The release introduces `upgrade` and `downgrade` subcommands for
   time format.
 - `tt connect`: add new `--evaler` option to support for customizing the way
   user input is processed.
+- `tt cat/play`: allows to specify a list of directories to search WAL files.
 
 ### Fixed
 

cli/cmd/cat.go

@@ -40,8 +40,10 @@ func NewCatCmd() *cobra.Command {
 				internalCatModule, args)
 			util.HandleCmdErr(cmd, err)
 		},
-		Example: "tt cat /path/to/xlog --timestamp 2024-11-13T14:02:36.818700000+00:00\n" +
-			"  tt cat /path/to/file.xlog /path/to/file.snap --timestamp=1731592956.818",
+		Example: "tt cat /path/to/file.snap /path/to/file.xlog /path/to/dir/ " +
+			"--timestamp 2024-11-13T14:02:36.818700000+00:00\n" +
+			"  tt cat /path/to/file.snap /path/to/file.xlog /path/to/dir/ " +
+			"--timestamp=1731592956.818",
 	}
 
 	catCmd.Flags().Uint64Var(&catFlags.To, "to", catFlags.To,
@@ -68,8 +70,15 @@ func internalCatModule(cmdCtx *cmdcontext.CmdCtx, args []string) error {
 		return errors.New("it is required to specify at least one .xlog or .snap file")
 	}
 
+	walFiles, err := util.CollectWALFiles(args)
+	if err != nil {
+		return util.InternalError(
+			"Internal error: could not collect WAL files: %s",
+			version.GetVersion, err)
+	}
+
 	// List of files is passed to lua cat script via environment variable in json format.
-	filesJson, err := json.Marshal(args)
+	filesJson, err := json.Marshal(walFiles)
 	if err != nil {
 		return util.InternalError(
 			"Internal error: problem with creating json params with files: %s",

cli/cmd/play.go

@@ -47,8 +47,10 @@ func NewPlayCmd() *cobra.Command {
 				internalPlayModule, args)
 			util.HandleCmdErr(cmd, err)
 		},
-		Example: "tt play uri /path/to/xlog --timestamp 2024-11-13T14:02:36.818700000+00:00\n" +
-			"  tt play uri /path/to/file.xlog /path/to/file.snap --timestamp=1731592956.818",
+		Example: "tt play uri /path/to/file.snap /path/to/file.xlog /path/to/dir/ " +
+			"--timestamp 2024-11-13T14:02:36.818700000+00:00\n" +
+			"  tt play uri /path/to/file.snap /path/to/file.xlog /path/to/dir/ " +
+			"--timestamp=1731592956.818",
 	}
 
 	playCmd.Flags().StringVarP(&playUsername, "username", "u", "", "username")
@@ -72,11 +74,22 @@ func NewPlayCmd() *cobra.Command {
 // internalPlayModule is a default play module.
 func internalPlayModule(cmdCtx *cmdcontext.CmdCtx, args []string) error {
 	if len(args) < 2 {
-		return errors.New("it is required to specify an URI and at least one .xlog or .snap file")
+		return errors.New("it is required to specify an URI and at least one .xlog/.snap file " +
+			"or directory")
 	}
 
+	walFiles, err := util.CollectWALFiles(args[1:])
+	if err != nil {
+		return util.InternalError(
+			"Internal error: could not collect WAL files: %s",
+			version.GetVersion, err)
+	}
+
+	// Re-create args with the URI in the first index, and all founded files after.
+	uriAndWalFiles := append([]string{args[0]}, walFiles...)
+
 	// List of files and URI is passed to lua play script via environment variable in json format.
-	filesAndUriJson, err := json.Marshal(args)
+	filesAndUriJson, err := json.Marshal(uriAndWalFiles)
 	if err != nil {
 		return util.InternalError(
 			"Internal error: problem with creating json params with files and uri: %s",

cli/util/util.go

@@ -17,6 +17,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime/debug"
+	"slices"
 	"strconv"
 	"strings"
 	"text/template"
@@ -967,3 +968,48 @@ func StringToTimestamp(input string) (string, error) {
 
 	return ts, nil
 }
+
+// CollectWALFiles globs files from args.
+func CollectWALFiles(paths []string) ([]string, error) {
+	if len(paths) < 1 {
+		return nil, errors.New("it is required to specify at least one .xlog/.snap file " +
+			"or directory")
+	}
+	collectedFiles := make([]string, 0)
+
+	sortSnapFilesFirst := func(left, right string) int {
+		reSnap := regexp.MustCompile(`\.snap`)
+		reXlog := regexp.MustCompile(`\.xlog`)
+		if reSnap.Match([]byte(left)) && reXlog.Match([]byte(right)) {
+			return -1
+		}
+		if reXlog.Match([]byte(left)) && reSnap.Match([]byte(right)) {
+			return 1
+		}
+		return 0
+	}
+
+	for _, path := range paths {
+		entry, err := os.Stat(path)
+		if err != nil {
+			return nil, err
+		}
+
+		if entry.IsDir() {
+			foundEntries, err := os.ReadDir(path)
+			if err != nil {
+				return nil, err
+			}
+			for _, entry := range foundEntries {
+				collectedFiles = append(collectedFiles, filepath.Join(path, entry.Name()))
+			}
+			slices.SortFunc(collectedFiles, sortSnapFilesFirst)
+
+			continue
+		}
+
+		collectedFiles = append(collectedFiles, path)
+	}
+
+	return collectedFiles, nil
+}

cli/util/util_test.go

@@ -797,3 +797,60 @@ func TestStringToTimestamp(t *testing.T) {
 		})
 	}
 }
+
+func TestCollectWALFiles(t *testing.T) {
+	srcDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(srcDir, "1.xlog"), []byte{}, 0644))
+	require.NoError(t, os.WriteFile(filepath.Join(srcDir, "2.xlog"), []byte{}, 0644))
+	require.NoError(t, os.WriteFile(filepath.Join(srcDir, "1.snap"), []byte{}, 0644))
+	require.NoError(t, os.WriteFile(filepath.Join(srcDir, "2.snap"), []byte{}, 0644))
+	snap1 := fmt.Sprintf("%s/%s", srcDir, "1.snap")
+	snap2 := fmt.Sprintf("%s/%s", srcDir, "2.snap")
+	xlog1 := fmt.Sprintf("%s/%s", srcDir, "1.xlog")
+	xlog2 := fmt.Sprintf("%s/%s", srcDir, "2.xlog")
+
+	tests := []struct {
+		name           string
+		input          []string
+		output         []string
+		expectedErrMsg string
+	}{
+		{
+			name:           "no_file",
+			expectedErrMsg: "it is required to specify at least one .xlog/.snap file or directory",
+		},
+		{
+			name:           "incorrect_file",
+			input:          []string{"file"},
+			expectedErrMsg: "stat file: no such file or directory",
+		},
+		{
+			name:   "one_file",
+			input:  []string{xlog1},
+			output: []string{xlog1},
+		},
+		{
+			name:   "directory",
+			input:  []string{srcDir},
+			output: []string{snap1, snap2, xlog1, xlog2},
+		},
+		{
+			name:   "mix",
+			input:  []string{srcDir, "util_test.go"},
+			output: []string{snap1, snap2, xlog1, xlog2, "util_test.go"},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			result, err := CollectWALFiles(test.input)
+
+			if test.expectedErrMsg != "" {
+				assert.EqualError(t, err, test.expectedErrMsg)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, test.output, result)
+			}
+		})
+	}
+}

test/integration/cat/test_cat.py

@@ -8,18 +8,18 @@
 from utils import run_command_and_get_output
 
 
-@pytest.mark.parametrize("args, found", [
+@pytest.mark.parametrize("args, expected_error", [
     (
         # Testing with unset .xlog or .snap file.
-        (),
+        "",
         "it is required to specify at least one .xlog or .snap file",
     ),
     (
         "path-to-non-existent-file",
-        "No such file or directory",
+        "error: could not collect WAL files",
     ),
 ])
-def test_cat_args_tests_failed(tt_cmd, tmp_path, args, found):
+def test_cat_args_tests_failed(tt_cmd, tmp_path, args, expected_error):
     # Copy the .xlog file to the "run" directory.
     test_xlog_file = os.path.join(os.path.dirname(__file__), "test_file", "test.xlog")
     test_snap_file = os.path.join(os.path.dirname(__file__), "test_file", "test.snap")
@@ -30,10 +30,10 @@ def test_cat_args_tests_failed(tt_cmd, tmp_path, args, found):
     cmd.extend(args)
     rc, output = run_command_and_get_output(cmd, cwd=tmp_path)
     assert rc == 1
-    assert found in output
+    assert expected_error in output
 
 
-@pytest.mark.parametrize("args, found", [
+@pytest.mark.parametrize("args, expected", [
     (
         ("test.snap", "--show-system", "--space=320", "--space=296", "--from=423", "--to=513"),
         ("lsn: 423", "lsn: 512", "space_id: 320", "space_id: 296"),
@@ -48,7 +48,7 @@ def test_cat_args_tests_failed(tt_cmd, tmp_path, args, found):
          'Result of cat: the file "test.snap" is processed below'),
     ),
 ])
-def test_cat_args_tests_successed(tt_cmd, tmp_path, args, found):
+def test_cat_args_tests_successed(tt_cmd, tmp_path, args, expected):
     # Copy the .xlog file to the "run" directory.
     test_xlog_file = os.path.join(os.path.dirname(__file__), "test_file", "test.xlog")
     test_snap_file = os.path.join(os.path.dirname(__file__), "test_file", "test.snap")
@@ -59,7 +59,30 @@ def test_cat_args_tests_successed(tt_cmd, tmp_path, args, found):
     cmd.extend(args)
     rc, output = run_command_and_get_output(cmd, cwd=tmp_path)
     assert rc == 0
-    for item in found:
+    for item in expected:
+        assert item in output
+
+
+@pytest.mark.parametrize("args, expected", [
+    (
+        ("test_file/test.xlog", "test_file/test.snap", "test_file"),
+        ('Result of cat: the file "test_file/test.xlog" is processed below',
+         'Result of cat: the file "test_file/test.snap" is processed below',
+         'Result of cat: the file "test_file/timestamp.snap" is processed below',
+         'Result of cat: the file "test_file/timestamp.xlog" is processed below'),
+    ),
+])
+def test_cat_directories_tests_successed(tt_cmd, tmp_path, args, expected):
+    # Copy files to the "run" directory.
+    test_src = os.path.join(os.path.dirname(__file__), "test_file")
+    test_dir = os.path.join(tmp_path, "test_file")
+    shutil.copytree(test_src, test_dir)
+
+    cmd = [tt_cmd, "cat"]
+    cmd.extend(args)
+    rc, output = run_command_and_get_output(cmd, cwd=tmp_path)
+    assert rc == 0
+    for item in expected:
         assert item in output
 
 

test/integration/play/test_play.py

@@ -1,5 +1,6 @@
 import os
 import re
+import shutil
 
 import pytest
 
@@ -21,23 +22,23 @@ def test_instance(request, tmp_path):
     return inst
 
 
-def test_play_non_existent_uri(tt_cmd, tmp_path):
+def test_play_non_existent_uri(tt_cmd, test_instance, tmp_path):
     # Testing with non-existent uri.
-    cmd = [tt_cmd, "play", "127.0.0.1:0", "_"]
+    cmd = [tt_cmd, "play", "127.0.0.1:0", f"{test_instance._tmpdir}/test.xlog"]
     rc, output = run_command_and_get_output(cmd, cwd=tmp_path)
     assert rc == 1
-    assert re.search(r"no connection to the host", output)
+    assert re.search(r'no connection to the host "127.0.0.1:0"', output)
 
 
 @pytest.mark.parametrize("args, play_error", [
     (
         # Testing with unset uri and .xlog or .snap file.
         "",
-        "required to specify an URI and at least one .xlog or .snap file",
+        "required to specify an URI and at least one .xlog/.snap file or directory",
     ),
     (
         "path-to-non-existent-file",
-        "No such file or directory",
+        "error: could not collect WAL files",
     ),
     (
         ("test.xlog", "--timestamp=abcdef", "--space=999"),
@@ -77,7 +78,7 @@ def test_play_test_remote_instance_timestamp_failed(tt_cmd, test_instance, args,
 ])
 def test_play_remote_instance_timestamp_valid(tt_cmd, test_instance,
                                               input, expected):
-    test_dir = os.path.join(os.path.dirname(__file__), "test_file/timestamp", )
+    test_dir = os.path.join(os.path.dirname(__file__), "test_file", )
 
     # Create space and primary index.
     cmd_space = [tt_cmd, "connect", f"test_user:secret@127.0.0.1:{test_instance.port}",
@@ -88,7 +89,7 @@ def test_play_remote_instance_timestamp_valid(tt_cmd, test_instance,
     # Play .xlog file to the instance.
     cmd_play = [tt_cmd, "play", f"127.0.0.1:{test_instance.port}",
                 "-u", "test_user", "-p", "secret",
-                f"{test_dir}/timestamp.xlog", f"--timestamp={input}"]
+                f"{test_dir}/timestamp/timestamp.xlog", f"--timestamp={input}"]
     rc, _ = run_command_and_get_output(cmd_play, cwd=test_instance._tmpdir)
     assert rc == 0
 
@@ -100,6 +101,39 @@ def test_play_remote_instance_timestamp_valid(tt_cmd, test_instance,
     assert cmd_output == expected
 
 
+@pytest.mark.parametrize("input, expected", [
+    (
+        ("test_file/test.xlog", "test_file/test.snap", "test_file/timestamp"),
+        ('Play is processing file "test_file/test.xlog"',
+         'Play is processing file "test_file/test.snap"',
+         'Play is processing file "test_file/timestamp/timestamp.snap"',
+         'Play is processing file "test_file/timestamp/timestamp.xlog"'),
+    ),
+])
+def test_play_directories_successful(tt_cmd, test_instance,
+                                     input, expected):
+    # Copy files to the "run" directory.
+    test_src = os.path.join(os.path.dirname(__file__), "test_file")
+    test_dir = os.path.join(test_instance._tmpdir, "test_file")
+    shutil.copytree(test_src, test_dir)
+
+    # Create space and primary index.
+    cmd_space = [tt_cmd, "connect", f"test_user:secret@127.0.0.1:{test_instance.port}",
+                 "-f", f"{test_dir}/create_space.lua", "-"]
+    rc, _ = run_command_and_get_output(cmd_space, cwd=test_instance._tmpdir)
+    assert rc == 0
+
+    # Play .xlog file to the remote instance.
+    cmd = [tt_cmd, "play", f"127.0.0.1:{test_instance.port}",
+           "-u", "test_user", "-p", "secret"]
+    cmd.extend(input)
+
+    rc, cmd_output = run_command_and_get_output(cmd, cwd=test_instance._tmpdir)
+    assert rc == 0
+    for item in expected:
+        assert item in cmd_output
+
+
 @pytest.mark.parametrize("opts", [
     pytest.param({"flags": ["--username=test_user", "--password=4"]}),
     pytest.param({"flags": ["--username=fry"]}),