From e7ef91556671cf20e670b87b478acd5d9d813613 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Wed, 3 Jul 2024 12:31:16 -0400 Subject: [PATCH] fix: logs for tasks (#2499) ## Description This PR manipulates `run_sh` and `run_python` logic to stream logs from the command or python script thats being executed on the container. This also allows retrieving task logs after a run has finished, whereas before, task containers would not return logs from `kurtosis service logs`. An improvement to this would be to enable streaming back logs via run output so users don't have to go through Docker Desktop or docker cli to view logs from a long running task that might have err'd. https://www.loom.com/share/89181c37442048b9bdc746bfd37fe2ec ## Is this change user facing? YES --- .../kurtosis_instruction/tasks/run_python.go | 6 +++--- .../kurtosis_instruction/tasks/run_sh.go | 2 +- .../kurtosis_instruction/tasks/tasks_shared.go | 14 ++++++++++++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go index cd117808ac..fd1e820452 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go @@ -320,7 +320,7 @@ func (builtin *RunPythonCapabilities) Execute(ctx context.Context, _ *builtin_ar if err != nil { return "", stacktrace.Propagate(err, "error occurred while preparing the sh command to execute on the image") } - fullCommandToRun := []string{shellWrapperCommand, "-c", commandToRun} + fullCommandToRun := getCommandToRunForStreamingLogs(commandToRun) // run the command passed in by user in the container runPythonExecutionResult, err := executeWithWait(ctx, builtin.serviceNetwork, builtin.name, builtin.wait, fullCommandToRun) @@ -418,7 +418,7 @@ func getPythonCommandToRun(builtin *RunPythonCapabilities) (string, error) { argumentsAsString := strings.Join(maybePythonArgumentsWithRuntimeValueReplaced, spaceDelimiter) runEscaped := strings.ReplaceAll(builtin.run, `"`, `\"`) if len(argumentsAsString) > 0 { - return fmt.Sprintf(`python -c "%s" %s`, runEscaped, argumentsAsString), nil + return fmt.Sprintf(`python -u -c "%s" %s`, runEscaped, argumentsAsString), nil } - return fmt.Sprintf(`python -c "%s"`, runEscaped), nil + return fmt.Sprintf(`python -u -c "%s"`, runEscaped), nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go index feacacf61d..afe8de2ee2 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go @@ -276,7 +276,7 @@ func (builtin *RunShCapabilities) Execute(ctx context.Context, _ *builtin_argume if err != nil { return "", stacktrace.Propagate(err, "error occurred while preparing the sh command to execute on the image") } - fullCommandToRun := []string{shellWrapperCommand, "-c", commandToRun} + fullCommandToRun := getCommandToRunForStreamingLogs(commandToRun) // run the command passed in by user in the container createDefaultDirectoryResult, err := executeWithWait(ctx, builtin.serviceNetwork, builtin.name, builtin.wait, fullCommandToRun) diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go index 379f67e68c..4548366abb 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go @@ -50,6 +50,7 @@ const ( runFilesArtifactsKey = "files_artifacts" shellWrapperCommand = "/bin/sh" + taskLogFilePath = "/tmp/kurtosis-task.log" noNameSet = "" uniqueNameGenErrStr = "error occurred while generating unique name for the file artifact" @@ -57,7 +58,16 @@ const ( tiniEnabled = true ) -var runTailCommandToPreventContainerToStopOnCreating = []string{"tail", "-f", "/dev/null"} +var runCommandToStreamTaskLogs = []string{shellWrapperCommand, "-c", fmt.Sprintf("touch %s && tail -F %s", taskLogFilePath, taskLogFilePath)} + +// Wraps [commandToRun] to enable streaming logs from tasks. +// Uses curly braces to execute the command(s) in the current shell. +// Adds an extra echo to ensure each log ends with a newline. +// Uses tee to direct output to the task log file while maintaining output to stdout. +// Redirects stderr to stdout. +func getCommandToRunForStreamingLogs(commandToRun string) []string { + return []string{shellWrapperCommand, "-c", fmt.Sprintf("{ %v; } %v %v %v %v %v %v %v %v", commandToRun, "2>&1", "|", "tee", taskLogFilePath, "&&", "echo", ">>", taskLogFilePath)} +} func parseStoreFilesArg(serviceNetwork service_network.ServiceNetwork, arguments *builtin_argument.ArgumentValuesSet) ([]*store_spec.StoreSpec, *startosis_errors.InterpretationError) { var result []*store_spec.StoreSpec @@ -276,7 +286,7 @@ func getServiceConfig( filesArtifactExpansion *service_directory.FilesArtifactsExpansion, envVars *map[string]string, ) (*service.ServiceConfig, error) { - serviceConfig, err := service.CreateServiceConfig(maybeImageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, nil, nil, runTailCommandToPreventContainerToStopOnCreating, nil, *envVars, filesArtifactExpansion, nil, 0, 0, service_config.DefaultPrivateIPAddrPlaceholder, 0, 0, map[string]string{}, nil, nil, map[string]string{}, image_download_mode.ImageDownloadMode_Missing, tiniEnabled) + serviceConfig, err := service.CreateServiceConfig(maybeImageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, nil, nil, runCommandToStreamTaskLogs, nil, *envVars, filesArtifactExpansion, nil, 0, 0, service_config.DefaultPrivateIPAddrPlaceholder, 0, 0, map[string]string{}, nil, nil, map[string]string{}, image_download_mode.ImageDownloadMode_Missing, tiniEnabled) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred creating service config") }