Skip to content

Commit

Permalink
Add log_manager_template for job proxy (#428)
Browse files Browse the repository at this point in the history 
* Add log_manager_template

* Add comment
  • Loading branch information
@l0kix2
l0kix2 authored Jan 20, 2025
1 parent e7a93a3 commit 16cf97b
Show file tree
Hide file tree
Showing 12 changed files with 232 additions and 8 deletions.
30 changes: 30 additions & 0 deletions pkg/ytconfig/canondata/TestGetExecNodeConfig/with-job-resources/test.canondata
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,36 @@
};
];
"flush_period"=3000;
"log_manager_template"={
writers={
debug={
type=file;
"file_name"="job-proxy.debug.log.zstd";
format="plain_text";
"compression_method"=zstd;
"enable_compression"=%true;
"enable_system_messages"=%true;
"rotation_policy"={
"rotation_period"=900000;
"max_total_size_to_keep"=3145728;
};
};
};
rules=[
{
"exclude_categories"=[
Bus;
Concurrency;
];
"min_level"=debug;
writers=[
debug;
];
family="plain_text";
};
];
"flush_period"=3000;
};
};
};
"job_proxy_authentication_manager"={
Expand Down
30 changes: 30 additions & 0 deletions pkg/ytconfig/canondata/TestGetExecNodeConfig/without-job-resources/test.canondata
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,36 @@
};
];
"flush_period"=3000;
"log_manager_template"={
writers={
debug={
type=file;
"file_name"="job-proxy.debug.log.zstd";
format="plain_text";
"compression_method"=zstd;
"enable_compression"=%true;
"enable_system_messages"=%true;
"rotation_policy"={
"rotation_period"=900000;
"max_total_size_to_keep"=3145728;
};
};
};
rules=[
{
"exclude_categories"=[
Bus;
Concurrency;
];
"min_level"=debug;
writers=[
debug;
];
family="plain_text";
};
];
"flush_period"=3000;
};
};
};
"job_proxy_authentication_manager"={
Expand Down
30 changes: 30 additions & 0 deletions ...ondata/TestGetExecNodeConfigWithCri/isolated-containers-with-job-resources/test.canondata
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,36 @@
};
];
"flush_period"=3000;
"log_manager_template"={
writers={
debug={
type=file;
"file_name"="job-proxy.debug.log.zstd";
format="plain_text";
"compression_method"=zstd;
"enable_compression"=%true;
"enable_system_messages"=%true;
"rotation_policy"={
"rotation_period"=900000;
"max_total_size_to_keep"=3145728;
};
};
};
rules=[
{
"exclude_categories"=[
Bus;
Concurrency;
];
"min_level"=debug;
writers=[
debug;
];
family="plain_text";
};
];
"flush_period"=3000;
};
};
"forward_all_environment_variables"=%true;
};
Expand Down
30 changes: 30 additions & 0 deletions ...ata/TestGetExecNodeConfigWithCri/isolated-containers-without-job-resources/test.canondata
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,36 @@
};
];
"flush_period"=3000;
"log_manager_template"={
writers={
debug={
type=file;
"file_name"="job-proxy.debug.log.zstd";
format="plain_text";
"compression_method"=zstd;
"enable_compression"=%true;
"enable_system_messages"=%true;
"rotation_policy"={
"rotation_period"=900000;
"max_total_size_to_keep"=3145728;
};
};
};
rules=[
{
"exclude_categories"=[
Bus;
Concurrency;
];
"min_level"=debug;
writers=[
debug;
];
family="plain_text";
};
];
"flush_period"=3000;
};
};
"forward_all_environment_variables"=%true;
};
Expand Down
30 changes: 30 additions & 0 deletions ...canondata/TestGetExecNodeConfigWithCri/single-container-with-job-resources/test.canondata
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,36 @@
};
];
"flush_period"=3000;
"log_manager_template"={
writers={
debug={
type=file;
"file_name"="job-proxy.debug.log.zstd";
format="plain_text";
"compression_method"=zstd;
"enable_compression"=%true;
"enable_system_messages"=%true;
"rotation_policy"={
"rotation_period"=900000;
"max_total_size_to_keep"=3145728;
};
};
};
rules=[
{
"exclude_categories"=[
Bus;
Concurrency;
];
"min_level"=debug;
writers=[
debug;
];
family="plain_text";
};
];
"flush_period"=3000;
};
};
"forward_all_environment_variables"=%true;
};
Expand Down
30 changes: 30 additions & 0 deletions ...ondata/TestGetExecNodeConfigWithCri/single-container-without-job-resources/test.canondata
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,36 @@
};
];
"flush_period"=3000;
"log_manager_template"={
writers={
debug={
type=file;
"file_name"="job-proxy.debug.log.zstd";
format="plain_text";
"compression_method"=zstd;
"enable_compression"=%true;
"enable_system_messages"=%true;
"rotation_policy"={
"rotation_period"=900000;
"max_total_size_to_keep"=3145728;
};
};
};
rules=[
{
"exclude_categories"=[
Bus;
Concurrency;
];
"min_level"=debug;
writers=[
debug;
];
family="plain_text";
};
];
"flush_period"=3000;
};
};
"forward_all_environment_variables"=%true;
};
Expand Down
30 changes: 30 additions & 0 deletions pkg/ytconfig/canondata/TestGetExecNodeWithoutYtsaurusConfig/test.canondata
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,36 @@
};
];
"flush_period"=3000;
"log_manager_template"={
writers={
debug={
type=file;
"file_name"="job-proxy.debug.log.zstd";
format="plain_text";
"compression_method"=zstd;
"enable_compression"=%true;
"enable_system_messages"=%true;
"rotation_policy"={
"rotation_period"=900000;
"max_total_size_to_keep"=3145728;
};
};
};
rules=[
{
"exclude_categories"=[
Bus;
Concurrency;
];
"min_level"=debug;
writers=[
debug;
];
family="plain_text";
};
];
"flush_period"=3000;
};
};
};
"job_proxy_authentication_manager"={
Expand Down
2 changes: 1 addition & 1 deletion pkg/ytconfig/canondata/TestGetStrawberryControllerConfig/test.canondata
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,4 @@
"http_controller_mappings"={
"*"=chyt;
};
}
}
2 changes: 1 addition & 1 deletion pkg/ytconfig/canondata/TestGetStrawberryControllerConfigWithCustomFamilies/test.canondata
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,4 @@
"superservice1.some.domain"=superservice1;
"superservice3.some.domain"=superservice3;
};
}
}
2 changes: 1 addition & 1 deletion ...tconfig/canondata/TestGetStrawberryControllerConfigWithExtendedHTTPMapping/test.canondata
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,4 @@
"*"=chyt;
"jupyt.some.domain"=jupyt;
};
}
}
10 changes: 10 additions & 0 deletions pkg/ytconfig/logging.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,16 @@ type Logging struct {
FlushPeriod int `yson:"flush_period"`
}

type JobProxyLogging struct {
// COMPAT(ignat)
// 23.2 — job_proxy_logging
// 24.1 — job_proxy/job_proxy_logging
// 24.2 — job_proxy/job_proxy_logging/log_manager_template
// Legacy fields can be removed with end of respective server version support.
Logging
LogManagerTemplate Logging `yson:"log_manager_template"`
}

type loggingBuilder struct {
loggingDirectory string
componentName string
Expand Down
14 changes: 9 additions & 5 deletions pkg/ytconfig/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,9 +153,9 @@ type JobResourceManager struct {
}

type JobProxy struct {
JobProxyAuthenticationManager Auth `yson:"job_proxy_authentication_manager"`
JobProxyLogging Logging `yson:"job_proxy_logging"`
ForwardAllEnvironmentVariables *bool `yson:"forward_all_environment_variables,omitempty"`
JobProxyAuthenticationManager Auth `yson:"job_proxy_authentication_manager"`
JobProxyLogging JobProxyLogging `yson:"job_proxy_logging"`
ForwardAllEnvironmentVariables *bool `yson:"forward_all_environment_variables,omitempty"`
}

type ExecNode struct {
Expand Down Expand Up @@ -561,15 +561,19 @@ func getExecNodeServerCarcass(spec *ytv1.ExecNodesSpec, commonSpec *ytv1.CommonS
}
}
jobProxyLoggingBuilder.logging.FlushPeriod = 3000
c.ExecNode.JobProxy.JobProxyLogging = jobProxyLoggingBuilder.logging
jobProxyLogging := jobProxyLoggingBuilder.logging
c.ExecNode.JobProxy.JobProxyLogging = JobProxyLogging{
Logging: jobProxyLogging,
LogManagerTemplate: jobProxyLogging,
}

c.ExecNode.JobProxy.JobProxyAuthenticationManager.RequireAuthentication = true
c.ExecNode.JobProxy.JobProxyAuthenticationManager.CypressTokenAuthenticator.Secure = true

// TODO(khlebnikov): Drop legacy fields depending on ytsaurus version.
c.ExecNode.JobController.ResourceLimitsLegacy = &c.JobResourceManager.ResourceLimits
c.ExecNode.JobController.GpuManagerLegacy = &c.ExecNode.GpuManager
c.ExecNode.JobProxyLoggingLegacy = &c.ExecNode.JobProxy.JobProxyLogging
c.ExecNode.JobProxyLoggingLegacy = &jobProxyLogging
c.ExecNode.JobProxyAuthenticationManagerLegacy = &c.ExecNode.JobProxy.JobProxyAuthenticationManager
c.ExecNode.DoNotSetUserIdLegacy = c.ExecNode.SlotManager.DoNotSetUserId
c.ExecNode.ForwardAllEnvironmentVariablesLegacy = c.ExecNode.JobProxy.ForwardAllEnvironmentVariables
Expand Down

0 comments on commit 16cf97b

Please sign in to comment.