diff --git a/docs/docset.yml b/docs/docset.yml new file mode 100644 index 00000000..afdd743b --- /dev/null +++ b/docs/docset.yml @@ -0,0 +1,490 @@ +project: 'Elastic Serverless Forwarder for AWS' +exclude: + - README-AWS.md +cross_links: + - docs-content + - elasticsearch + - integration-docs + - logstash +toc: + - toc: reference +subs: + ref: "https://www.elastic.co/guide/en/elasticsearch/reference/current" + ref-bare: "https://www.elastic.co/guide/en/elasticsearch/reference" + ref-8x: "https://www.elastic.co/guide/en/elasticsearch/reference/8.1" + ref-80: "https://www.elastic.co/guide/en/elasticsearch/reference/8.0" + ref-7x: "https://www.elastic.co/guide/en/elasticsearch/reference/7.17" + ref-70: "https://www.elastic.co/guide/en/elasticsearch/reference/7.0" + ref-60: "https://www.elastic.co/guide/en/elasticsearch/reference/6.0" + ref-64: "https://www.elastic.co/guide/en/elasticsearch/reference/6.4" + xpack-ref: "https://www.elastic.co/guide/en/x-pack/6.2" + logstash-ref: "https://www.elastic.co/guide/en/logstash/current" + kibana-ref: "https://www.elastic.co/guide/en/kibana/current" + kibana-ref-all: "https://www.elastic.co/guide/en/kibana" + beats-ref-root: "https://www.elastic.co/guide/en/beats" + beats-ref: "https://www.elastic.co/guide/en/beats/libbeat/current" + beats-ref-60: "https://www.elastic.co/guide/en/beats/libbeat/6.0" + beats-ref-63: "https://www.elastic.co/guide/en/beats/libbeat/6.3" + beats-devguide: "https://www.elastic.co/guide/en/beats/devguide/current" + auditbeat-ref: "https://www.elastic.co/guide/en/beats/auditbeat/current" + packetbeat-ref: "https://www.elastic.co/guide/en/beats/packetbeat/current" + metricbeat-ref: "https://www.elastic.co/guide/en/beats/metricbeat/current" + filebeat-ref: "https://www.elastic.co/guide/en/beats/filebeat/current" + functionbeat-ref: "https://www.elastic.co/guide/en/beats/functionbeat/current" + winlogbeat-ref: "https://www.elastic.co/guide/en/beats/winlogbeat/current" + heartbeat-ref: "https://www.elastic.co/guide/en/beats/heartbeat/current" + journalbeat-ref: "https://www.elastic.co/guide/en/beats/journalbeat/current" + ingest-guide: "https://www.elastic.co/guide/en/ingest/current" + fleet-guide: "https://www.elastic.co/guide/en/fleet/current" + apm-guide-ref: "https://www.elastic.co/guide/en/apm/guide/current" + apm-guide-7x: "https://www.elastic.co/guide/en/apm/guide/7.17" + apm-app-ref: "https://www.elastic.co/guide/en/kibana/current" + apm-agents-ref: "https://www.elastic.co/guide/en/apm/agent" + apm-android-ref: "https://www.elastic.co/guide/en/apm/agent/android/current" + apm-py-ref: "https://www.elastic.co/guide/en/apm/agent/python/current" + apm-py-ref-3x: "https://www.elastic.co/guide/en/apm/agent/python/3.x" + apm-node-ref-index: "https://www.elastic.co/guide/en/apm/agent/nodejs" + apm-node-ref: "https://www.elastic.co/guide/en/apm/agent/nodejs/current" + apm-node-ref-1x: "https://www.elastic.co/guide/en/apm/agent/nodejs/1.x" + apm-rum-ref: "https://www.elastic.co/guide/en/apm/agent/rum-js/current" + apm-ruby-ref: "https://www.elastic.co/guide/en/apm/agent/ruby/current" + apm-java-ref: "https://www.elastic.co/guide/en/apm/agent/java/current" + apm-go-ref: "https://www.elastic.co/guide/en/apm/agent/go/current" + apm-dotnet-ref: "https://www.elastic.co/guide/en/apm/agent/dotnet/current" + apm-php-ref: "https://www.elastic.co/guide/en/apm/agent/php/current" + apm-ios-ref: "https://www.elastic.co/guide/en/apm/agent/swift/current" + apm-lambda-ref: "https://www.elastic.co/guide/en/apm/lambda/current" + apm-attacher-ref: "https://www.elastic.co/guide/en/apm/attacher/current" + docker-logging-ref: "https://www.elastic.co/guide/en/beats/loggingplugin/current" + esf-ref: "https://www.elastic.co/guide/en/esf/current" + kinesis-firehose-ref: "https://www.elastic.co/guide/en/kinesis/{{kinesis_version}}" + estc-welcome-current: "https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions/current" + estc-welcome: "https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions/current" + estc-welcome-all: "https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions" + hadoop-ref: "https://www.elastic.co/guide/en/elasticsearch/hadoop/current" + stack-ref: "https://www.elastic.co/guide/en/elastic-stack/current" + stack-ref-67: "https://www.elastic.co/guide/en/elastic-stack/6.7" + stack-ref-68: "https://www.elastic.co/guide/en/elastic-stack/6.8" + stack-ref-70: "https://www.elastic.co/guide/en/elastic-stack/7.0" + stack-ref-80: "https://www.elastic.co/guide/en/elastic-stack/8.0" + stack-ov: "https://www.elastic.co/guide/en/elastic-stack-overview/current" + stack-gs: "https://www.elastic.co/guide/en/elastic-stack-get-started/current" + stack-gs-current: "https://www.elastic.co/guide/en/elastic-stack-get-started/current" + javaclient: "https://www.elastic.co/guide/en/elasticsearch/client/java-api/current" + java-api-client: "https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/current" + java-rest: "https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current" + jsclient: "https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current" + jsclient-current: "https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current" + es-ruby-client: "https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/current" + es-dotnet-client: "https://www.elastic.co/guide/en/elasticsearch/client/net-api/current" + es-php-client: "https://www.elastic.co/guide/en/elasticsearch/client/php-api/current" + es-python-client: "https://www.elastic.co/guide/en/elasticsearch/client/python-api/current" + defguide: "https://www.elastic.co/guide/en/elasticsearch/guide/2.x" + painless: "https://www.elastic.co/guide/en/elasticsearch/painless/current" + plugins: "https://www.elastic.co/guide/en/elasticsearch/plugins/current" + plugins-8x: "https://www.elastic.co/guide/en/elasticsearch/plugins/8.1" + plugins-7x: "https://www.elastic.co/guide/en/elasticsearch/plugins/7.17" + plugins-6x: "https://www.elastic.co/guide/en/elasticsearch/plugins/6.8" + glossary: "https://www.elastic.co/guide/en/elastic-stack-glossary/current" + upgrade_guide: "https://www.elastic.co/products/upgrade_guide" + blog-ref: "https://www.elastic.co/blog/" + curator-ref: "https://www.elastic.co/guide/en/elasticsearch/client/curator/current" + curator-ref-current: "https://www.elastic.co/guide/en/elasticsearch/client/curator/current" + metrics-ref: "https://www.elastic.co/guide/en/metrics/current" + metrics-guide: "https://www.elastic.co/guide/en/metrics/guide/current" + logs-ref: "https://www.elastic.co/guide/en/logs/current" + logs-guide: "https://www.elastic.co/guide/en/logs/guide/current" + uptime-guide: "https://www.elastic.co/guide/en/uptime/current" + observability-guide: "https://www.elastic.co/guide/en/observability/current" + observability-guide-all: "https://www.elastic.co/guide/en/observability" + siem-guide: "https://www.elastic.co/guide/en/siem/guide/current" + security-guide: "https://www.elastic.co/guide/en/security/current" + security-guide-all: "https://www.elastic.co/guide/en/security" + endpoint-guide: "https://www.elastic.co/guide/en/endpoint/current" + sql-odbc: "https://www.elastic.co/guide/en/elasticsearch/sql-odbc/current" + ecs-ref: "https://www.elastic.co/guide/en/ecs/current" + ecs-logging-ref: "https://www.elastic.co/guide/en/ecs-logging/overview/current" + ecs-logging-go-logrus-ref: "https://www.elastic.co/guide/en/ecs-logging/go-logrus/current" + ecs-logging-go-zap-ref: "https://www.elastic.co/guide/en/ecs-logging/go-zap/current" + ecs-logging-go-zerolog-ref: "https://www.elastic.co/guide/en/ecs-logging/go-zap/current" + ecs-logging-java-ref: "https://www.elastic.co/guide/en/ecs-logging/java/current" + ecs-logging-dotnet-ref: "https://www.elastic.co/guide/en/ecs-logging/dotnet/current" + ecs-logging-nodejs-ref: "https://www.elastic.co/guide/en/ecs-logging/nodejs/current" + ecs-logging-php-ref: "https://www.elastic.co/guide/en/ecs-logging/php/current" + ecs-logging-python-ref: "https://www.elastic.co/guide/en/ecs-logging/python/current" + ecs-logging-ruby-ref: "https://www.elastic.co/guide/en/ecs-logging/ruby/current" + ml-docs: "https://www.elastic.co/guide/en/machine-learning/current" + eland-docs: "https://www.elastic.co/guide/en/elasticsearch/client/eland/current" + eql-ref: "https://eql.readthedocs.io/en/latest/query-guide" + extendtrial: "https://www.elastic.co/trialextension" + wikipedia: "https://en.wikipedia.org/wiki" + forum: "https://discuss.elastic.co/" + xpack-forum: "https://discuss.elastic.co/c/50-x-pack" + security-forum: "https://discuss.elastic.co/c/x-pack/shield" + watcher-forum: "https://discuss.elastic.co/c/x-pack/watcher" + monitoring-forum: "https://discuss.elastic.co/c/x-pack/marvel" + graph-forum: "https://discuss.elastic.co/c/x-pack/graph" + apm-forum: "https://discuss.elastic.co/c/apm" + enterprise-search-ref: "https://www.elastic.co/guide/en/enterprise-search/current" + app-search-ref: "https://www.elastic.co/guide/en/app-search/current" + workplace-search-ref: "https://www.elastic.co/guide/en/workplace-search/current" + enterprise-search-node-ref: "https://www.elastic.co/guide/en/enterprise-search-clients/enterprise-search-node/current" + enterprise-search-php-ref: "https://www.elastic.co/guide/en/enterprise-search-clients/php/current" + enterprise-search-python-ref: "https://www.elastic.co/guide/en/enterprise-search-clients/python/current" + enterprise-search-ruby-ref: "https://www.elastic.co/guide/en/enterprise-search-clients/ruby/current" + elastic-maps-service: "https://maps.elastic.co" + integrations-docs: "https://docs.elastic.co/en/integrations" + integrations-devguide: "https://www.elastic.co/guide/en/integrations-developer/current" + time-units: "https://www.elastic.co/guide/en/elasticsearch/reference/current/api-conventions.html#time-units" + byte-units: "https://www.elastic.co/guide/en/elasticsearch/reference/current/api-conventions.html#byte-units" + apm-py-ref-v: "https://www.elastic.co/guide/en/apm/agent/python/current" + apm-node-ref-v: "https://www.elastic.co/guide/en/apm/agent/nodejs/current" + apm-rum-ref-v: "https://www.elastic.co/guide/en/apm/agent/rum-js/current" + apm-ruby-ref-v: "https://www.elastic.co/guide/en/apm/agent/ruby/current" + apm-java-ref-v: "https://www.elastic.co/guide/en/apm/agent/java/current" + apm-go-ref-v: "https://www.elastic.co/guide/en/apm/agent/go/current" + apm-ios-ref-v: "https://www.elastic.co/guide/en/apm/agent/swift/current" + apm-dotnet-ref-v: "https://www.elastic.co/guide/en/apm/agent/dotnet/current" + apm-php-ref-v: "https://www.elastic.co/guide/en/apm/agent/php/current" + ecloud: "Elastic Cloud" + esf: "Elastic Serverless Forwarder" + ess: "Elasticsearch Service" + ece: "Elastic Cloud Enterprise" + eck: "Elastic Cloud on Kubernetes" + serverless-full: "Elastic Cloud Serverless" + serverless-short: "Serverless" + es-serverless: "Elasticsearch Serverless" + es3: "Elasticsearch Serverless" + obs-serverless: "Elastic Observability Serverless" + sec-serverless: "Elastic Security Serverless" + serverless-docs: "https://docs.elastic.co/serverless" + cloud: "https://www.elastic.co/guide/en/cloud/current" + ess-utm-params: "?page=docs&placement=docs-body" + ess-baymax: "?page=docs&placement=docs-body" + ess-trial: "https://cloud.elastic.co/registration?page=docs&placement=docs-body" + ess-product: "https://www.elastic.co/cloud/elasticsearch-service?page=docs&placement=docs-body" + ess-console: "https://cloud.elastic.co?page=docs&placement=docs-body" + ess-console-name: "Elasticsearch Service Console" + ess-deployments: "https://cloud.elastic.co/deployments?page=docs&placement=docs-body" + ece-ref: "https://www.elastic.co/guide/en/cloud-enterprise/current" + eck-ref: "https://www.elastic.co/guide/en/cloud-on-k8s/current" + ess-leadin: "You can run Elasticsearch on your own hardware or use our hosted Elasticsearch Service that is available on AWS, GCP, and Azure. https://cloud.elastic.co/registration{ess-utm-params}[Try the Elasticsearch Service for free]." + ess-leadin-short: "Our hosted Elasticsearch Service is available on AWS, GCP, and Azure, and you can https://cloud.elastic.co/registration{ess-utm-params}[try it for free]." + ess-icon: "image:https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg[link=\"https://cloud.elastic.co/registration{ess-utm-params}\", title=\"Supported on Elasticsearch Service\"]" + ece-icon: "image:https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud_ece.svg[link=\"https://cloud.elastic.co/registration{ess-utm-params}\", title=\"Supported on Elastic Cloud Enterprise\"]" + cloud-only: "This feature is designed for indirect use by https://cloud.elastic.co/registration{ess-utm-params}[Elasticsearch Service], https://www.elastic.co/guide/en/cloud-enterprise/{ece-version-link}[Elastic Cloud Enterprise], and https://www.elastic.co/guide/en/cloud-on-k8s/current[Elastic Cloud on Kubernetes]. Direct use is not supported." + ess-setting-change: "image:https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg[link=\"{ess-trial}\", title=\"Supported on {ess}\"] indicates a change to a supported https://www.elastic.co/guide/en/cloud/current/ec-add-user-settings.html[user setting] for Elasticsearch Service." + ess-skip-section: "If you use Elasticsearch Service, skip this section. Elasticsearch Service handles these changes for you." + api-cloud: "https://www.elastic.co/docs/api/doc/cloud" + api-ece: "https://www.elastic.co/docs/api/doc/cloud-enterprise" + api-kibana-serverless: "https://www.elastic.co/docs/api/doc/serverless" + es-feature-flag: "This feature is in development and not yet available for use. This documentation is provided for informational purposes only." + es-ref-dir: "'{{elasticsearch-root}}/docs/reference'" + apm-app: "APM app" + uptime-app: "Uptime app" + synthetics-app: "Synthetics app" + logs-app: "Logs app" + metrics-app: "Metrics app" + infrastructure-app: "Infrastructure app" + siem-app: "SIEM app" + security-app: "Elastic Security app" + ml-app: "Machine Learning" + dev-tools-app: "Dev Tools" + ingest-manager-app: "Ingest Manager" + stack-manage-app: "Stack Management" + stack-monitor-app: "Stack Monitoring" + alerts-ui: "Alerts and Actions" + rules-ui: "Rules" + rac-ui: "Rules and Connectors" + connectors-ui: "Connectors" + connectors-feature: "Actions and Connectors" + stack-rules-feature: "Stack Rules" + user-experience: "User Experience" + ems: "Elastic Maps Service" + ems-init: "EMS" + hosted-ems: "Elastic Maps Server" + ipm-app: "Index Pattern Management" + ingest-pipelines: "ingest pipelines" + ingest-pipelines-app: "Ingest Pipelines" + ingest-pipelines-cap: "Ingest pipelines" + ls-pipelines: "Logstash pipelines" + ls-pipelines-app: "Logstash Pipelines" + maint-windows: "maintenance windows" + maint-windows-app: "Maintenance Windows" + maint-windows-cap: "Maintenance windows" + custom-roles-app: "Custom Roles" + data-source: "data view" + data-sources: "data views" + data-source-caps: "Data View" + data-sources-caps: "Data Views" + data-source-cap: "Data view" + data-sources-cap: "Data views" + project-settings: "Project settings" + manage-app: "Management" + index-manage-app: "Index Management" + data-views-app: "Data Views" + rules-app: "Rules" + saved-objects-app: "Saved Objects" + tags-app: "Tags" + api-keys-app: "API keys" + transforms-app: "Transforms" + connectors-app: "Connectors" + files-app: "Files" + reports-app: "Reports" + maps-app: "Maps" + alerts-app: "Alerts" + crawler: "Enterprise Search web crawler" + ents: "Enterprise Search" + app-search-crawler: "App Search web crawler" + agent: "Elastic Agent" + agents: "Elastic Agents" + fleet: "Fleet" + fleet-server: "Fleet Server" + integrations-server: "Integrations Server" + ingest-manager: "Ingest Manager" + ingest-management: "ingest management" + package-manager: "Elastic Package Manager" + integrations: "Integrations" + package-registry: "Elastic Package Registry" + artifact-registry: "Elastic Artifact Registry" + aws: "AWS" + stack: "Elastic Stack" + xpack: "X-Pack" + es: "Elasticsearch" + kib: "Kibana" + esms: "Elastic Stack Monitoring Service" + esms-init: "ESMS" + ls: "Logstash" + beats: "Beats" + auditbeat: "Auditbeat" + filebeat: "Filebeat" + heartbeat: "Heartbeat" + metricbeat: "Metricbeat" + packetbeat: "Packetbeat" + winlogbeat: "Winlogbeat" + functionbeat: "Functionbeat" + journalbeat: "Journalbeat" + es-sql: "Elasticsearch SQL" + esql: "ES|QL" + elastic-agent: "Elastic Agent" + k8s: "Kubernetes" + log-driver-long: "Elastic Logging Plugin for Docker" + security: "X-Pack security" + security-features: "security features" + operator-feature: "operator privileges feature" + es-security-features: "Elasticsearch security features" + stack-security-features: "Elastic Stack security features" + endpoint-sec: "Endpoint Security" + endpoint-cloud-sec: "Endpoint and Cloud Security" + elastic-defend: "Elastic Defend" + elastic-sec: "Elastic Security" + elastic-endpoint: "Elastic Endpoint" + swimlane: "Swimlane" + sn: "ServiceNow" + sn-itsm: "ServiceNow ITSM" + sn-itom: "ServiceNow ITOM" + sn-sir: "ServiceNow SecOps" + jira: "Jira" + ibm-r: "IBM Resilient" + webhook: "Webhook" + webhook-cm: "Webhook - Case Management" + opsgenie: "Opsgenie" + bedrock: "Amazon Bedrock" + gemini: "Google Gemini" + hive: "TheHive" + monitoring: "X-Pack monitoring" + monitor-features: "monitoring features" + stack-monitor-features: "Elastic Stack monitoring features" + watcher: "Watcher" + alert-features: "alerting features" + reporting: "X-Pack reporting" + report-features: "reporting features" + graph: "X-Pack graph" + graph-features: "graph analytics features" + searchprofiler: "Search Profiler" + xpackml: "X-Pack machine learning" + ml: "machine learning" + ml-cap: "Machine learning" + ml-init: "ML" + ml-features: "machine learning features" + stack-ml-features: "Elastic Stack machine learning features" + ccr: "cross-cluster replication" + ccr-cap: "Cross-cluster replication" + ccr-init: "CCR" + ccs: "cross-cluster search" + ccs-cap: "Cross-cluster search" + ccs-init: "CCS" + ilm: "index lifecycle management" + ilm-cap: "Index lifecycle management" + ilm-init: "ILM" + dlm: "data lifecycle management" + dlm-cap: "Data lifecycle management" + dlm-init: "DLM" + search-snap: "searchable snapshot" + search-snaps: "searchable snapshots" + search-snaps-cap: "Searchable snapshots" + slm: "snapshot lifecycle management" + slm-cap: "Snapshot lifecycle management" + slm-init: "SLM" + rollup-features: "data rollup features" + ipm: "index pattern management" + ipm-cap: "Index pattern" + rollup: "rollup" + rollup-cap: "Rollup" + rollups: "rollups" + rollups-cap: "Rollups" + rollup-job: "rollup job" + rollup-jobs: "rollup jobs" + rollup-jobs-cap: "Rollup jobs" + dfeed: "datafeed" + dfeeds: "datafeeds" + dfeed-cap: "Datafeed" + dfeeds-cap: "Datafeeds" + ml-jobs: "machine learning jobs" + ml-jobs-cap: "Machine learning jobs" + anomaly-detect: "anomaly detection" + anomaly-detect-cap: "Anomaly detection" + anomaly-job: "anomaly detection job" + anomaly-jobs: "anomaly detection jobs" + anomaly-jobs-cap: "Anomaly detection jobs" + dataframe: "data frame" + dataframes: "data frames" + dataframe-cap: "Data frame" + dataframes-cap: "Data frames" + watcher-transform: "payload transform" + watcher-transforms: "payload transforms" + watcher-transform-cap: "Payload transform" + watcher-transforms-cap: "Payload transforms" + transform: "transform" + transforms: "transforms" + transform-cap: "Transform" + transforms-cap: "Transforms" + dataframe-transform: "transform" + dataframe-transform-cap: "Transform" + dataframe-transforms: "transforms" + dataframe-transforms-cap: "Transforms" + dfanalytics-cap: "Data frame analytics" + dfanalytics: "data frame analytics" + dataframe-analytics-config: "'{dataframe} analytics config'" + dfanalytics-job: "'{dataframe} analytics job'" + dfanalytics-jobs: "'{dataframe} analytics jobs'" + dfanalytics-jobs-cap: "'{dataframe-cap} analytics jobs'" + cdataframe: "continuous data frame" + cdataframes: "continuous data frames" + cdataframe-cap: "Continuous data frame" + cdataframes-cap: "Continuous data frames" + cdataframe-transform: "continuous transform" + cdataframe-transforms: "continuous transforms" + cdataframe-transforms-cap: "Continuous transforms" + ctransform: "continuous transform" + ctransform-cap: "Continuous transform" + ctransforms: "continuous transforms" + ctransforms-cap: "Continuous transforms" + oldetection: "outlier detection" + oldetection-cap: "Outlier detection" + olscore: "outlier score" + olscores: "outlier scores" + fiscore: "feature influence score" + evaluatedf-api: "evaluate {dataframe} analytics API" + evaluatedf-api-cap: "Evaluate {dataframe} analytics API" + binarysc: "binary soft classification" + binarysc-cap: "Binary soft classification" + regression: "regression" + regression-cap: "Regression" + reganalysis: "regression analysis" + reganalysis-cap: "Regression analysis" + depvar: "dependent variable" + feature-var: "feature variable" + feature-vars: "feature variables" + feature-vars-cap: "Feature variables" + classification: "classification" + classification-cap: "Classification" + classanalysis: "classification analysis" + classanalysis-cap: "Classification analysis" + infer-cap: "Inference" + infer: "inference" + lang-ident-cap: "Language identification" + lang-ident: "language identification" + data-viz: "Data Visualizer" + file-data-viz: "File Data Visualizer" + feat-imp: "feature importance" + feat-imp-cap: "Feature importance" + nlp: "natural language processing" + nlp-cap: "Natural language processing" + apm-agent: "APM agent" + apm-go-agent: "Elastic APM Go agent" + apm-go-agents: "Elastic APM Go agents" + apm-ios-agent: "Elastic APM iOS agent" + apm-ios-agents: "Elastic APM iOS agents" + apm-java-agent: "Elastic APM Java agent" + apm-java-agents: "Elastic APM Java agents" + apm-dotnet-agent: "Elastic APM .NET agent" + apm-dotnet-agents: "Elastic APM .NET agents" + apm-node-agent: "Elastic APM Node.js agent" + apm-node-agents: "Elastic APM Node.js agents" + apm-php-agent: "Elastic APM PHP agent" + apm-php-agents: "Elastic APM PHP agents" + apm-py-agent: "Elastic APM Python agent" + apm-py-agents: "Elastic APM Python agents" + apm-ruby-agent: "Elastic APM Ruby agent" + apm-ruby-agents: "Elastic APM Ruby agents" + apm-rum-agent: "Elastic APM Real User Monitoring (RUM) JavaScript agent" + apm-rum-agents: "Elastic APM RUM JavaScript agents" + apm-lambda-ext: "Elastic APM AWS Lambda extension" + project-monitors: "project monitors" + project-monitors-cap: "Project monitors" + private-location: "Private Location" + private-locations: "Private Locations" + pwd: "YOUR_PASSWORD" + esh: "ES-Hadoop" + default-dist: "default distribution" + oss-dist: "OSS-only distribution" + observability: "Observability" + api-request-title: "Request" + api-prereq-title: "Prerequisites" + api-description-title: "Description" + api-path-parms-title: "Path parameters" + api-query-parms-title: "Query parameters" + api-request-body-title: "Request body" + api-response-codes-title: "Response codes" + api-response-body-title: "Response body" + api-example-title: "Example" + api-examples-title: "Examples" + api-definitions-title: "Properties" + multi-arg: "†footnoteref:[multi-arg,This parameter accepts multiple arguments.]" + multi-arg-ref: "†footnoteref:[multi-arg]" + yes-icon: "image:https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png[Yes,20,15]" + no-icon: "image:https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png[No,20,15]" + es-repo: "https://github.com/elastic/elasticsearch/" + es-issue: "https://github.com/elastic/elasticsearch/issues/" + es-pull: "https://github.com/elastic/elasticsearch/pull/" + es-commit: "https://github.com/elastic/elasticsearch/commit/" + kib-repo: "https://github.com/elastic/kibana/" + kib-issue: "https://github.com/elastic/kibana/issues/" + kibana-issue: "'{kib-repo}issues/'" + kib-pull: "https://github.com/elastic/kibana/pull/" + kibana-pull: "'{kib-repo}pull/'" + kib-commit: "https://github.com/elastic/kibana/commit/" + ml-repo: "https://github.com/elastic/ml-cpp/" + ml-issue: "https://github.com/elastic/ml-cpp/issues/" + ml-pull: "https://github.com/elastic/ml-cpp/pull/" + ml-commit: "https://github.com/elastic/ml-cpp/commit/" + apm-repo: "https://github.com/elastic/apm-server/" + apm-issue: "https://github.com/elastic/apm-server/issues/" + apm-pull: "https://github.com/elastic/apm-server/pull/" + kibana-blob: "https://github.com/elastic/kibana/blob/current/" + apm-get-started-ref: "https://www.elastic.co/guide/en/apm/get-started/current" + apm-server-ref: "https://www.elastic.co/guide/en/apm/server/current" + apm-server-ref-v: "https://www.elastic.co/guide/en/apm/server/current" + apm-server-ref-m: "https://www.elastic.co/guide/en/apm/server/master" + apm-server-ref-62: "https://www.elastic.co/guide/en/apm/server/6.2" + apm-server-ref-64: "https://www.elastic.co/guide/en/apm/server/6.4" + apm-server-ref-70: "https://www.elastic.co/guide/en/apm/server/7.0" + apm-overview-ref-v: "https://www.elastic.co/guide/en/apm/get-started/current" + apm-overview-ref-70: "https://www.elastic.co/guide/en/apm/get-started/7.0" + apm-overview-ref-m: "https://www.elastic.co/guide/en/apm/get-started/master" + infra-guide: "https://www.elastic.co/guide/en/infrastructure/guide/current" + a-data-source: "a data view" + icon-bug: "pass:[]" + icon-checkInCircleFilled: "pass:[]" + icon-warningFilled: "pass:[]" diff --git a/docs/en/aws-deploy-elastic-serverless-forwarder.asciidoc b/docs/en/aws-deploy-elastic-serverless-forwarder.asciidoc deleted file mode 100644 index d5225c65..00000000 --- a/docs/en/aws-deploy-elastic-serverless-forwarder.asciidoc +++ /dev/null @@ -1,691 +0,0 @@ -:aws: AWS - -[[aws-deploy-elastic-serverless-forwarder]] -= Deploy Elastic Serverless Forwarder - -++++ -Deploy serverless forwarder -++++ -:keywords: serverless, AWS, SAR -:description: Deploy the Elastic Serverless Forwarder using Kibana and the AWS Serverless Application Repository (SAR). - -To deploy Elastic Serverless Forwarder, you have to: - -* <> -* <> -* <> -* <> - -[discrete] -[[aws-serverless-forwarder-deploy-prereq]] -== Prerequisites -This documentation assumes you have some familiarity with {aws} services, and you have correctly created and configured the necessary {aws} objects. -// Need more details on pre-reqs for other input types - -NOTE: This page describes the basic steps required to deploy Elastic Serverless -Forwarder for {aws}— for additional information on configuration topics such as permissions and automatic routing, and parsing and enriching data, see <>. - -[discrete] -[[aws-serverless-forwarder-deploy-direct-note]] -== Deploying directly without SAR -If the customization options available when deploying via Serverless Application Repository (SAR) are not sufficient, from version `1.6.0` and above you can <> to your {aws} Account without using SAR. This enables you to customize the event source settings for the inputs (i.e. triggers) one-by-one. - -[discrete] -[[aws-serverless-forwarder-deploy-kibana]] -== Install {aws} integration assets in {kib} - -. Go to **Integrations** in {kib} and search for {aws} (or select the **{aws}** - category to filter the list). -. Click the {aws} integration, select **Settings** and click -**Install {aws} assets** and confirm to install all the {aws} integration assets. - -[role="screenshot"] -image::images/aws-serverless-forwarder-install-assets.png[Find and install AWS integration assets in {kib}] - -Adding integrations from {kib} provides appropriate pre-built dashboards, -ingest node configurations, and other assets that help you get the most out of -the data you ingest. The integrations use https://www.elastic.co/guide/en/elasticsearch/reference/current/data-streams.html[data streams] -with specific https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme[naming conventions] -that provide you with more granular controls and flexibility on managing data ingestion. - -NOTE: We recommend using integration assets to get started but the forwarder supports writing to any index, alias, or custom data stream. This enables existing Elasticsearch users to re-use index templates, ingest pipelines, or dashboards that are already created and connected to existing processes or systems. If you already have an existing index or data stream you intend to send data to, then you can skip this deployment step. - -[discrete] -[[sample-s3-config-file]] -== Create and upload `config.yaml` to S3 bucket - -Elastic Serverless Forwarder requires a `config.yaml` file to be uploaded to an S3 bucket and referenced by the `S3_CONFIG_FILE` environment variable. - -Save the following YAML content as `config.yaml` and edit as required before uploading to an S3 bucket. You should remove any inputs or arguments you are not using, and ensure you have entered the correct URLs and credentials as per the inline comments. - -[source, yaml] ----- - -inputs: - - type: "s3-sqs" - id: "arn:aws:sqs:%REGION%:%ACCOUNT%:%QUEUENAME%" - outputs: - - type: "elasticsearch" - args: - # either elasticsearch_url or cloud_id, elasticsearch_url takes precedence if both are included - elasticsearch_url: "http(s)://domain.tld:port" - cloud_id: "cloud_id:bG9jYWxob3N0OjkyMDAkMA==" - # either api_key or username/password, username/password takes precedence if both are included - api_key: "YXBpX2tleV9pZDphcGlfa2V5X3NlY3JldAo=" - username: "username" - password: "password" - es_datastream_name: "logs-generic-default" - es_dead_letter_index: "esf-dead-letter-index" # optional - batch_max_actions: 500 # optional: default value is 500 - batch_max_bytes: 10485760 # optional: default value is 10485760 - - type: "logstash" - args: - logstash_url: "http(s)://host:port" - username: "username" #optional - password: "password" #optional - max_batch_size: 500 #optional - compression_level: 1 #optional - ssl_assert_fingerprint: "22:F7:FB:84:1D:43:3E:E7:BB:F9:72:F3:D8:97:AD:7C:86:E3:08:42" #optional - - type: "sqs" - id: "arn:aws:sqs:%REGION%:%ACCOUNT%:%QUEUENAME%" - outputs: - - type: "elasticsearch" - args: - # either elasticsearch_url or cloud_id, elasticsearch_url takes precedence if both are included - elasticsearch_url: "http(s)://domain.tld:port" - cloud_id: "cloud_id:bG9jYWxob3N0OjkyMDAkMA==" - # either api_key or username/password, username/password takes precedence if both are included - api_key: "YXBpX2tleV9pZDphcGlfa2V5X3NlY3JldAo=" - username: "username" - password: "password" - es_datastream_name: "logs-generic-default" - es_dead_letter_index: "esf-dead-letter-index" # optional - batch_max_actions: 500 # optional: default value is 500 - batch_max_bytes: 10485760 # optional: default value is 10485760 - - type: "logstash" - args: - logstash_url: "http(s)://host:port" - username: "username" #optional - password: "password" #optional - max_batch_size: 500 #optional - compression_level: 1 #optional - ssl_assert_fingerprint: "22:F7:FB:84:1D:43:3E:E7:BB:F9:72:F3:D8:97:AD:7C:86:E3:08:42" #optional - - type: "kinesis-data-stream" - id: "arn:aws:kinesis:%REGION%:%ACCOUNT%:stream/%STREAMNAME%" - outputs: - - type: "elasticsearch" - args: - # either elasticsearch_url or cloud_id, elasticsearch_url takes precedence if both are included - elasticsearch_url: "http(s)://domain.tld:port" - cloud_id: "cloud_id:bG9jYWxob3N0OjkyMDAkMA==" - # either api_key or username/password, username/password takes precedence if both are included - api_key: "YXBpX2tleV9pZDphcGlfa2V5X3NlY3JldAo=" - username: "username" - password: "password" - es_datastream_name: "logs-generic-default" - es_dead_letter_index: "esf-dead-letter-index" # optional - batch_max_actions: 500 # optional: default value is 500 - batch_max_bytes: 10485760 # optional: default value is 10485760 - - type: "logstash" - args: - logstash_url: "http(s)://host:port" - username: "username" #optional - password: "password" #optional - max_batch_size: 500 #optional - compression_level: 1 #optional - ssl_assert_fingerprint: "22:F7:FB:84:1D:43:3E:E7:BB:F9:72:F3:D8:97:AD:7C:86:E3:08:42" #optional - - type: "cloudwatch-logs" - id: "arn:aws:logs:%AWS_REGION%:%AWS_ACCOUNT_ID%:log-group:%LOG_GROUP_NAME%:*" - outputs: - - type: "elasticsearch" - args: - # either elasticsearch_url or cloud_id, elasticsearch_url takes precedence if both are included - elasticsearch_url: "http(s)://domain.tld:port" - cloud_id: "cloud_id:bG9jYWxob3N0OjkyMDAkMA==" - # either api_key or username/password, username/password takes precedence if both are included - api_key: "YXBpX2tleV9pZDphcGlfa2V5X3NlY3JldAo=" - username: "username" - password: "password" - es_datastream_name: "logs-generic-default" - es_dead_letter_index: "esf-dead-letter-index" # optional - batch_max_actions: 500 # optional: default value is 500 - batch_max_bytes: 10485760 # optional: default value is 10485760 - - type: "logstash" - args: - logstash_url: "http(s)://host:port" - username: "username" #optional - password: "password" #optional - max_batch_size: 500 #optional - compression_level: 1 #optional - ssl_assert_fingerprint: "22:F7:FB:84:1D:43:3E:E7:BB:F9:72:F3:D8:97:AD:7C:86:E3:08:42" #optional - - type: "cloudwatch-logs" - id: "arn:aws:logs:%AWS_REGION%:%AWS_ACCOUNT_ID%:log-group:%LOG_GROUP_NAME%:log-stream:%LOG_STREAM_NAME%" - outputs: - - type: "elasticsearch" - args: - # either elasticsearch_url or cloud_id, elasticsearch_url takes precedence if both are included - elasticsearch_url: "http(s)://domain.tld:port" - cloud_id: "cloud_id:bG9jYWxob3N0OjkyMDAkMA==" - # either api_key or username/password, username/password takes precedence if both are included - api_key: "YXBpX2tleV9pZDphcGlfa2V5X3NlY3JldAo=" - username: "username" - password: "password" - es_datastream_name: "logs-generic-default" - es_dead_letter_index: "esf-dead-letter-index" # optional - batch_max_actions: 500 # optional: default value is 500 - batch_max_bytes: 10485760 # optional: default value is 10485760 - - type: "logstash" - args: - logstash_url: "http(s)://host:port" - username: "username" #optional - password: "password" #optional - max_batch_size: 500 #optional - compression_level: 1 #optional - ssl_assert_fingerprint: "22:F7:FB:84:1D:43:3E:E7:BB:F9:72:F3:D8:97:AD:7C:86:E3:08:42" #optional ----- - -WARNING: All versions up to 1.14.0 (included) only allow one output per type. So if the `output.type` chosen by a user is `elasticsearch`, then the user can only configure one output for it. - -[discrete] -[[s3-config-file-fields]] -=== Fields - -//convert to description list? - -`inputs.[]`: - -A list of inputs (i.e. triggers) for the Elastic Serverless Forwarder Lambda function. - -`inputs.[].type`: - -The type of trigger input (`cloudwatch-logs`, `kinesis-data-stream`, `sqs` and `s3-sqs` are currently supported). - -`inputs.[].id`: - -The ARN of the trigger input according to the type. Multiple input entries can have different unique ids with the same type. -Inputs of type `cloudwatch-logs` accept both CloudWatch Logs Log Group and CloudWatch Logs Log Stream ARNs. - -`inputs.[].outputs`: - -A list of outputs (i.e. forwarding targets) for the Elastic Serverless Forwarder Lambda function. You can have multiple outputs for an input, but only one output can be defined per type. - -`inputs.[].outputs.[].type`: - -The type of the forwarding target output. Currently only the following outputs are supported: - - * `elasticsearch` - * preview:[] `logstash` - -Each type can only be used for a maximum of one output up to and including 1.14.0 version. If {ls} is chosen as an output, Elastic Serverless Forwarder expects the {logstash-ref}/plugins-inputs-elastic_serverless_forwarder.html[`elastic_serverless_forwarder`] Logstash input to be installed, enabled, and properly configured. For more information about installing Logstash plugins, refer to the {logstash-ref}/working-with-plugins.html#installing-plugins[Logstash documentation]. - -`inputs.[].outputs.[].args`: -Custom init arguments for the specified forwarding target output. - -For `elasticsearch` the following arguments are supported: - - * `args.elasticsearch_url`: URL of elasticsearch endpoint in the format `http(s)://domain.tld:port`. Mandatory when `args.cloud_id` is not provided. Will take precedence over `args.cloud_id` if both are defined. - * `args.cloud_id`: Cloud ID of elasticsearch endpoint. Mandatory when `args.elasticsearch_url` is not provided. Will be ignored if `args.elasticsearch_url` is defined. - * `args.username`: Username of the elasticsearch instance to connect to. Mandatory when `args.api_key` is not provided. Will take precedence over `args.api_key` if both are defined. - * `args.password` Password of the elasticsearch instance to connect to. Mandatory when `args.api_key` is not provided. Will take precedence over `args.api_key` if both are defined. - * `args.api_key`: API key of elasticsearch endpoint in the format `base64encode(api_key_id:api_key_secret)`. Mandatory when `args.username` and `args.password` are not provided. Will be ignored if `args.username`/`args.password` are defined. - * `args.es_datastream_name`: Name of data stream or index where logs should be forwarded to. Lambda supports automatic routing of various {aws} service logs to the corresponding data streams for further processing and storage in the {es} cluster. It supports automatic routing of `aws.cloudtrail`, `aws.cloudwatch_logs`, `aws.elb_logs`, `aws.firewall_logs`, `aws.vpcflow`, and `aws.waf` logs. For other log types, if using data streams, you can optionally set its value in the configuration file according to the naming convention for data streams and available integrations. If the `es_datastream_name` is not specified and it cannot be matched with any of the above {aws} services, then the value will be set to `logs-generic-default`. In versions **v0.29.1** and below, this configuration parameter was named `es_index_or_datastream_name`. Rename the configuration parameter to `es_datastream_name` in your `config.yaml` file on the S3 bucket to continue using it in the future version. The older name `es_index_or_datastream_name` is deprecated as of version **v0.30.0**. The related backward compatibility code is removed from version **v1.0.0**. - * `args.es_dead_letter_index`: Name of data stream or index where logs should be redirected to, in case indexing to `args.es_datastream_name` returned an error. The elasticseach output will NOT forward retryable errors (connection failures, HTTP status code 429) to the dead letter index. - * `args.batch_max_actions`: (Optional) Maximum number of actions to send in a single bulk request. Default value: 500. - * `args.batch_max_bytes`: (Optional) Maximum size in bytes to send in a single bulk request. Default value: 10485760 (10MB). - * `args.ssl_assert_fingerprint`: (Optional) SSL fingerprint for self-signed SSL certificate on HTTPS transport. The default value is an empty string, meaning the HTTP client requires a valid certificate. - -. Here is a sample error indexed in the dead letter index: -+ -[source, json] ----- -{ - "@timestamp": "2024-10-07T05:57:59.448925Z", - "message": "{\"hey\":{\"message\":\"hey there\"},\"_id\":\"e6542822-4583-438d-9b4d-1a3023b5eeb9\",\"_op_type\":\"create\",\"_index\":\"logs-succeed.pr793-default\"}", - "error": { - "message": "[1:30] failed to parse field [hey] of type [keyword] in document with id 'e6542822-4583-438d-9b4d-1a3023b5eeb9'. Preview of field's value: '{message=hey there}'", - "type": "document_parsing_exception" - }, - "http": { - "response": { - "status_code": 400 - } - } -} ----- - -For `logstash` the following arguments are supported: - - * `args.logstash_url`: URL of {ls} endpoint in the format `http(s)://host:port` - * `args.username`: (Optional) Username of the {ls} instance to connect to. Mandatory if HTTP Basic authentication is enabled in {ls}. - * `args.password`: (Optional) Password of the {ls} instance to connect to. Mandatory if HTTP Basic authentication is enabled in {ls}. - * `args.max_batch_size`: (Optional) Maximum number of events to send in a single HTTP(s) request. Default value: 500 - * `args.compression_level`: (Optional) The GZIP compression level for HTTP(s) requests towards {ls}. It can be any integer value between 1 (minimum compression, best performance, highest amount of bytes sent) and 9 (maximum compression, worst performance, lowest amount of bytes sent). Default value: 1 - * `args.ssl_assert_fingerprint`: (Optional) SSL fingerprint for self-signed SSL certificate on HTTPS transport. The default value is an empty string, meaning the HTTP client requires a valid certificate. - -[discrete] -[[aws-serverless-forwarder-define-deploy-parameters]] -== Define deployment parameters -Whichever SAR deployment method you choose, you must define the following parameters correctly for your setup. This section explains the types of parameters and provides guidance on how to set them to match your deployment(s). - -[discrete] -=== General configuration -These parameters define the general configuration and behaviour for the forwarder. - -- `ElasticServerlessForwarderS3ConfigFile`: Set this value to the location of your `config.yaml` in S3 URL format: `s3://bucket-name/config-file-name`. This will populate the `S3_CONFIG_FILE` environment variable for the forwarder. -- `ElasticServerlessForwarderSSMSecrets`: Add a comma delimited list of {aws} SSM Secrets ARNs used in the `config.yml` (if any). -- `ElasticServerlessForwarderKMSKeys`: Add a comma delimited list of {aws} KMS Keys ARNs to be used for decrypting {aws} SSM Secrets, Kinesis Data Streams, SQS queue, or S3 buckets (if any). - -[NOTE] -==== -Make sure you include all the KMS keys used to encrypt the data. For example, S3 buckets are often encrypted, so the Lambda function needs access to that key to get the object. -==== - -[discrete] -=== Inputs -These parameters define your specific <> or 'event triggers'. - -- `ElasticServerlessForwarderSQSEvents`: Add a comma delimited list of Direct SQS queue ARNs to set as event triggers for the forwarder (if any). -- `ElasticServerlessForwarderSQSEvents2`: Add a comma delimited list of Direct SQS queue ARNs to set as event triggers for the forwarder (if limit is reach on ElasticServerlessForwarderSQSEvents). -- `ElasticServerlessForwarderS3SQSEvents`: Add a comma delimited list of S3 SQS Event Notifications ARNs to set as event triggers for the forwarder (if any). -- `ElasticServerlessForwarderS3SQSEvents2`: Add a comma delimited list of S3 SQS Event Notifications ARNs to set as event triggers for the forwarder (if limit is reach on ElasticServerlessForwarderS3SQSEvents). -- `ElasticServerlessForwarderKinesisEvents`: Add a comma delimited list of Kinesis Data Stream ARNs to set as event triggers for the forwarder (if any). -- `ElasticServerlessForwarderKinesisEvents2`: Add a comma delimited list of Kinesis Data Stream ARNs to set as event triggers for the forwarder (if limit is reach on ElasticServerlessForwarderKinesisEvents). -- `ElasticServerlessForwarderCloudWatchLogsEvents`: Add a comma delimited list of Cloudwatch Logs log group ARNs to set subscription filters on the forwarder (if any). -- `ElasticServerlessForwarderCloudWatchLogsEvents2`: Add a comma delimited list of Cloudwatch Logs log group ARNs to set subscription filters on the forwarder (if limit is reach on ElasticServerlessForwarderCloudWatchLogsEvents). - -[NOTE] -==== -Make sure you reference the ARNs specified in your `config.yaml`, and leave any settings for unused inputs blank. -==== - -[discrete] -=== S3 Bucket permissions -These parameters define the permissions required in order to access the associated S3 Buckets. - -- `ElasticServerlessForwarderS3Buckets`: Add a comma delimited list of S3 bucket ARNs that are sources for the S3 SQS Event Notifications (if any). - -[discrete] -=== Network - -To attach the Elastic Serverless Forwarder to a specific {aws} VPC, specify the security group IDs and subnet IDs that belong to the {aws} VPC. This requirement is related to the https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-lambda-function-vpcconfig.html[CloudFormation VPCConfig property]. - -These are the parameters: - -- `ElasticServerlessForwarderSecurityGroups`: Add a comma delimited list of security group IDs to attach to the forwarder. -- `ElasticServerlessForwarderSubnets`: Add a comma delimited list of subnet IDs for the forwarder. - -Both parameters are required in order to attach the Elastic Serverless Forwarder to a specific {aws} VPC. -Leave both parameters blank if you don't want the forwarder to belong to any specific {aws} VPC. - -If the Elastic Serverless Forwarder is attached to a VPC, you need to https://docs.aws.amazon.com/vpc/latest/privatelink/create-interface-endpoint.html[create VPC endpoints] for S3 and SQS, and for *every* service you define as an input for the forwarder. S3 and SQS VPC endpoints are always required for reading the `config.yaml` uploaded to S3 and managing the continuing queue and the replay queue, regardless of the <> used. If you use <>, you need to create a VPC endpoint for EC2, too. - -NOTE: Refer to the {cloud}/ec-traffic-filtering-vpc.html[AWS PrivateLink traffic filters] documentation to find your VPC endpoint ID and the hostname to use in the `config.yml` in order to access your Elasticsearch cluster over PrivateLink. - -[discrete] -[[aws-serverless-forwarder-deploy-terraform]] -== Deploy Elastic Serverless Forwarder from Terraform - -The terraform files to deploy ESF can be found in https://github.com/elastic/terraform-elastic-esf[`esf-terraform` repository]. There are two requirements to deploy these files: https://curl.se/download.html[curl] and https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli[terraform]. Refer to the https://github.com/elastic/terraform-elastic-esf/blob/main/README.md[README file] to learn how to use it. - - -[discrete] -[[aws-serverless-forwarder-deploy-sar]] -== Deploy Elastic Serverless Forwarder from SAR - -There are several deployment methods available via the {aws} Serverless Application Repository (SAR): - -* <> -* <> -* <> - -NOTE: To deploy the forwarder directly without using SAR, refer to <> - -[discrete] -[[aws-serverless-forwarder-deploy-console]] -=== Deploy using {aws} Console - -NOTE: Only one deployment per region is allowed when using the {aws} console directly. - -. Log in to {aws} console and open **Lambda**. -. Click **Applications** and then **Create application**. -. Click **Serverless application** and search for **elastic-serverless-forwarder**. -. Select **elastic-serverless-forwarder** from the search results (ignoring any application beginning *helper-*). -+ -[role="screenshot"] -image::images/aws-serverless-forwarder-create-function.png[Create Elastic Serverless Forwarder Lambda function within SAR] -+ -. Complete the **Application settings** according to <>. -You must specify the parameters even if they already exist in the `config.yaml` file. Depends on the input type, at least -one if the parameters `ElasticServerlessForwarderSQSEvents`, `ElasticServerlessForwarderS3SQSEvents`, -`ElasticServerlessForwarderKinesisEvents`, `ElasticServerlessForwarderCloudWatchLogsEvents` should de defined. -. After your settings have been added, click **Deploy**. -. On the Applications page for **serverlessrepo-elastic-serverless-forwarder**, click **Deployments**. -. Refresh the **Deployment history** until you see the `Create complete` status update. It should take around 5 minutes to deploy — if the deployment fails for any reason, the create events will be rolled back and you will be able to see an explanation for which event failed. -. (Optional) To enable Elastic APM instrumentation for your new deployment: - * Go to **Lambda > Functions** within {aws} console, and find and select the function with **serverlessrepo-**. - * Go to **Configuration** tab and select **Environment Variables** - * Add the following environment variables: - - | Key | Value | - |---------------------------|--------| - |`ELASTIC_APM_ACTIVE` | `true` | - |`ELASTIC_APM_SECRET_TOKEN` | token | - |`ELASTIC_APM_SERVER_URL` | url | - -NOTE: If you have already successfully deployed the forwarder but want to update the application (for example, if a new version of the Lambda function is released), you should go through this deploy step again and use the same **Application name**. This will ensure the function is updated rather than duplicated or created anew. - -[discrete] -[[aws-serverless-forwarder-deploy-cloudformation]] -=== Deploy using Cloudformation - -. Use the following code to get the semantic version of the latest application: -+ -[source, bash] ----- -aws serverlessrepo list-application-versions --application-id arn:aws:serverlessrepo:eu-central-1:267093732750:applications/elastic-serverless-forwarder ----- -+ - -. Save the following YAML content as `sar-application.yaml` and fill in the correct parameters according to <>: -+ -[source, yaml] ----- - Transform: AWS::Serverless-2016-10-31 - Resources: - SarCloudformationDeployment: - Type: AWS::Serverless::Application - Properties: - Location: - ApplicationId: 'arn:aws:serverlessrepo:eu-central-1:267093732750:applications/elastic-serverless-forwarder' - SemanticVersion: '%SEMANTICVERSION%' ## SET TO CORRECT SEMANTIC VERSION (MUST BE GREATER THAN 1.6.0) - Parameters: - ElasticServerlessForwarderS3ConfigFile: "" - ElasticServerlessForwarderSSMSecrets: "" - ElasticServerlessForwarderKMSKeys: "" - ElasticServerlessForwarderSQSEvents: "" - ElasticServerlessForwarderSQSEvents2: "" ## IF SEMANTIC VERSION GREATER THAN 1.12.0 - ElasticServerlessForwarderS3SQSEvents: "" - ElasticServerlessForwarderS3SQSEvents2: "" ## IF SEMANTIC VERSION GREATER THAN 1.12.0 - ElasticServerlessForwarderKinesisEvents: "" - ElasticServerlessForwarderKinesisEvents2: "" ## IF SEMANTIC VERSION GREATER THAN 1.12.0 - ElasticServerlessForwarderCloudWatchLogsEvents: "" - ElasticServerlessForwarderCloudWatchLogsEvents2: "" ## IF SEMANTIC VERSION GREATER THAN 1.12.0 - ElasticServerlessForwarderS3Buckets: "" - ElasticServerlessForwarderSecurityGroups: "" - ElasticServerlessForwarderSubnets: "" ----- -+ - -. Deploy the Lambda function from SAR by running the following command: -+ -[source, shell] ----- - aws cloudformation deploy --template-file sar-application.yaml --stack-name esf-cloudformation-deployment --capabilities CAPABILITY_IAM CAPABILITY_AUTO_EXPAND ----- - - -NOTE: Starting from **v1.4.0**, if you want to update the Events settings for the forwarder, you do not need to manually delete existing settings before applying new settings. - -[discrete] -[[aws-serverless-forwarder-deploy-sar-terraform]] -=== Deploy from SAR using Terraform - -. Save the following yaml content as `sar-application.tf` and fill in the correct parameters according to <>: -+ -[source, yaml] ----- - provider "aws" { - region = "" ## FILL WITH THE AWS REGION WHERE YOU WANT TO DEPLOY THE ELASTIC SERVERLESS FORWARDER - } - data "aws_serverlessapplicationrepository_application" "esf_sar" { - application_id = "arn:aws:serverlessrepo:eu-central-1:267093732750:applications/elastic-serverless-forwarder" - } - resource "aws_serverlessapplicationrepository_cloudformation_stack" "esf_cf_stak" { - name = "terraform-elastic-serverless-forwarder" - application_id = data.aws_serverlessapplicationrepository_application.esf_sar.application_id - semantic_version = data.aws_serverlessapplicationrepository_application.esf_sar.semantic_version - capabilities = data.aws_serverlessapplicationrepository_application.esf_sar.required_capabilities - parameters = { - ElasticServerlessForwarderS3ConfigFile = "" - ElasticServerlessForwarderSSMSecrets = "" - ElasticServerlessForwarderKMSKeys = "" - ElasticServerlessForwarderSQSEvents = "" - ElasticServerlessForwarderS3SQSEvents = "" - ElasticServerlessForwarderKinesisEvents = "" - ElasticServerlessForwarderCloudWatchLogsEvents = "" - ElasticServerlessForwarderS3Buckets = "" - ElasticServerlessForwarderSecurityGroups = "" - ElasticServerlessForwarderSubnets = "" - } - } ----- -+ - -. Deploy the function from SAR by running the following commands: -+ -[source, shell] ----- - terraform init - terraform apply ----- -+ - - -[NOTE] -==== -From **v1.4.0** and above, if you want to update the Events settings for the deployment, it is no longer required to manually delete existing settings before applying the new settings. - -Due to a https://github.com/hashicorp/terraform-provider-aws/issues/24771[Terraform bug] related to `aws_serverlessapplicationrepository_application`, if you want to delete existing Event parameters you have to set the related `aws_serverlessapplicationrepository_cloudformation_stack.parameters` to a blank space value (`" "`) instead of an empty string (`""`). -==== - -[discrete] -[[aws-serverless-forwarder-direct-deploy]] -== Deploy Elastic Serverless Forwarder directly - -For more customisation options during deployment, from version `1.6.0` and above you can deploy the Elastic Serverless Forwarder directly to your {aws} Account without using SAR. This enables you to customize the event source settings for the inputs (i.e. triggers) one-by-one. - -To deploy the forwarder directly, you have to: - -* <> -* <> -* <> -* <> - -[discrete] -[[sample-direct-publish-config-file]] -=== Create `publish-config.yaml` for the publishing script - -To deploy the forwarder directly, you need to define a `publish-config.yaml` file and pass this as an argument in the <>. - -Save the following YAML content as `publish-config.yaml` and edit as required before running the publishing script. You should remove any inputs or arguments you are not using. - -[source, yaml] ----- - -kinesis-data-stream: - - arn: "arn:aws:kinesis:%REGION%:%ACCOUNT%:stream/%STREAMNAME%" - batch_size: 10 - batching_window_in_second: 0 - starting_position: TRIM_HORIZON - starting_position_timestamp: 0 - parallelization_factor: 1 -sqs: - - arn: "arn:aws:sqs:%REGION%:%ACCOUNT%:%QUEUENAME%" - batch_size: 10 - batching_window_in_second: 0 -s3-sqs: - - arn: "arn:aws:sqs:%REGION%:%ACCOUNT%:%QUEUENAME%" - batch_size: 10 - batching_window_in_second: 0 -cloudwatch-logs: - - arn: "arn:aws:logs:%AWS_REGION%:%AWS_ACCOUNT_ID%:log-group:%LOG_GROUP_NAME%:*" - - arn: "arn:aws:logs:%AWS_REGION%:%AWS_ACCOUNT_ID%:log-group:%LOG_GROUP_NAME%:log-stream:%LOG_STREAM_NAME%" -ssm-secrets: - - "arn:aws:secretsmanager:%AWS_REGION%:%AWS_ACCOUNT_ID%:secret:%SECRET_NAME%" -kms-keys: - - "arn:aws:kms:%AWS_REGION%:%AWS_ACCOUNT_ID%:key/%KMS_KEY_UUID%" -s3-buckets: - - "arn:aws:s3:::%BUCKET_NAME%" -subnets: - - "%SUBNET_ID%" -security-groups: - - "%SECURITY_ID%" -s3-config-file: "s3://%S3_CONFIG_BUCKET_NAME%/%S3_CONFIG_OBJECT_KEY%" -continuing-queue: - batch_size: 10 - batching_window_in_second: 0 - ----- - -[discrete] -[[direct-publish-config-file-fields]] -=== Fields - -|=== - -| `kinesis-data-stream.[]` | List of <> (i.e. triggers) for the forwarder, matching those defined in your <>. - -| `kinesis-data-stream.[].arn` | ARN of the {aws} Kinesis Data Stream. - -| `kinesis-data-stream.[].batch_size` | Set this value above the default (`10`) if you experience ingestion delays in your output *and* `GetRecords.IteratorAgeMilliseconds` and `IncomingRecords` Kinesis CloudWatch metrics for the <> keep increasing *and* the average execution time of the forwarder is below 14 minutes. This will increase the number of records the forwarder will process in a single execution for the <>. - -| `kinesis-data-stream.[].batching_window_in_second` | Set this value above the default (`0`) if you experience ingestion delays in your output *and* `GetRecords.IteratorAgeMilliseconds` and `IncomingRecords` Kinesis CloudWatch metrics for the <> keep increasing *and* the average execution time of the forwarder is below 14 minutes. This will increase the number of records the forwarder will process in a single execution for the <>. - -| `kinesis-data-stream.[].starting_position` | Change this value from the default (`TRIM_HORIZON`) if you want to change the starting position of the records processed by the forwarder for the <>. - -| `kinesis-data-stream.[].starting_position_timestamp` | Set this value to the time from which to start reading (in Unix time seconds) if you set `ElasticServerlessForwarderKinesisStartingPosition` to "AT_TIMESTAMP". - -| `kinesis-data-stream.[].parallelization_factor` | Defines the number of forwarder functions that can run concurrently per shard (default is `1`). Increase this value if you experience ingestion delays in your output *and* `GetRecords.IteratorAgeMilliseconds` and `IncomingRecords` Kinesis CloudWatch metrics for the <> keep increasing *and* the average execution time of the forwarder is below 14 minutes. This will increase the number of records processed concurrently for <>. For more info, refer to https://docs.aws.amazon.com/lambda/latest/dg/with-kinesis.html[AWS Kinesis docs]. - -| `sqs.[]` | List of <> (i.e. triggers) for the forwarder, matching those defined in your <>. - -| `sqs.[].arn` | ARN of the {aws} SQS queue trigger input. - -| `sqs.[].batch_size` | Set this value above the default (`10`) if you experience ingestion delays in your output *and* `ApproximateNumberOfMessagesVisible` and `ApproximateAgeOfOldestMessage` SQS CloudWatch metrics for the <> keep increasing *and* the average execution time of the forwarder is below 14 minutes. This will increase the number of messages the forwarder will process in a single execution for the <>. - -| `sqs.[].batching_window_in_second` | Set this value above the default (`0`) if you experience ingestion delays in your output *and* `ApproximateNumberOfMessagesVisible` and `ApproximateAgeOfOldestMessage` SQS CloudWatch metrics for the <> keep increasing *and* the average execution time of the forwarder is below 14 minutes. This will increase the number of messages the forwarder will process in a single execution for the <>. - -| `s3-sqs.[]` | List of <> (i.e. triggers) for the forwarder, matching those defined in your <>. - -| `s3-sqs.[].arn` | ARN of the {aws} SQS queue receiving S3 Notifications as trigger input. - -| `s3-sqs.[].batch_size` | Set this value above the default (`10`) if you experience ingestion delays in your output *and* `ApproximateNumberOfMessagesVisible` and `ApproximateAgeOfOldestMessage` SQS CloudWatch metrics for the <> keep increasing *and* the average execution time of the forwarder is below 14 minutes. This will increase the number of messages the forwarder will process in a single execution for the <>. - -| `s3-sqs.[].batching_window_in_second` | Set this value above the default (`0`) if you experience ingestion delays in your output *and* `ApproximateNumberOfMessagesVisible` and `ApproximateAgeOfOldestMessage` SQS CloudWatch metrics for the <> keep increasing *and* the average execution time of the forwarder is below 14 minutes. This will increase the number of messages the forwarder will process in a single execution for the <>. - -| `cloudwatch-logs.[]` | List of <> (i.e. triggers) for the forwarder, matching those defined in your <>. - -| `cloudwatch-logs.[].arn` | ARN of the {aws} CloudWatch Logs trigger input (accepts both CloudWatch Logs Log Group and CloudWatch Logs Log Stream ARNs). - -| `ssm-secrets.[]` | List of {aws} SSM Secrets ARNs used in your `config.yml` (if any). - -| `kms-keys.[]` | List of {aws} KMS Keys ARNs to be used for decrypting {aws} SSM Secrets, Kinesis Data Streams or SQS queues (if any). - -| `s3-buckets.[]` | List of S3 bucket ARNs that are sources for the S3 SQS Event Notifications (if any). - -| `subnets.[]` | A list of subnets IDs for the forwarder. Along with `security-groups.[]`, these settings will define the {aws} VPC the forwarder will belong to. Leave blank if you don't want the forwarder to belong to any specific {aws} VPC. - -| `security-groups.[]` | List of security group IDs to attach to the forwarder. Along with `subnets.[]`, these settings will define the {aws} VPC the forwarder will belong to. Leave blank if you don't want to have the forwarder belong to any specific {aws} VPC. - -| `s3-config-file` | Set this value to the location of your forwarder configuration file in S3 URL format: `s3://bucket-name/config-file-name`. This will populate the `S3_CONFIG_FILE` environment variable for the forwarder. - -| `continuing-queue.batch_size` | Set this value above the default (`10`) if you experience ingestion delays in your output *and* `ApproximateNumberOfMessagesVisible` and `ApproximateAgeOfOldestMessage` SQS CloudWatch metrics for the continuing queue keep increasing *and* the average execution time of the forwarder is below 14 minutes. This will increase the number of messages the forwarder will process in a single execution for the continuing queue. - -| `continuing-queue.batching_window_in_second` | Set this value above the default (`0`) if you experience ingestion delays in your output *and* `ApproximateNumberOfMessagesVisible` and `ApproximateAgeOfOldestMessage` SQS CloudWatch metrics for the continuing queue keep increasing *and* the average execution time of the forwarder is below 14 minutes. This will increase the number of messages the forwarder will process in a single execution for the continuing queue. - -|=== - -[discrete] -[[aws-serverless-forwarder-run-publish-script]] -=== Run the publishing script - -A bash script for publishing the Elastic Serverless Forwarder directly to your {aws} account is available from the https://github.com/elastic/elastic-serverless-forwarder[Elastic Serverless Forwarder repository]. - -Download the https://raw.githubusercontent.com/elastic/elastic-serverless-forwarder/lambda-v1.8.0/publish_lambda.sh[`publish_lambda.sh` script] and follow the instructions below. - -[discrete] -==== Script arguments - -[source, bash] ----- - - $ ./publish_lambda.sh - AWS CLI (https://aws.amazon.com/cli/), SAM (https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/install-sam-cli.html) and Python3.9 with pip3 required - Please, before launching the tool execute "$ pip3 install ruamel.yaml" -Usage: ./publish_lambda.sh config-path lambda-name forwarder-tag bucket-name region [custom-role-prefix] - Arguments: - config-path: full path to the publish configuration - lambda-name: name of the lambda to be published in the account - forwarder-tag: tag of the elastic serverless forwarder to publish - bucket-name: bucket name where to store the zip artifact for the lambda - (it will be created if it doesn't exists, otherwise - you need already to have proper access to it) - region: region where to publish in - custom-role-prefix: role/policy prefix to add in case customization is needed (optional) - (please note that the prefix will be added to both role/policy naming) ----- - -[discrete] -==== Prerequisites - -- Python3.9 with pip3 is required to run the script -- https://aws.amazon.com/cli/[{aws} CLI], https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/install-sam-cli.html[SAM CLI] and the https://pypi.org/project/ruamel.yaml/[ruamel.yaml package] must also be installed - -[source, bash] ----- - -$ pip3 install awscli aws-sam-cli ruamel.yaml - ----- - -[discrete] -==== Running the script -Assuming `publish-config.yaml` in saved in the same directory you intend to run `publish_lambda.sh` from, here's an example: - -[source, bash] ----- - -$ ./publish_lambda.sh publish-config.yaml forwarder-lambda lambda-v1.6.0 s3-lambda-artifact-bucket-name eu-central-1 - ----- - -[discrete] -==== Updating to a new version via script -You can update the version of a published Elastic Serverless Forwarder without changing its configuration by running the publishing script again and passing a *new* https://github.com/elastic/elastic-serverless-forwarder/tags[`forwarder-tag`]: - -[source, bash] ----- - -$ ./publish_lambda.sh publish-config.yaml forwarder-lambda lambda-v1.7.0 s3-lambda-artifact-bucket-name eu-central-1 - ----- - -NOTE: The above examples show the forwarder being updated from `lambda-v1.6.0` to `lambda-v1.7.0`. - -[discrete] -==== Changing configuration via script -If you want to change the configuration of a published Elastic Serverless Forwarder without changing its version, you can update the `publish-config.yaml` and run the script again using the *same* `forwarder-tag`: - -[source, bash] ----- - -$ ./publish_lambda.sh publish-config.yaml forwarder-lambda lambda-v1.6.0 s3-lambda-artifact-bucket-name eu-central-1 - ----- - -NOTE: The above example shows an existing `lambda-v1.6.0` configuration being updated without changing version. - -[discrete] -==== Using the script for multiple deployments -If you want to use the publish script for deploying the forwarder with different configurations, create two different `publish-config.yaml` files with unique names and run the publishing script twice, with correct references to the `config-path` and `lambda-name`: - -[source, bash] ----- - -$ ./publish_lambda.sh publish-config-for-first-lambda.yaml first-lambda lambda-v1.6.0 s3-lambda-artifact-bucket-name eu-central-1 - -$ ./publish_lambda.sh publish-config-for-second-lambda.yaml second-lambda lambda-v1.6.0 ss3-lambda-artifact-bucket-name eu-central-1 - ----- - -NOTE: The above example publishes two versions of the forwarder, each with different configurations i.e. `publish-config-for-first-lambda.yaml` and `first-lambda` vs. `publish-config-for-second-lambda.yaml` and `second-lambda`. diff --git a/docs/en/aws-elastic-serverless-forwarder.asciidoc b/docs/en/aws-elastic-serverless-forwarder.asciidoc deleted file mode 100644 index ce103c1a..00000000 --- a/docs/en/aws-elastic-serverless-forwarder.asciidoc +++ /dev/null @@ -1,150 +0,0 @@ -:aws: AWS - -[[aws-elastic-serverless-forwarder]] -= Elastic Serverless Forwarder for AWS - -++++ -Elastic Serverless Forwarder for AWS -++++ -:keywords: serverless -:description: The Elastic Serverless Forwarder is an Amazon Web Services ({aws}) Lambda function that ships logs from your {aws} environment to Elastic. - -The Elastic Serverless Forwarder is an Amazon Web Services ({aws}) Lambda function that ships logs from your {aws} environment to Elastic. - -The Elastic Serverless Forwarder works with {stack} 7.17 and later. - -IMPORTANT: Using Elastic Serverless Forwarder may result in additional charges. To learn -how to minimize additional charges, refer to <>. - -[discrete] -[[aws-serverless-forwarder-overview]] -= Overview -The Elastic Serverless Forwarder can forward {aws} data to cloud-hosted, self-managed Elastic environments, or preview:[]{ls}. It supports the following inputs: - -- Amazon S3 (via SQS event notifications) -- Amazon Kinesis Data Streams -- Amazon CloudWatch Logs subscription filters -- Amazon SQS message payload - -[role="screenshot"] -image::images/aws-serverless-lambda-flow.png[AWS Lambda flow] - -Elastic Serverless Forwarder ensures <> of the forwarded message. - -When you successfully deploy the forwarder, an SQS _continuing queue_ is automatically created in Lambda to ensure no data is lost. By default, the forwarder runs for a maximum of 15 minutes, so it's possible that {aws} may exit the function in the middle of processing event data. The forwarder handles this scenario by keeping track of the last offset processed. When the queue triggers a new function invocation, the forwarder will start where the last function run stopped. - -The forwarder uses a _replay queue_ (also automatically created during deployment) to handle any ingestion-related exception or fail scenarios. Data in the replay queue is stored as individual events. Lambda keeps track of any failed events and writes them to the replay queue that can then be consumed by adding it as an additional SQS trigger via Lambda. - -You can use the <> file to configure the service for each input and output type, including information such as SQS queue ARN (Amazon Resource Number) and {es} or {ls} connection details. You can create multiple input sections within the configuration file to map different inputs to specific log types. - -There is no need to define a specific input in the <> for the continuing queue and the replay queue. - -The forwarder also supports writing directly to an index, alias, or custom data stream. This enables existing {es} users to re-use index templates, ingest pipelines, or dashboards that are already created and connected to other processes. - -[discrete] -[[aws-serverless-forwarder-inputs]] -= Inputs - -[discrete] -[[aws-serverless-forwarder-inputs-s3]] -== Amazon S3 (via SQS event notifications) - -The forwarder can ingest logs contained in an Amazon Simple Storage Service (S3) bucket through a Simple Queue Service (SQS) notification (`s3:ObjectCreated`) and send them to Elastic. The SQS queue serves as a trigger for the forwarder. When a new log file is written to an S3 bucket and meets the user-defined criteria (including prefix/suffix), an SQS notification is generated that triggers the Lambda function. - -You can set up separate SQS queues for each type of log (for example, `aws.vpcflow`, `aws.cloudtrail`, `aws.waf`). A single configuration file can have many input sections, pointing to different SQS queues that match specific log types. The `es_datastream_name` parameter in the config file is optional. The forwarder supports automatic routing of various {aws} service logs to the corresponding data streams for further processing and storage in the {es} cluster. It supports automatic routing of `aws.cloudtrail`, `aws.cloudwatch_logs`, `aws.elb_logs`, `aws.firewall_logs`, `aws.vpcflow`, and `aws.waf` logs. - -For other log types, you can optionally set the `es_datastream_name` value in the configuration file according to the naming convention of the {es} data stream and integration. If the `es_datastream_name` is not specified, and the log cannot be matched with any of the above {aws} services, then the dataset will be set to `generic` and the namespace set to `default`, pointing to the data stream name `logs-generic-default`. - -For more information on creating SQS event notifications for S3 buckets, read the https://docs.aws.amazon.com/AmazonS3/latest/userguide/ways-to-add-notification-config-to-bucket.html[{aws} documentation]. - -NOTE: You must set a visibility timeout of `910` seconds for any SQS queues you want to use as a trigger. This is 10 seconds greater than the Elastic Serverless Forwarder Lambda timeout. - -[discrete] -[[aws-serverless-forwarder-inputs-kinesis]] -== Amazon Kinesis Data Streams - -The forwarder can ingest logs contained in the payload of a Kinesis Data Stream record and send them to Elastic. The Kinesis Data Stream serves as a trigger for the forwarder. When a new record gets written to a Kinesis Data Stream, it triggers the Lambda function. - -You can set up separate Kinesis Data Streams for each type of log. The `es_datastream_name` parameter in the config file is mandatory. If this value is set to an {es} data stream, the type of log must be correctly defined with configuration parameters. A single configuration file can have many input sections, pointing to different data streams that match specific log types. - -[discrete] -[[aws-serverless-forwarder-inputs-cloudwatch]] -== Amazon CloudWatch Logs subscription filters - -The forwarder can ingest logs contained in the message payload of CloudWatch Logs events and send them to Elastic. The CloudWatch Logs service serves as a trigger for the forwarder. When a new event gets written to a CloudWatch Logs log stream, it triggers the Lambda function. - -You can set up separate CloudWatch Logs groups for each type of log. The `es_datastream_name` parameter in the config file is mandatory. If this value is set to an {es} data stream, the type of log must be correctly defined with configuration parameters. A single configuration file can have many input sections, pointing to different CloudWatch Logs groups that match specific log types. - -[discrete] -[[aws-serverless-forwarder-inputs-direct]] -== Amazon SQS message payload - -The forwarder can ingest logs contained within the payload of an Amazon SQS body record and send them to Elastic. The SQS queue serves as a trigger for the forwarder. When a new record gets written to an SQS queue, the Lambda function triggers. - -You can set up a separate SQS queue for each type of log. The config parameter for {es} output `es_datastream_name` is mandatory. If this value is set to an {es} data stream, the type of log must be correctly defined with configuration parameters. A single configuration file can have many input sections, pointing to different SQS queues that match specific log types. - -[discrete] -[[aws-serverless-forwarder-at-least-once-delivery]] -= At-least-once delivery -The Elastic Serverless Forwarder ensures at-least-once delivery of the forwarded messages by using the continuing queue and replay queue. - -[discrete] -[[aws-serverless-forwarder-at-least-once-delivery-continuing-queue]] -== Continuing queue - -The Elastic Serverless Forwarder can run for a maximum amount of time of 15 minutes. Different inputs can trigger the Elastic Serverless Forwarder, with different payload sizes for each execution trigger. The size of the payload impacts the number of events to be forwarded. Configuration settings may also impact the number of events, such as <>, <>, and <>. - -The continuing queue helps ensure at-least-once delivery when the maximum amount of time (15 minutes) is not sufficient to forward all the events resulting from a single execution of the Elastic Serverless Forwarder. - -For this scenario, a grace period of two minutes is reserved at the end of the 15 minute timeout to handle any remaining events that have not been processed. -At the beginning of this grace period, event forwarding is halted. The remaining time is dedicated to sending a copy of the original messages that contain the remaining events to the continuing queue. -This mechanism removes the need to handle partial processing of the trigger at the input level, which is not always possible (for example in the case of <>) or desirable because it forces users to conform to a specific configuration of the {aws} resources used as inputs. - -Each message in the continuing queue contains metadata related to the last offset processed and a reference to the original input. - -NOTE: You can remove a specific input as a trigger of the Elastic Serverless Forwarder. However, before removing its definition from the <>, ensure that all the events generated while the input was still a trigger are fully processed, including the ones in the messages copied to the continuing queue. The handling of the messages in the continuing queue requires a lookup of the original input in the `config.yml`. - -In the unlikely scenario that the Elastic Serverless Forwarder exceeds its maximum allocated execution time and is forcefully terminated, the continuing queue will not be properly populated with a copy of the messages left to be processed. In this scenario, all or a portion of the messages might be lost depending on the specific {aws} resource used as input and its configuration. - -An {aws} SQS https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-dead-letter-queues.html[Dead Letter Queue] is created for the continuing queue. - -When the Elastic Serverless Forwarder is triggered by the continuing queue, in the unlikely scenario that it exceeds its maximum allocated execution time and is forcefully terminated, the messages in the payload that triggered the Elastic Serverless Forwarder execution are not deleted from the continuing queue and another Elastic Serverless Forwarder execution is triggered. The continuing queue is configured for a number of 3 maximum receives before a message is sent to the DLQ. - -[discrete] -[[aws-serverless-forwarder-at-least-once-delivery-replay-queue]] -== Replay queue - -The Elastic Serverless Forwarder forwards events to the outputs defined for a specific input. Events to be forwarded are grouped in batches that can be configured according to the specific output. -Failures can happen when forwarding events to an output. Depending on the output type, the granularity of the failure can either be for the whole batch of events, or for single events in the batch. -There are multiple reasons for a failure to happen, including, but not limited to, network connectivity or the output service being unavailable or under stress. - -The replay queue helps ensure at-least-once delivery when a failure in forwarding an event happens. - -For this scenario, after a batch of events is forwarded, a copy of all the events in the batch that failed to be forwarded is sent to the replay queue. Each message sent to the replay queue contains exactly one event that failed to be forwarded. - -It is possible to enable the replay queue as a trigger of the Elastic Serverless Forwarder in order to forward the events in the queue again. - -NOTE: Before enabling or disabling the replay queue as a trigger of the Elastic Serverless Forwarder, consider the specific reason why the forwarding failures occurred. In most cases, you should resolve the underlying issue causing the failures before trying to forward the events in the queue again. Depending on the nature and impact of the issue, forwarding the events again without solving the problem may produce new failures and events going back to the replay queue. In some scenarios, like the output service being under stress, it is recommended that you disable the replay queue as a trigger of the Elastic Serverless Forwarder, since continuing to forward the events could worsen the issue. - -When the Elastic Serverless Forwarder is triggered by the replay queue and all events are successfully forwarded, the Elastic Serverless Forwarded execution succeeds, and the messages in the trigger payload are removed automatically from the replay queue. - -However, if any events fail again to be forwarded, all messages in the trigger payload that contain successful events are deleted, and a specific expected exception is raised. The Elastic Serverless Forwarder execution is marked as failed, and any failed messages are sent back to the replay queue. - -The messages in the replay queue contain metadata with references to the original input and the original output of the events. - -NOTE: You can remove a specific input as a trigger of the Elastic Serverless Forwarder. However, before removing its definition from <>, ensure that all the events that failed to be ingested while the input was still a trigger are fully processed. The handling of the messages in the replay queue requires a lookup of the original input and output in the `config.yml`. - -An {aws} SQS https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-dead-letter-queues.html[Dead Letter Queue (DLQ)] is created for the replay queue. - -The same message can go back to the replay queue up to three times. After reaching the configured number of 3 maximum receives, the message will be sent to the DLQ. -The same message can go back to the replay queue either because it contains an event that failed again to be forwarded, according to the planned design, or in the unlikely scenario that the Elastic Serverless Forwarder triggered by the queue exceeds its maximum allocated execution time and is forcefully terminated. In this scenario the messages will not be lost and will eventually be sent to the DQL. - -[discrete] -[[aws-serverless-forwarder-get-started]] -= Get started - -- <> - -- <> - -- <> diff --git a/docs/en/aws-serverless-troubleshooting.asciidoc b/docs/en/aws-serverless-troubleshooting.asciidoc deleted file mode 100644 index 87c199a7..00000000 --- a/docs/en/aws-serverless-troubleshooting.asciidoc +++ /dev/null @@ -1,74 +0,0 @@ -[[aws-serverless-troubleshooting]] -= Troubleshooting and error handling - -++++ -Troubleshooting -++++ - -[discrete] -== Troubleshooting deployment errors -You can view the status of deployment actions and get additional information on events, including why a particular event fails e.g. misconfiguration details. - -. On the Applications page for **serverlessrepo-elastic-serverless-forwarder**, click **Deployments**. -. You can view the **Deployment history** here and refresh the page for updates as the application deploys. It should take around 5 minutes to deploy — if the deployment fails for any reason, the create events will be rolled back, and you will be able to see an explanation for which event failed. - -NOTE: For example, if you don't increase the visibility timeout for an SQS queue as described in <>, you will see a `CREATE_FAILED`**Status** for the event, and the **Status reason** provides additional detail. - -[discrete] -[[preventing-unexpected-costs]] -== Preventing unexpected costs -It is important to monitor the Elastic Serverless Forwarder Lambda function for timeouts to prevent unexpected costs. You can use the https://docs.elastic.co/en/integrations/aws/lambda[AWS Lambda integration] for this. If the timeouts are constant, you should throttle the Lambda function to stop its execution before proceeding with any troubleshooting steps. In most cases, constant timeouts will cause the records and messages from the event triggers to go back to their sources and trigger the function again, which will cause further timeouts and force a loop that will incure unexpected high costs. For more information on throttling Lambda functions, refer to https://docs.aws.amazon.com/lambda/latest/operatorguide/throttling.html[AWS docs]. - -// is it clear how you would throttle the Lambda function? should we detail and number these steps? - -[discrete] -=== Increase debug information -To help with debugging, you can increase the amount of logging detail by adding an environment variable as follows: - -. Select the serverless forwarder **application** from **Lambda > Functions** -. Click **Configuration** and select **Environment Variables** and choose **Edit** -. Click **Add environment variable** and enter `LOG_LEVEL` as **Key** and `DEBUG` as **Value** and click **Save** - -// confirm where this is visible - only in CloudWatch or also within ES messages? - -[discrete] -[[aws-serverless-troubleshooting-event-id-format]] -== Using the Event ID format (version 1.6.0 and above) - -// this is not ideal to describe version changes in troubleshooting but we can edit and integrate better when we create a versioned version of these docs - -Version 1.6.0 introduces a new event ID format that prevents duplicate ID errors when a high volume of events is ingested to {es}. This new format combines a timestamp with data specific to the relevant AWS resource, extracted from the AWS Lambda event received by the forwarder. - -The timestamp is used as a prefix for the ID, because identifiers that gradually increase over time generally result in better indexing performance in {es}, based on sorting order rather than completely random identifiers. For more information, please refer to https://www.elastic.co/blog/efficient-duplicate-prevention-for-event-based-data-in-elasticsearch[this Elastic blog on event-based data]. - -// Leaving the blog link in for context though this is not ideal and would be better to link to existing docs instead - -NOTE: If old events that are already published to {es} using a version of Elastic Serverless Forwarder before v1.6.0 are ingested again, they will be treated as new events and published to {es} as duplicates. - -[discrete] -== Error handling - -There are two kind of errors that can occur during execution of the forwarder: - -. Errors _before_ the ingestion phase begins -. Errors _during_ the ingestion phase - -[discrete] -=== Errors before ingestion -For errors that occur before ingestion begins (1), the function will return a failure. These errors are mostly due to misconfiguration, incorrect permissions for AWS resources, etc. Most importantly, when an error occurs at this stage we don’t have any status for events that are ingested, so there’s no requirement to keep data, and the function can fail safely. In the case of SQS messages and Kinesis data stream records, both will go back into the queue and trigger the function again with the same payload. - -[discrete] -=== Errors during ingestion -For errors that occur during ingestion (2), the situation is different. We do have a status for *N* failed events out of *X* total events; if we fail the whole function then all *X* events will be processed again. While the *N* failed ones could potentially succeed, the remaining *X-N* will definitely fail, because the data streams are append-only (the function would attempt to recreate already ingested documents using the same document ID). - -So the forwarder won't return a failure for errors during ingestion; instead, the payload of the event that failed to be ingested will be sent to a replay SQS queue, which is automatically set up during the deployment. The replay SQS queue is not set as an Event Source Mapping for the function by default, which means you can investigate and consume (or not) the message as preferred. - -You can temporarily set the replay SQS queue as an Event Source Mapping for the forwarder, which means messages in the queue will be consumed by the function and ingestion retried for transient failures. If the failure persists, the affected log entry will be moved to a DLQ after three retries. - -Every other error that occurs during the execution of the forwarder is silently ignored, and reported to the APM server if instrumentation is enabled. - -[discrete] -=== Execution timeout -There is a grace period of 2 minutes before the timeout of the Lambda function where no more ingestion will occur. Instead, during this grace period the forwarder will collect and handle any unprocessed payloads in the batch of the input used as trigger. - -For CloudWatch Logs event, Kinesis data stream, S3 SQS Event Notifications and direct SQS message payload inputs, the unprocessed batch will be sent to the SQS continuing queue. diff --git a/docs/en/index.asciidoc b/docs/en/index.asciidoc deleted file mode 100644 index f71a911a..00000000 --- a/docs/en/index.asciidoc +++ /dev/null @@ -1,14 +0,0 @@ -:doctype: book - -= Elastic Serverless Forwarder Guide - -include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[] -include::{docs-root}/shared/attributes.asciidoc[] - -:y: image:images/green-check.svg[yes] -:n: image:images/red-x.svg[no] - -include::aws-elastic-serverless-forwarder.asciidoc[leveloffset=+1] -include::aws-deploy-elastic-serverless-forwarder.asciidoc[leveloffset=+1] -include::aws-elastic-serverless-forwarder-configuration.asciidoc[leveloffset=+1] -include::aws-serverless-troubleshooting.asciidoc[leveloffset=+1] diff --git a/docs/en/images/aws-serverless-forwarder-create-function.png b/docs/images/aws-serverless-forwarder-create-function.png similarity index 100% rename from docs/en/images/aws-serverless-forwarder-create-function.png rename to docs/images/aws-serverless-forwarder-create-function.png diff --git a/docs/en/images/aws-serverless-forwarder-install-assets.png b/docs/images/aws-serverless-forwarder-install-assets.png similarity index 100% rename from docs/en/images/aws-serverless-forwarder-install-assets.png rename to docs/images/aws-serverless-forwarder-install-assets.png diff --git a/docs/en/images/aws-serverless-lambda-flow.png b/docs/images/aws-serverless-lambda-flow.png similarity index 100% rename from docs/en/images/aws-serverless-lambda-flow.png rename to docs/images/aws-serverless-lambda-flow.png diff --git a/docs/en/images/false-after-multi.png b/docs/images/false-after-multi.png similarity index 100% rename from docs/en/images/false-after-multi.png rename to docs/images/false-after-multi.png diff --git a/docs/en/images/false-before-multi.png b/docs/images/false-before-multi.png similarity index 100% rename from docs/en/images/false-before-multi.png rename to docs/images/false-before-multi.png diff --git a/docs/en/images/multiline-regexp-test-repl-main.png b/docs/images/multiline-regexp-test-repl-main.png similarity index 100% rename from docs/en/images/multiline-regexp-test-repl-main.png rename to docs/images/multiline-regexp-test-repl-main.png diff --git a/docs/en/images/multiline-regexp-test-repl-run.png b/docs/images/multiline-regexp-test-repl-run.png similarity index 100% rename from docs/en/images/multiline-regexp-test-repl-run.png rename to docs/images/multiline-regexp-test-repl-run.png diff --git a/docs/en/images/true-after-multi.png b/docs/images/true-after-multi.png similarity index 100% rename from docs/en/images/true-after-multi.png rename to docs/images/true-after-multi.png diff --git a/docs/en/images/true-before-multi.png b/docs/images/true-before-multi.png similarity index 100% rename from docs/en/images/true-before-multi.png rename to docs/images/true-before-multi.png diff --git a/docs/reference/aws-deploy-elastic-serverless-forwarder.md b/docs/reference/aws-deploy-elastic-serverless-forwarder.md new file mode 100644 index 00000000..626d6cab --- /dev/null +++ b/docs/reference/aws-deploy-elastic-serverless-forwarder.md @@ -0,0 +1,652 @@ +--- +navigation_title: "Deploy serverless forwarder" +mapped_pages: + - https://www.elastic.co/guide/en/esf/current/aws-deploy-elastic-serverless-forwarder.html +--- + +# Deploy Elastic Serverless Forwarder [aws-deploy-elastic-serverless-forwarder] + +To deploy Elastic Serverless Forwarder, you have to: + +* [Install AWS integration assets in {{kib}}](#aws-serverless-forwarder-deploy-kibana) +* [Create and upload `config.yaml` to S3 bucket](#sample-s3-config-file) +* [Define deployment parameters](#aws-serverless-forwarder-define-deploy-parameters) +* [Deploy Elastic Serverless Forwarder from SAR](#aws-serverless-forwarder-deploy-sar) + + +## Prerequisites [aws-serverless-forwarder-deploy-prereq] + +This documentation assumes you have some familiarity with AWS services, and you have correctly created and configured the necessary AWS objects. + +::::{note} +This page describes the basic steps required to deploy Elastic Serverless Forwarder for AWS— for additional information on configuration topics such as permissions and automatic routing, and parsing and enriching data, see [Configuration options](/reference/aws-elastic-serverless-forwarder-configuration.md). +:::: + + + +## Deploying directly without SAR [aws-serverless-forwarder-deploy-direct-note] + +If the customization options available when deploying via Serverless Application Repository (SAR) are not sufficient, from version `1.6.0` and above you can [deploy the Elastic Serverless Forwarder directly](#aws-serverless-forwarder-direct-deploy) to your AWS Account without using SAR. This enables you to customize the event source settings for the inputs (i.e. triggers) one-by-one. + + +## Install AWS integration assets in {{kib}} [aws-serverless-forwarder-deploy-kibana] + +1. Go to **Integrations** in {{kib}} and search for AWS (or select the **AWS** category to filter the list). +2. Click the AWS integration, select **Settings** and click **Install AWS assets** and confirm to install all the AWS integration assets. + +:::{image} ../images/aws-serverless-forwarder-install-assets.png +:alt: Find and install AWS integration assets in {kib} +:class: screenshot +::: + +Adding integrations from {{kib}} provides appropriate pre-built dashboards, ingest node configurations, and other assets that help you get the most out of the data you ingest. The integrations use [data streams](docs-content://manage-data/data-store/data-streams.md) with specific [naming conventions](https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme) that provide you with more granular controls and flexibility on managing data ingestion. + +::::{note} +We recommend using integration assets to get started but the forwarder supports writing to any index, alias, or custom data stream. This enables existing Elasticsearch users to re-use index templates, ingest pipelines, or dashboards that are already created and connected to existing processes or systems. If you already have an existing index or data stream you intend to send data to, then you can skip this deployment step. +:::: + + + +## Create and upload `config.yaml` to S3 bucket [sample-s3-config-file] + +Elastic Serverless Forwarder requires a `config.yaml` file to be uploaded to an S3 bucket and referenced by the `S3_CONFIG_FILE` environment variable. + +Save the following YAML content as `config.yaml` and edit as required before uploading to an S3 bucket. You should remove any inputs or arguments you are not using, and ensure you have entered the correct URLs and credentials as per the inline comments. + +```yaml +inputs: + - type: "s3-sqs" + id: "arn:aws:sqs:%REGION%:%ACCOUNT%:%QUEUENAME%" + outputs: + - type: "elasticsearch" + args: + # either elasticsearch_url or cloud_id, elasticsearch_url takes precedence if both are included + elasticsearch_url: "http(s)://domain.tld:port" + cloud_id: "cloud_id:bG9jYWxob3N0OjkyMDAkMA==" + # either api_key or username/password, username/password takes precedence if both are included + api_key: "YXBpX2tleV9pZDphcGlfa2V5X3NlY3JldAo=" + username: "username" + password: "password" + es_datastream_name: "logs-generic-default" + es_dead_letter_index: "esf-dead-letter-index" # optional + batch_max_actions: 500 # optional: default value is 500 + batch_max_bytes: 10485760 # optional: default value is 10485760 + - type: "logstash" + args: + logstash_url: "http(s)://host:port" + username: "username" #optional + password: "password" #optional + max_batch_size: 500 #optional + compression_level: 1 #optional + ssl_assert_fingerprint: "22:F7:FB:84:1D:43:3E:E7:BB:F9:72:F3:D8:97:AD:7C:86:E3:08:42" #optional + - type: "sqs" + id: "arn:aws:sqs:%REGION%:%ACCOUNT%:%QUEUENAME%" + outputs: + - type: "elasticsearch" + args: + # either elasticsearch_url or cloud_id, elasticsearch_url takes precedence if both are included + elasticsearch_url: "http(s)://domain.tld:port" + cloud_id: "cloud_id:bG9jYWxob3N0OjkyMDAkMA==" + # either api_key or username/password, username/password takes precedence if both are included + api_key: "YXBpX2tleV9pZDphcGlfa2V5X3NlY3JldAo=" + username: "username" + password: "password" + es_datastream_name: "logs-generic-default" + es_dead_letter_index: "esf-dead-letter-index" # optional + batch_max_actions: 500 # optional: default value is 500 + batch_max_bytes: 10485760 # optional: default value is 10485760 + - type: "logstash" + args: + logstash_url: "http(s)://host:port" + username: "username" #optional + password: "password" #optional + max_batch_size: 500 #optional + compression_level: 1 #optional + ssl_assert_fingerprint: "22:F7:FB:84:1D:43:3E:E7:BB:F9:72:F3:D8:97:AD:7C:86:E3:08:42" #optional + - type: "kinesis-data-stream" + id: "arn:aws:kinesis:%REGION%:%ACCOUNT%:stream/%STREAMNAME%" + outputs: + - type: "elasticsearch" + args: + # either elasticsearch_url or cloud_id, elasticsearch_url takes precedence if both are included + elasticsearch_url: "http(s)://domain.tld:port" + cloud_id: "cloud_id:bG9jYWxob3N0OjkyMDAkMA==" + # either api_key or username/password, username/password takes precedence if both are included + api_key: "YXBpX2tleV9pZDphcGlfa2V5X3NlY3JldAo=" + username: "username" + password: "password" + es_datastream_name: "logs-generic-default" + es_dead_letter_index: "esf-dead-letter-index" # optional + batch_max_actions: 500 # optional: default value is 500 + batch_max_bytes: 10485760 # optional: default value is 10485760 + - type: "logstash" + args: + logstash_url: "http(s)://host:port" + username: "username" #optional + password: "password" #optional + max_batch_size: 500 #optional + compression_level: 1 #optional + ssl_assert_fingerprint: "22:F7:FB:84:1D:43:3E:E7:BB:F9:72:F3:D8:97:AD:7C:86:E3:08:42" #optional + - type: "cloudwatch-logs" + id: "arn:aws:logs:%AWS_REGION%:%AWS_ACCOUNT_ID%:log-group:%LOG_GROUP_NAME%:*" + outputs: + - type: "elasticsearch" + args: + # either elasticsearch_url or cloud_id, elasticsearch_url takes precedence if both are included + elasticsearch_url: "http(s)://domain.tld:port" + cloud_id: "cloud_id:bG9jYWxob3N0OjkyMDAkMA==" + # either api_key or username/password, username/password takes precedence if both are included + api_key: "YXBpX2tleV9pZDphcGlfa2V5X3NlY3JldAo=" + username: "username" + password: "password" + es_datastream_name: "logs-generic-default" + es_dead_letter_index: "esf-dead-letter-index" # optional + batch_max_actions: 500 # optional: default value is 500 + batch_max_bytes: 10485760 # optional: default value is 10485760 + - type: "logstash" + args: + logstash_url: "http(s)://host:port" + username: "username" #optional + password: "password" #optional + max_batch_size: 500 #optional + compression_level: 1 #optional + ssl_assert_fingerprint: "22:F7:FB:84:1D:43:3E:E7:BB:F9:72:F3:D8:97:AD:7C:86:E3:08:42" #optional + - type: "cloudwatch-logs" + id: "arn:aws:logs:%AWS_REGION%:%AWS_ACCOUNT_ID%:log-group:%LOG_GROUP_NAME%:log-stream:%LOG_STREAM_NAME%" + outputs: + - type: "elasticsearch" + args: + # either elasticsearch_url or cloud_id, elasticsearch_url takes precedence if both are included + elasticsearch_url: "http(s)://domain.tld:port" + cloud_id: "cloud_id:bG9jYWxob3N0OjkyMDAkMA==" + # either api_key or username/password, username/password takes precedence if both are included + api_key: "YXBpX2tleV9pZDphcGlfa2V5X3NlY3JldAo=" + username: "username" + password: "password" + es_datastream_name: "logs-generic-default" + es_dead_letter_index: "esf-dead-letter-index" # optional + batch_max_actions: 500 # optional: default value is 500 + batch_max_bytes: 10485760 # optional: default value is 10485760 + - type: "logstash" + args: + logstash_url: "http(s)://host:port" + username: "username" #optional + password: "password" #optional + max_batch_size: 500 #optional + compression_level: 1 #optional + ssl_assert_fingerprint: "22:F7:FB:84:1D:43:3E:E7:BB:F9:72:F3:D8:97:AD:7C:86:E3:08:42" #optional +``` + +::::{warning} +All versions up to 1.14.0 (included) only allow one output per type. So if the `output.type` chosen by a user is `elasticsearch`, then the user can only configure one output for it. +:::: + + + +### Fields [s3-config-file-fields] + +`inputs.[]`: + +A list of inputs (i.e. triggers) for the Elastic Serverless Forwarder Lambda function. + +`inputs.[].type`: + +The type of trigger input (`cloudwatch-logs`, `kinesis-data-stream`, `sqs` and `s3-sqs` are currently supported). + +`inputs.[].id`: + +The ARN of the trigger input according to the type. Multiple input entries can have different unique ids with the same type. Inputs of type `cloudwatch-logs` accept both CloudWatch Logs Log Group and CloudWatch Logs Log Stream ARNs. + +`inputs.[].outputs`: + +A list of outputs (i.e. forwarding targets) for the Elastic Serverless Forwarder Lambda function. You can have multiple outputs for an input, but only one output can be defined per type. + +`inputs.[].outputs.[].type`: + +The type of the forwarding target output. Currently only the following outputs are supported: + +* `elasticsearch` +* [preview] `logstash` + +Each type can only be used for a maximum of one output up to and including 1.14.0 version. If {{ls}} is chosen as an output, Elastic Serverless Forwarder expects the [`elastic_serverless_forwarder`](logstash://docs/reference/plugins-inputs-elastic_serverless_forwarder.md) Logstash input to be installed, enabled, and properly configured. For more information about installing Logstash plugins, refer to the [Logstash documentation](logstash://docs/reference/working-with-plugins.md#installing-plugins). + +`inputs.[].outputs.[].args`: Custom init arguments for the specified forwarding target output. + +For `elasticsearch` the following arguments are supported: + +* `args.elasticsearch_url`: URL of elasticsearch endpoint in the format `http(s)://domain.tld:port`. Mandatory when `args.cloud_id` is not provided. Will take precedence over `args.cloud_id` if both are defined. +* `args.cloud_id`: Cloud ID of elasticsearch endpoint. Mandatory when `args.elasticsearch_url` is not provided. Will be ignored if `args.elasticsearch_url` is defined. +* `args.username`: Username of the elasticsearch instance to connect to. Mandatory when `args.api_key` is not provided. Will take precedence over `args.api_key` if both are defined. +* `args.password` Password of the elasticsearch instance to connect to. Mandatory when `args.api_key` is not provided. Will take precedence over `args.api_key` if both are defined. +* `args.api_key`: API key of elasticsearch endpoint in the format `base64encode(api_key_id:api_key_secret)`. Mandatory when `args.username` and `args.password` are not provided. Will be ignored if `args.username`/`args.password` are defined. +* `args.es_datastream_name`: Name of data stream or index where logs should be forwarded to. Lambda supports automatic routing of various AWS service logs to the corresponding data streams for further processing and storage in the {{es}} cluster. It supports automatic routing of `aws.cloudtrail`, `aws.cloudwatch_logs`, `aws.elb_logs`, `aws.firewall_logs`, `aws.vpcflow`, and `aws.waf` logs. For other log types, if using data streams, you can optionally set its value in the configuration file according to the naming convention for data streams and available integrations. If the `es_datastream_name` is not specified and it cannot be matched with any of the above AWS services, then the value will be set to `logs-generic-default`. In versions **v0.29.1** and below, this configuration parameter was named `es_index_or_datastream_name`. Rename the configuration parameter to `es_datastream_name` in your `config.yaml` file on the S3 bucket to continue using it in the future version. The older name `es_index_or_datastream_name` is deprecated as of version **v0.30.0**. The related backward compatibility code is removed from version **v1.0.0**. +* `args.es_dead_letter_index`: Name of data stream or index where logs should be redirected to, in case indexing to `args.es_datastream_name` returned an error. The elasticseach output will NOT forward retryable errors (connection failures, HTTP status code 429) to the dead letter index. +* `args.batch_max_actions`: (Optional) Maximum number of actions to send in a single bulk request. Default value: 500. +* `args.batch_max_bytes`: (Optional) Maximum size in bytes to send in a single bulk request. Default value: 10485760 (10MB). +* `args.ssl_assert_fingerprint`: (Optional) SSL fingerprint for self-signed SSL certificate on HTTPS transport. The default value is an empty string, meaning the HTTP client requires a valid certificate. + + 1. Here is a sample error indexed in the dead letter index: + + ```json + { + "@timestamp": "2024-10-07T05:57:59.448925Z", + "message": "{\"hey\":{\"message\":\"hey there\"},\"_id\":\"e6542822-4583-438d-9b4d-1a3023b5eeb9\",\"_op_type\":\"create\",\"_index\":\"logs-succeed.pr793-default\"}", + "error": { + "message": "[1:30] failed to parse field [hey] of type [keyword] in document with id 'e6542822-4583-438d-9b4d-1a3023b5eeb9'. Preview of field's value: '{message=hey there}'", + "type": "document_parsing_exception" + }, + "http": { + "response": { + "status_code": 400 + } + } + } + ``` + + +For `logstash` the following arguments are supported: + +* `args.logstash_url`: URL of {{ls}} endpoint in the format `http(s)://host:port` +* `args.username`: (Optional) Username of the {{ls}} instance to connect to. Mandatory if HTTP Basic authentication is enabled in {{ls}}. +* `args.password`: (Optional) Password of the {{ls}} instance to connect to. Mandatory if HTTP Basic authentication is enabled in {{ls}}. +* `args.max_batch_size`: (Optional) Maximum number of events to send in a single HTTP(s) request. Default value: 500 +* `args.compression_level`: (Optional) The GZIP compression level for HTTP(s) requests towards {{ls}}. It can be any integer value between 1 (minimum compression, best performance, highest amount of bytes sent) and 9 (maximum compression, worst performance, lowest amount of bytes sent). Default value: 1 +* `args.ssl_assert_fingerprint`: (Optional) SSL fingerprint for self-signed SSL certificate on HTTPS transport. The default value is an empty string, meaning the HTTP client requires a valid certificate. + + +## Define deployment parameters [aws-serverless-forwarder-define-deploy-parameters] + +Whichever SAR deployment method you choose, you must define the following parameters correctly for your setup. This section explains the types of parameters and provides guidance on how to set them to match your deployment(s). + + +### General configuration [_general_configuration] + +These parameters define the general configuration and behaviour for the forwarder. + +* `ElasticServerlessForwarderS3ConfigFile`: Set this value to the location of your `config.yaml` in S3 URL format: `s3://bucket-name/config-file-name`. This will populate the `S3_CONFIG_FILE` environment variable for the forwarder. +* `ElasticServerlessForwarderSSMSecrets`: Add a comma delimited list of AWS SSM Secrets ARNs used in the `config.yml` (if any). +* `ElasticServerlessForwarderKMSKeys`: Add a comma delimited list of AWS KMS Keys ARNs to be used for decrypting AWS SSM Secrets, Kinesis Data Streams, SQS queue, or S3 buckets (if any). + +::::{note} +Make sure you include all the KMS keys used to encrypt the data. For example, S3 buckets are often encrypted, so the Lambda function needs access to that key to get the object. + +:::: + + + +### Inputs [_inputs] + +These parameters define your specific [Inputs](/reference/index.md#aws-serverless-forwarder-inputs) or *event triggers*. + +* `ElasticServerlessForwarderSQSEvents`: Add a comma delimited list of Direct SQS queue ARNs to set as event triggers for the forwarder (if any). +* `ElasticServerlessForwarderSQSEvents2`: Add a comma delimited list of Direct SQS queue ARNs to set as event triggers for the forwarder (if limit is reach on ElasticServerlessForwarderSQSEvents). +* `ElasticServerlessForwarderS3SQSEvents`: Add a comma delimited list of S3 SQS Event Notifications ARNs to set as event triggers for the forwarder (if any). +* `ElasticServerlessForwarderS3SQSEvents2`: Add a comma delimited list of S3 SQS Event Notifications ARNs to set as event triggers for the forwarder (if limit is reach on ElasticServerlessForwarderS3SQSEvents). +* `ElasticServerlessForwarderKinesisEvents`: Add a comma delimited list of Kinesis Data Stream ARNs to set as event triggers for the forwarder (if any). +* `ElasticServerlessForwarderKinesisEvents2`: Add a comma delimited list of Kinesis Data Stream ARNs to set as event triggers for the forwarder (if limit is reach on ElasticServerlessForwarderKinesisEvents). +* `ElasticServerlessForwarderCloudWatchLogsEvents`: Add a comma delimited list of Cloudwatch Logs log group ARNs to set subscription filters on the forwarder (if any). +* `ElasticServerlessForwarderCloudWatchLogsEvents2`: Add a comma delimited list of Cloudwatch Logs log group ARNs to set subscription filters on the forwarder (if limit is reach on ElasticServerlessForwarderCloudWatchLogsEvents). + +::::{note} +Make sure you reference the ARNs specified in your `config.yaml`, and leave any settings for unused inputs blank. + +:::: + + + +### S3 Bucket permissions [_s3_bucket_permissions] + +These parameters define the permissions required in order to access the associated S3 Buckets. + +* `ElasticServerlessForwarderS3Buckets`: Add a comma delimited list of S3 bucket ARNs that are sources for the S3 SQS Event Notifications (if any). + + +### Network [_network] + +To attach the Elastic Serverless Forwarder to a specific AWS VPC, specify the security group IDs and subnet IDs that belong to the AWS VPC. This requirement is related to the [CloudFormation VPCConfig property](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-lambda-function-vpcconfig.md). + +These are the parameters: + +* `ElasticServerlessForwarderSecurityGroups`: Add a comma delimited list of security group IDs to attach to the forwarder. +* `ElasticServerlessForwarderSubnets`: Add a comma delimited list of subnet IDs for the forwarder. + +Both parameters are required in order to attach the Elastic Serverless Forwarder to a specific AWS VPC. Leave both parameters blank if you don’t want the forwarder to belong to any specific AWS VPC. + +If the Elastic Serverless Forwarder is attached to a VPC, you need to [create VPC endpoints](https://docs.aws.amazon.com/vpc/latest/privatelink/create-interface-endpoint.md) for S3 and SQS, and for **every** service you define as an input for the forwarder. S3 and SQS VPC endpoints are always required for reading the `config.yaml` uploaded to S3 and managing the continuing queue and the replay queue, regardless of the [Inputs](/reference/index.md#aws-serverless-forwarder-inputs) used. If you use [Amazon CloudWatch Logs subscription filters](/reference/index.md#aws-serverless-forwarder-inputs-cloudwatch), you need to create a VPC endpoint for EC2, too. + +::::{note} +Refer to the [AWS PrivateLink traffic filters](docs-content://deploy-manage/security/aws-privatelink-traffic-filters.md) documentation to find your VPC endpoint ID and the hostname to use in the `config.yml` in order to access your Elasticsearch cluster over PrivateLink. +:::: + + + +## Deploy Elastic Serverless Forwarder from Terraform [aws-serverless-forwarder-deploy-terraform] + +The terraform files to deploy ESF can be found in [`esf-terraform` repository](https://github.com/elastic/terraform-elastic-esf). There are two requirements to deploy these files: [curl](https://curl.se/download.md) and [terraform](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli). Refer to the [README file](https://github.com/elastic/terraform-elastic-esf/blob/main/README.md) to learn how to use it. + + +## Deploy Elastic Serverless Forwarder from SAR [aws-serverless-forwarder-deploy-sar] + +There are several deployment methods available via the AWS Serverless Application Repository (SAR): + +* [Deploy using AWS Console](#aws-serverless-forwarder-deploy-console) +* [Deploy using Cloudformation](#aws-serverless-forwarder-deploy-cloudformation) +* [Deploy from SAR using Terraform](#aws-serverless-forwarder-deploy-sar-terraform) + +::::{note} +To deploy the forwarder directly without using SAR, refer to [Deploy Elastic Serverless Forwarder directly](#aws-serverless-forwarder-direct-deploy) +:::: + + + +### Deploy using AWS Console [aws-serverless-forwarder-deploy-console] + +::::{note} +Only one deployment per region is allowed when using the AWS console directly. +:::: + + +1. Log in to AWS console and open **Lambda**. +2. Click **Applications** and then **Create application**. +3. Click **Serverless application** and search for **elastic-serverless-forwarder**. +4. Select **elastic-serverless-forwarder** from the search results (ignoring any application beginning **helper-**). + + :::{image} ../images/aws-serverless-forwarder-create-function.png + :alt: Create Elastic Serverless Forwarder Lambda function within SAR + :class: screenshot + ::: + +5. Complete the **Application settings** according to [Define deployment parameters](#aws-serverless-forwarder-define-deploy-parameters). You must specify the parameters even if they already exist in the `config.yaml` file. Depends on the input type, at least one if the parameters `ElasticServerlessForwarderSQSEvents`, `ElasticServerlessForwarderS3SQSEvents`, `ElasticServerlessForwarderKinesisEvents`, `ElasticServerlessForwarderCloudWatchLogsEvents` should de defined. +6. After your settings have been added, click **Deploy**. +7. On the Applications page for **serverlessrepo-elastic-serverless-forwarder**, click **Deployments**. +8. Refresh the **Deployment history** until you see the `Create complete` status update. It should take around 5 minutes to deploy — if the deployment fails for any reason, the create events will be rolled back and you will be able to see an explanation for which event failed. +9. (Optional) To enable Elastic APM instrumentation for your new deployment: + + * Go to **Lambda > Functions** within AWS console, and find and select the function with **serverlessrepo-**. + * Go to **Configuration** tab and select **Environment Variables** + * Add the following environment variables: + + ``` + | Key | Value | + |---------------------------|--------| + |`ELASTIC_APM_ACTIVE` | `true` | + |`ELASTIC_APM_SECRET_TOKEN` | token | + |`ELASTIC_APM_SERVER_URL` | url | + ``` + + +::::{note} +If you have already successfully deployed the forwarder but want to update the application (for example, if a new version of the Lambda function is released), you should go through this deploy step again and use the same **Application name**. This will ensure the function is updated rather than duplicated or created anew. +:::: + + + +### Deploy using Cloudformation [aws-serverless-forwarder-deploy-cloudformation] + +1. Use the following code to get the semantic version of the latest application: + + ```bash + aws serverlessrepo list-application-versions --application-id arn:aws:serverlessrepo:eu-central-1:267093732750:applications/elastic-serverless-forwarder + ``` + +2. Save the following YAML content as `sar-application.yaml` and fill in the correct parameters according to [Define deployment parameters](#aws-serverless-forwarder-define-deploy-parameters): + + ```yaml + Transform: AWS::Serverless-2016-10-31 + Resources: + SarCloudformationDeployment: + Type: AWS::Serverless::Application + Properties: + Location: + ApplicationId: 'arn:aws:serverlessrepo:eu-central-1:267093732750:applications/elastic-serverless-forwarder' + SemanticVersion: '%SEMANTICVERSION%' ## SET TO CORRECT SEMANTIC VERSION (MUST BE GREATER THAN 1.6.0) + Parameters: + ElasticServerlessForwarderS3ConfigFile: "" + ElasticServerlessForwarderSSMSecrets: "" + ElasticServerlessForwarderKMSKeys: "" + ElasticServerlessForwarderSQSEvents: "" + ElasticServerlessForwarderSQSEvents2: "" ## IF SEMANTIC VERSION GREATER THAN 1.12.0 + ElasticServerlessForwarderS3SQSEvents: "" + ElasticServerlessForwarderS3SQSEvents2: "" ## IF SEMANTIC VERSION GREATER THAN 1.12.0 + ElasticServerlessForwarderKinesisEvents: "" + ElasticServerlessForwarderKinesisEvents2: "" ## IF SEMANTIC VERSION GREATER THAN 1.12.0 + ElasticServerlessForwarderCloudWatchLogsEvents: "" + ElasticServerlessForwarderCloudWatchLogsEvents2: "" ## IF SEMANTIC VERSION GREATER THAN 1.12.0 + ElasticServerlessForwarderS3Buckets: "" + ElasticServerlessForwarderSecurityGroups: "" + ElasticServerlessForwarderSubnets: "" + ``` + +3. Deploy the Lambda function from SAR by running the following command: + + ```shell + aws cloudformation deploy --template-file sar-application.yaml --stack-name esf-cloudformation-deployment --capabilities CAPABILITY_IAM CAPABILITY_AUTO_EXPAND + ``` + + +::::{note} +Starting from **v1.4.0**, if you want to update the Events settings for the forwarder, you do not need to manually delete existing settings before applying new settings. +:::: + + + +### Deploy from SAR using Terraform [aws-serverless-forwarder-deploy-sar-terraform] + +1. Save the following yaml content as `sar-application.tf` and fill in the correct parameters according to [Define deployment parameters](#aws-serverless-forwarder-define-deploy-parameters): + + ```yaml + provider "aws" { + region = "" ## FILL WITH THE AWS REGION WHERE YOU WANT TO DEPLOY THE ELASTIC SERVERLESS FORWARDER + } + data "aws_serverlessapplicationrepository_application" "esf_sar" { + application_id = "arn:aws:serverlessrepo:eu-central-1:267093732750:applications/elastic-serverless-forwarder" + } + resource "aws_serverlessapplicationrepository_cloudformation_stack" "esf_cf_stak" { + name = "terraform-elastic-serverless-forwarder" + application_id = data.aws_serverlessapplicationrepository_application.esf_sar.application_id + semantic_version = data.aws_serverlessapplicationrepository_application.esf_sar.semantic_version + capabilities = data.aws_serverlessapplicationrepository_application.esf_sar.required_capabilities + parameters = { + ElasticServerlessForwarderS3ConfigFile = "" + ElasticServerlessForwarderSSMSecrets = "" + ElasticServerlessForwarderKMSKeys = "" + ElasticServerlessForwarderSQSEvents = "" + ElasticServerlessForwarderS3SQSEvents = "" + ElasticServerlessForwarderKinesisEvents = "" + ElasticServerlessForwarderCloudWatchLogsEvents = "" + ElasticServerlessForwarderS3Buckets = "" + ElasticServerlessForwarderSecurityGroups = "" + ElasticServerlessForwarderSubnets = "" + } + } + ``` + +2. Deploy the function from SAR by running the following commands: + + ```shell + terraform init + terraform apply + ``` + + +::::{note} +From **v1.4.0** and above, if you want to update the Events settings for the deployment, it is no longer required to manually delete existing settings before applying the new settings. + +Due to a [Terraform bug](https://github.com/hashicorp/terraform-provider-aws/issues/24771) related to `aws_serverlessapplicationrepository_application`, if you want to delete existing Event parameters you have to set the related `aws_serverlessapplicationrepository_cloudformation_stack.parameters` to a blank space value (`" "`) instead of an empty string (`""`). + +:::: + + + +## Deploy Elastic Serverless Forwarder directly [aws-serverless-forwarder-direct-deploy] + +For more customisation options during deployment, from version `1.6.0` and above you can deploy the Elastic Serverless Forwarder directly to your AWS Account without using SAR. This enables you to customize the event source settings for the inputs (i.e. triggers) one-by-one. + +To deploy the forwarder directly, you have to: + +* [Install AWS integration assets in {{kib}}](#aws-serverless-forwarder-deploy-kibana) +* [Create and upload `config.yaml` to S3 bucket](#sample-s3-config-file) +* [Create `publish-config.yaml` for the publishing script](#sample-direct-publish-config-file) +* [Run the publishing script](#aws-serverless-forwarder-run-publish-script) + + +### Create `publish-config.yaml` for the publishing script [sample-direct-publish-config-file] + +To deploy the forwarder directly, you need to define a `publish-config.yaml` file and pass this as an argument in the [publishing script](#aws-serverless-forwarder-run-publish-script). + +Save the following YAML content as `publish-config.yaml` and edit as required before running the publishing script. You should remove any inputs or arguments you are not using. + +```yaml +kinesis-data-stream: + - arn: "arn:aws:kinesis:%REGION%:%ACCOUNT%:stream/%STREAMNAME%" + batch_size: 10 + batching_window_in_second: 0 + starting_position: TRIM_HORIZON + starting_position_timestamp: 0 + parallelization_factor: 1 +sqs: + - arn: "arn:aws:sqs:%REGION%:%ACCOUNT%:%QUEUENAME%" + batch_size: 10 + batching_window_in_second: 0 +s3-sqs: + - arn: "arn:aws:sqs:%REGION%:%ACCOUNT%:%QUEUENAME%" + batch_size: 10 + batching_window_in_second: 0 +cloudwatch-logs: + - arn: "arn:aws:logs:%AWS_REGION%:%AWS_ACCOUNT_ID%:log-group:%LOG_GROUP_NAME%:*" + - arn: "arn:aws:logs:%AWS_REGION%:%AWS_ACCOUNT_ID%:log-group:%LOG_GROUP_NAME%:log-stream:%LOG_STREAM_NAME%" +ssm-secrets: + - "arn:aws:secretsmanager:%AWS_REGION%:%AWS_ACCOUNT_ID%:secret:%SECRET_NAME%" +kms-keys: + - "arn:aws:kms:%AWS_REGION%:%AWS_ACCOUNT_ID%:key/%KMS_KEY_UUID%" +s3-buckets: + - "arn:aws:s3:::%BUCKET_NAME%" +subnets: + - "%SUBNET_ID%" +security-groups: + - "%SECURITY_ID%" +s3-config-file: "s3://%S3_CONFIG_BUCKET_NAME%/%S3_CONFIG_OBJECT_KEY%" +continuing-queue: + batch_size: 10 + batching_window_in_second: 0 +``` + + +### Fields [direct-publish-config-file-fields] + +| | | +| --- | --- | +| `kinesis-data-stream.[]` | List of [Amazon Kinesis Data Streams](/reference/index.md#aws-serverless-forwarder-inputs-kinesis) (i.e. triggers) for the forwarder, matching those defined in your [Create and upload `config.yaml` to S3 bucket](#sample-s3-config-file). | +| `kinesis-data-stream.[].arn` | ARN of the AWS Kinesis Data Stream. | +| `kinesis-data-stream.[].batch_size` | Set this value above the default (`10`) if you experience ingestion delays in your output **and** `GetRecords.IteratorAgeMilliseconds` and `IncomingRecords` Kinesis CloudWatch metrics for the [Amazon Kinesis Data Streams](/reference/index.md#aws-serverless-forwarder-inputs-kinesis) keep increasing **and** the average execution time of the forwarder is below 14 minutes. This will increase the number of records the forwarder will process in a single execution for the [Amazon Kinesis Data Streams](/reference/index.md#aws-serverless-forwarder-inputs-kinesis). | +| `kinesis-data-stream.[].batching_window_in_second` | Set this value above the default (`0`) if you experience ingestion delays in your output **and** `GetRecords.IteratorAgeMilliseconds` and `IncomingRecords` Kinesis CloudWatch metrics for the [Amazon Kinesis Data Streams](/reference/index.md#aws-serverless-forwarder-inputs-kinesis) keep increasing **and** the average execution time of the forwarder is below 14 minutes. This will increase the number of records the forwarder will process in a single execution for the [Amazon Kinesis Data Streams](/reference/index.md#aws-serverless-forwarder-inputs-kinesis). | +| `kinesis-data-stream.[].starting_position` | Change this value from the default (`TRIM_HORIZON`) if you want to change the starting position of the records processed by the forwarder for the [Amazon Kinesis Data Streams](/reference/index.md#aws-serverless-forwarder-inputs-kinesis). | +| `kinesis-data-stream.[].starting_position_timestamp` | Set this value to the time from which to start reading (in Unix time seconds) if you set `ElasticServerlessForwarderKinesisStartingPosition` to "AT_TIMESTAMP". | +| `kinesis-data-stream.[].parallelization_factor` | Defines the number of forwarder functions that can run concurrently per shard (default is `1`). Increase this value if you experience ingestion delays in your output **and** `GetRecords.IteratorAgeMilliseconds` and `IncomingRecords` Kinesis CloudWatch metrics for the [Amazon Kinesis Data Streams](/reference/index.md#aws-serverless-forwarder-inputs-kinesis) keep increasing **and** the average execution time of the forwarder is below 14 minutes. This will increase the number of records processed concurrently for [Amazon Kinesis Data Streams](/reference/index.md#aws-serverless-forwarder-inputs-kinesis). For more info, refer to [AWS Kinesis docs](https://docs.aws.amazon.com/lambda/latest/dg/with-kinesis.md). | +| `sqs.[]` | List of [Amazon SQS message payload](/reference/index.md#aws-serverless-forwarder-inputs-direct) (i.e. triggers) for the forwarder, matching those defined in your [Create and upload `config.yaml` to S3 bucket](#sample-s3-config-file). | +| `sqs.[].arn` | ARN of the AWS SQS queue trigger input. | +| `sqs.[].batch_size` | Set this value above the default (`10`) if you experience ingestion delays in your output **and** `ApproximateNumberOfMessagesVisible` and `ApproximateAgeOfOldestMessage` SQS CloudWatch metrics for the [Amazon SQS message payload](/reference/index.md#aws-serverless-forwarder-inputs-direct) keep increasing **and** the average execution time of the forwarder is below 14 minutes. This will increase the number of messages the forwarder will process in a single execution for the [Amazon SQS message payload](/reference/index.md#aws-serverless-forwarder-inputs-direct). | +| `sqs.[].batching_window_in_second` | Set this value above the default (`0`) if you experience ingestion delays in your output **and** `ApproximateNumberOfMessagesVisible` and `ApproximateAgeOfOldestMessage` SQS CloudWatch metrics for the [Amazon SQS message payload](/reference/index.md#aws-serverless-forwarder-inputs-direct) keep increasing **and** the average execution time of the forwarder is below 14 minutes. This will increase the number of messages the forwarder will process in a single execution for the [Amazon SQS message payload](/reference/index.md#aws-serverless-forwarder-inputs-direct). | +| `s3-sqs.[]` | List of [Amazon S3 (via SQS event notifications)](/reference/index.md#aws-serverless-forwarder-inputs-s3) (i.e. triggers) for the forwarder, matching those defined in your [Create and upload `config.yaml` to S3 bucket](#sample-s3-config-file). | +| `s3-sqs.[].arn` | ARN of the AWS SQS queue receiving S3 Notifications as trigger input. | +| `s3-sqs.[].batch_size` | Set this value above the default (`10`) if you experience ingestion delays in your output **and** `ApproximateNumberOfMessagesVisible` and `ApproximateAgeOfOldestMessage` SQS CloudWatch metrics for the [Amazon S3 (via SQS event notifications)](/reference/index.md#aws-serverless-forwarder-inputs-s3) keep increasing **and** the average execution time of the forwarder is below 14 minutes. This will increase the number of messages the forwarder will process in a single execution for the [Amazon S3 (via SQS event notifications)](/reference/index.md#aws-serverless-forwarder-inputs-s3). | +| `s3-sqs.[].batching_window_in_second` | Set this value above the default (`0`) if you experience ingestion delays in your output **and** `ApproximateNumberOfMessagesVisible` and `ApproximateAgeOfOldestMessage` SQS CloudWatch metrics for the [Amazon S3 (via SQS event notifications)](/reference/index.md#aws-serverless-forwarder-inputs-s3) keep increasing **and** the average execution time of the forwarder is below 14 minutes. This will increase the number of messages the forwarder will process in a single execution for the [Amazon S3 (via SQS event notifications)](/reference/index.md#aws-serverless-forwarder-inputs-s3). | +| `cloudwatch-logs.[]` | List of [Amazon CloudWatch Logs subscription filters](/reference/index.md#aws-serverless-forwarder-inputs-cloudwatch) (i.e. triggers) for the forwarder, matching those defined in your [Create and upload `config.yaml` to S3 bucket](#sample-s3-config-file). | +| `cloudwatch-logs.[].arn` | ARN of the AWS CloudWatch Logs trigger input (accepts both CloudWatch Logs Log Group and CloudWatch Logs Log Stream ARNs). | +| `ssm-secrets.[]` | List of AWS SSM Secrets ARNs used in your `config.yml` (if any). | +| `kms-keys.[]` | List of AWS KMS Keys ARNs to be used for decrypting AWS SSM Secrets, Kinesis Data Streams or SQS queues (if any). | +| `s3-buckets.[]` | List of S3 bucket ARNs that are sources for the S3 SQS Event Notifications (if any). | +| `subnets.[]` | A list of subnets IDs for the forwarder. Along with `security-groups.[]`, these settings will define the AWS VPC the forwarder will belong to. Leave blank if you don’t want the forwarder to belong to any specific AWS VPC. | +| `security-groups.[]` | List of security group IDs to attach to the forwarder. Along with `subnets.[]`, these settings will define the AWS VPC the forwarder will belong to. Leave blank if you don’t want to have the forwarder belong to any specific AWS VPC. | +| `s3-config-file` | Set this value to the location of your forwarder configuration file in S3 URL format: `s3://bucket-name/config-file-name`. This will populate the `S3_CONFIG_FILE` environment variable for the forwarder. | +| `continuing-queue.batch_size` | Set this value above the default (`10`) if you experience ingestion delays in your output **and** `ApproximateNumberOfMessagesVisible` and `ApproximateAgeOfOldestMessage` SQS CloudWatch metrics for the continuing queue keep increasing **and** the average execution time of the forwarder is below 14 minutes. This will increase the number of messages the forwarder will process in a single execution for the continuing queue. | +| `continuing-queue.batching_window_in_second` | Set this value above the default (`0`) if you experience ingestion delays in your output **and** `ApproximateNumberOfMessagesVisible` and `ApproximateAgeOfOldestMessage` SQS CloudWatch metrics for the continuing queue keep increasing **and** the average execution time of the forwarder is below 14 minutes. This will increase the number of messages the forwarder will process in a single execution for the continuing queue. | + + +### Run the publishing script [aws-serverless-forwarder-run-publish-script] + +A bash script for publishing the Elastic Serverless Forwarder directly to your AWS account is available from the [Elastic Serverless Forwarder repository](https://github.com/elastic/elastic-serverless-forwarder). + +Download the [`publish_lambda.sh` script](https://raw.githubusercontent.com/elastic/elastic-serverless-forwarder/lambda-v1.8.0/publish_lambda.sh) and follow the instructions below. + + +#### Script arguments [_script_arguments] + +```bash + $ ./publish_lambda.sh + AWS CLI (https://aws.amazon.com/cli/), SAM (https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/install-sam-cli.html) and Python3.9 with pip3 required + Please, before launching the tool execute "$ pip3 install ruamel.yaml" +Usage: ./publish_lambda.sh config-path lambda-name forwarder-tag bucket-name region [custom-role-prefix] + Arguments: + config-path: full path to the publish configuration + lambda-name: name of the lambda to be published in the account + forwarder-tag: tag of the elastic serverless forwarder to publish + bucket-name: bucket name where to store the zip artifact for the lambda + (it will be created if it doesn't exists, otherwise + you need already to have proper access to it) + region: region where to publish in + custom-role-prefix: role/policy prefix to add in case customization is needed (optional) + (please note that the prefix will be added to both role/policy naming) +``` + + +#### Prerequisites [_prerequisites] + +* Python3.9 with pip3 is required to run the script +* [AWS CLI](https://aws.amazon.com/cli/), [SAM CLI](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/install-sam-cli.md) and the [ruamel.yaml package](https://pypi.org/project/ruamel.yaml/) must also be installed + +```bash +$ pip3 install awscli aws-sam-cli ruamel.yaml +``` + + +#### Running the script [_running_the_script] + +Assuming `publish-config.yaml` in saved in the same directory you intend to run `publish_lambda.sh` from, here’s an example: + +```bash +$ ./publish_lambda.sh publish-config.yaml forwarder-lambda lambda-v1.6.0 s3-lambda-artifact-bucket-name eu-central-1 +``` + + +#### Updating to a new version via script [_updating_to_a_new_version_via_script] + +You can update the version of a published Elastic Serverless Forwarder without changing its configuration by running the publishing script again and passing a **new** [`forwarder-tag`](https://github.com/elastic/elastic-serverless-forwarder/tags): + +```bash +$ ./publish_lambda.sh publish-config.yaml forwarder-lambda lambda-v1.7.0 s3-lambda-artifact-bucket-name eu-central-1 +``` + +::::{note} +The above examples show the forwarder being updated from `lambda-v1.6.0` to `lambda-v1.7.0`. +:::: + + + +#### Changing configuration via script [_changing_configuration_via_script] + +If you want to change the configuration of a published Elastic Serverless Forwarder without changing its version, you can update the `publish-config.yaml` and run the script again using the **same** `forwarder-tag`: + +```bash +$ ./publish_lambda.sh publish-config.yaml forwarder-lambda lambda-v1.6.0 s3-lambda-artifact-bucket-name eu-central-1 +``` + +::::{note} +The above example shows an existing `lambda-v1.6.0` configuration being updated without changing version. +:::: + + + +#### Using the script for multiple deployments [_using_the_script_for_multiple_deployments] + +If you want to use the publish script for deploying the forwarder with different configurations, create two different `publish-config.yaml` files with unique names and run the publishing script twice, with correct references to the `config-path` and `lambda-name`: + +```bash +$ ./publish_lambda.sh publish-config-for-first-lambda.yaml first-lambda lambda-v1.6.0 s3-lambda-artifact-bucket-name eu-central-1 + +$ ./publish_lambda.sh publish-config-for-second-lambda.yaml second-lambda lambda-v1.6.0 ss3-lambda-artifact-bucket-name eu-central-1 +``` + +::::{note} +The above example publishes two versions of the forwarder, each with different configurations i.e. `publish-config-for-first-lambda.yaml` and `first-lambda` vs. `publish-config-for-second-lambda.yaml` and `second-lambda`. +:::: diff --git a/docs/en/aws-elastic-serverless-forwarder-configuration.asciidoc b/docs/reference/aws-elastic-serverless-forwarder-configuration.md similarity index 66% rename from docs/en/aws-elastic-serverless-forwarder-configuration.asciidoc rename to docs/reference/aws-elastic-serverless-forwarder-configuration.md index 45af6827..77a5ba72 100644 --- a/docs/en/aws-elastic-serverless-forwarder-configuration.asciidoc +++ b/docs/reference/aws-elastic-serverless-forwarder-configuration.md @@ -1,95 +1,77 @@ -:aws: AWS +--- +navigation_title: "Configuration options" +mapped_pages: + - https://www.elastic.co/guide/en/esf/current/aws-elastic-serverless-forwarder-configuration.html +--- -[[aws-elastic-serverless-forwarder-configuration]] -= Configuration options for Elastic Serverless Forwarder +# Configuration options for Elastic Serverless Forwarder [aws-elastic-serverless-forwarder-configuration] -++++ -Configuration options -++++ -:keywords: serverless -:description: Learn configuration options for Elastic Serverless Forwarder and how to transform your AWS data as it is collected from Amazon Web Services ({aws}). +Learn more about configuration options for Elastic Serverless Forwarder, including more detail on permissions and policies, automatic routing of AWS service logs, and how to use AWS Secrets Manager for authentication. -Learn more about configuration options for Elastic Serverless Forwarder, including more detail on permissions and policies, automatic routing of {aws} service logs, and how to use AWS Secrets Manager for authentication. +You can transform or enrich data from AWS as it is parsed by the forwarder. This includes examples such as using tags and filters to organise your data and exclude specific messages, automatically discovering and collecting JSON content, and managing multiline messages effectively. -You can transform or enrich data from {aws} as it is parsed by the forwarder. This includes examples such as using tags and filters to organise your data and exclude specific messages, automatically discovering and collecting JSON content, and managing multiline messages effectively. +* [Permissions and policies](#lambda-permissions-policies) +* [Use AWS Secrets Manager](#use-secrets-manager) +* [Route AWS service logs](#aws-serverless-route-service-logs) +* [Use tags and filters](#aws-serverless-use-tags-filters) +* [JSON content discovery](#aws-serverless-json-content) +* [Manage multiline messages](#aws-serverless-manage-multiline-messages) -- <> -- <> +## Permissions and policies [lambda-permissions-policies] -- <> +A Lambda function’s execution role is an AWS Identity and Access Management (IAM) role that grants the function permission to access AWS services and resources. This role is automatically created when the function is deployed and Lambda assumes this role when the function is invoked. -- <> - -- <> - -- <> - -[discrete] -[[lambda-permissions-policies]] -== Permissions and policies - -A Lambda function's execution role is an {aws} Identity and Access Management (IAM) role that grants the function permission to access {aws} services and resources. This role is automatically created when the function is deployed and Lambda assumes this role when the function is invoked. - -When you provide the ARNs of {aws} resources the forwarder will interact with, the Cloudformation template will create the correct IAM role with the appropriate IAM policies. +When you provide the ARNs of AWS resources the forwarder will interact with, the Cloudformation template will create the correct IAM role with the appropriate IAM policies. You can view the execution role associated with your Lambda function from the **Configuration > Permissions** section within . By default this role starts with the name **serverlessrepo-**. When the role is created, a custom policy is added to grant Lambda minimum permissions to be able to use the configured SQS queue, S3 buckets, Kinesis data stream, CloudWatch Logs log groups, Secrets manager (if using), and SQS replay queue. The forwarder is granted the following `ManagedPolicyArns` permissions, which are automatically added by default to the Events configuration (if relevant): -[source, bash] ----- +```bash arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole arn:aws:iam::aws:policy/service-role/AWSLambdaKinesisExecutionRole arn:aws:iam::aws:policy/service-role/AWSLambdaSQSQueueExecutionRole ----- +``` In addition to these basic permissions, the following permissions are added when the function is created by the Cloudformation template of the function: * For SQS queue resources specified in the `SQS_CONTINUE_URL` and `SQS_REPLAY_URL` environment variables, the following action is allowed: `sqs:SendMessage` - * For S3 bucket resources specified in the `S3_CONFIG_FILE` environment variable, the following action is allowed on the S3 buckets' config file object key: `s3:GetObject` - * For every S3 bucket resource sending notifications to SQS queues, the following action is allowed on the S3 buckets: `s3:ListBucket` - * For every S3 bucket resource sending notifications to SQS queues, the following action is allowed on the S3 buckets' keys: `s3:GetObject` - * For every Secret Manager secret that you want to refer in the `config.yaml` file, the following action is allowed: `secretsmanager:GetSecretValue` - * Excepting the default key used to encrypt your Secret Manager secrets with, the following action is allowed for every decrypt key: `kms:Decrypt` - * If any CloudWatch Logs log groups are set as Lambda inputs, the following actions are allowed for the resource: -** `arn:aws:logs:%AWS_REGION%:%AWS_ACCOUNT_ID%:log-group:*:*` -** `logs:DescribeLogGroups` -[discrete] -[[lambda-policy-cloudwatch]] -=== Lambda resource-based policy for CloudWatch Logs subscription filter input + * `arn:aws:logs:%AWS_REGION%:%AWS_ACCOUNT_ID%:log-group:*:*` + * `logs:DescribeLogGroups` + + + +### Lambda resource-based policy for CloudWatch Logs subscription filter input [lambda-policy-cloudwatch] For CloudWatch Logs subscription filter log group resources that you want to use as triggers for the forwarder, the following is allowed as a resource-based policy in separate Policy statements: -[source, yaml] ----- +```yaml * Principal: logs.%AWS_REGION%.amazonaws.com * Action: lambda:InvokeFunction * Source ARN: arn:aws:logs:%AWS_REGION%:%AWS_ACCOUNT_ID%:log-group:%LOG_GROUP_NAME%:* ----- +``` + -[discrete] -[[use-secrets-manager]] -== Use {aws} Secrets Manager +## Use AWS Secrets Manager [use-secrets-manager] -{aws} Secrets Manager enables you to replace hardcoded credentials in your code, including passwords, with an API call to Secrets Manager to retrieve the secret programmatically. For more info, refer to the https://docs.aws.amazon.com/secretsmanager/index.html[{aws} Secrets Manager documentation]. +AWS Secrets Manager enables you to replace hardcoded credentials in your code, including passwords, with an API call to Secrets Manager to retrieve the secret programmatically. For more info, refer to the [AWS Secrets Manager documentation](https://docs.aws.amazon.com/secretsmanager/index.md). There are 2 types of secrets that can be used: -- SecretString (plain text or key/value pairs) -- SecretBinary +* SecretString (plain text or key/value pairs) +* SecretBinary -The following code shows API calls to {aws} Secrets Manager: +The following code shows API calls to AWS Secrets Manager: -[source, yaml] ----- +```yaml inputs: - type: "s3-sqs" id: "arn:aws:sqs:%REGION%:%ACCOUNT%:%QUEUENAME%" @@ -100,43 +82,39 @@ inputs: username: "arn:aws:secretsmanager:eu-west-1:123456789:secret:es_secrets:username" password: "arn:aws:secretsmanager:eu-west-1:123456789:secret:es_secrets:password" es_datastream_name: "logs-generic-default" ----- +``` To use a **plain text** or **binary** secret, note the following format for the ARN: -[source, yaml] ----- +```yaml arn:aws:secretsmanager:AWS_REGION:AWS_ACCOUNT_ID:secret:SECRET_NAME ----- +``` In order to use a **key/value** pair secret, you need to provide the key at the end of the arn, as per: -[source, yaml] ----- +```yaml arn:aws:secretsmanager:AWS_REGION:AWS_ACCOUNT_ID:secret:SECRET_NAME:SECRET_KEY ----- +``` -[NOTE] -==== +::::{note} * Secrets from different regions are supported, but the only version currently retrieved for a secret is `AWSCURRENT`. * You cannot use the same secret for both plain text and key/value pairs. * Secrets are case-sensitive. * Any configuration error or typo in the `config.yaml` file will be ignored (or exceptions raised) and secrets will not be retrieved. -* Keys must exist in the {aws} Secrets Manager. +* Keys must exist in the AWS Secrets Manager. * Empty values for a given key are not allowed. -==== -// link to guide/en/apm/lambda/current/aws-lambda-secrets-manager.html? +:::: + + + +## Route AWS service logs [aws-serverless-route-service-logs] -[discrete] -[[aws-serverless-route-service-logs]] -== Route AWS service logs +For `S3 SQS Event Notifications` inputs, the Elastic Serverless Forwarder supports automatic routing of several AWS service logs to the corresponding [integration data streams](integration-docs://docs/reference/index.md) for further processing and storage in the {{es}} cluster. -For `S3 SQS Event Notifications` inputs, the Elastic Serverless Forwarder supports automatic routing of several AWS service logs to the corresponding https://docs.elastic.co/en/integrations[integration data streams] for further processing and storage in the {es} cluster. -[discrete] -[[aws-serverless-automatic-routing]] -=== Automatic routing +### Automatic routing [aws-serverless-automatic-routing] + Elastic Serverless Forwarder supports automatic routing of the following logs to the corresponding default integration data stream: * AWS CloudTrail (`aws.cloudtrail`) @@ -150,28 +128,23 @@ For these use cases, setting the `es_datastream_name` field in the configuration For most other use cases, you will need to set the `es_datastream_name` field in the configuration file to route the data to a specific data stream or index. This value should be set in the following use cases: -- You want to write the data to a specific index, alias, or custom data stream, and not to the default integration data stream. This can help some users to use existing {es} assets like index templates, ingest pipelines, or dashboards, that are already set up and connected to business processes. -- When using `Kinesis Data Stream`, `CloudWatch Logs subscription filter` or `Direct SQS message payload` inputs. Only the `S3 SQS Event Notifications` input method supports automatic routing to default integration data streams for several AWS service logs. -- When using `S3 SQS Event Notifications` but where the log type is something **other than** AWS CloudTrail (`aws.cloudtrail`), Amazon CloudWatch Logs (`aws.cloudwatch_logs`), Elastic Load Balancing (`aws.elb_logs`), AWS Network Firewall (`aws.firewall_logs`), Amazon VPC Flow (`aws.vpcflow`), and AWS Web Application Firewall (`aws.waf`). +* You want to write the data to a specific index, alias, or custom data stream, and not to the default integration data stream. This can help some users to use existing {{es}} assets like index templates, ingest pipelines, or dashboards, that are already set up and connected to business processes. +* When using `Kinesis Data Stream`, `CloudWatch Logs subscription filter` or `Direct SQS message payload` inputs. Only the `S3 SQS Event Notifications` input method supports automatic routing to default integration data streams for several AWS service logs. +* When using `S3 SQS Event Notifications` but where the log type is something **other than** AWS CloudTrail (`aws.cloudtrail`), Amazon CloudWatch Logs (`aws.cloudwatch_logs`), Elastic Load Balancing (`aws.elb_logs`), AWS Network Firewall (`aws.firewall_logs`), Amazon VPC Flow (`aws.vpcflow`), and AWS Web Application Firewall (`aws.waf`). If the `es_datastream_name` is not specified, and the log cannot be matched with any of the above AWS services, then the dataset will be set to `generic` and the namespace set to `default`, pointing to the data stream name `logs-generic-default`. -[discrete] -[[aws-serverless-use-tags-filters]] -== Use tags and filters + +## Use tags and filters [aws-serverless-use-tags-filters] You can use tags and filters to tag and filter messages based on regular expressions. -// expand? should we move this into Configure topic? -[discrete] -[[aws-serverless-use-tags]] -=== Use custom tags +### Use custom tags [aws-serverless-use-tags] You can add custom tags to filter and categorize items in events. -[source, yaml] ----- +```yaml inputs: - type: "s3-sqs" id: "arn:aws:sqs:%REGION%:%ACCOUNT%:%QUEUENAME%" @@ -186,7 +159,7 @@ inputs: username: "arn:aws:secretsmanager:eu-west-1:123456789:secret:es_secrets:username" password: "arn:aws:secretsmanager:eu-west-1:123456789:secret:es_secrets:password" es_datastream_name: "logs-generic-default" ----- +``` Using the above example configuration, the tags will be set in the following way: @@ -194,20 +167,19 @@ Using the above example configuration, the tags will be set in the following way The `forwarded` tag is always appended and the `generic` tag in this example comes from the dataset. -[NOTE] -==== -- Tags must be defined within `inputs` in the `config.yaml` file. -- Each tag must be a string and added to the list. -==== +::::{note} +* Tags must be defined within `inputs` in the `config.yaml` file. +* Each tag must be a string and added to the list. + +:::: + + -[discrete] -[[aws-serverless-define-include-exclude-filters]] -=== Define include/exclude filters +### Define include/exclude filters [aws-serverless-define-include-exclude-filters] You can define multiple filters for inputs to include or exclude events from data ingestion. -[source, yaml] ----- +```yaml inputs: - type: "s3-sqs" id: "arn:aws:sqs:%REGION%:%ACCOUNT%:%QUEUENAME%" @@ -223,27 +195,26 @@ inputs: username: "arn:aws:secretsmanager:eu-west-1:123456789:secret:es_secrets:username" password: "arn:aws:secretsmanager:eu-west-1:123456789:secret:es_secrets:password" es_datastream_name: "logs-generic-default" ----- +``` You can define a list of regular expressions within `inputs.[].include`. If this list is populated, only messages **matching any** of the defined regular expressions will be forwarded to the outputs. You can define a list of regular expressions within `inputs.[].exclude`. If this list is populated, only messages **not matching any** of the defined regular expressions will be forwarded to the outputs i.e. every message will be forwarded to the outputs unless it matches any of the defined regular expressions. -[NOTE] -==== +::::{note} Both config parameters are optional, and can be set independently of each other. In terms of rule precedence, the exclude filter is applied first and then the include filter, so exclude takes precedence if both are specified. -All regular expressions are case-sensitive and should follow https://docs.python.org/3.9/library/re.html#regular-expression-syntax[Python's 3.9 regular expression syntax]. +All regular expressions are case-sensitive and should follow [Python’s 3.9 regular expression syntax](https://docs.python.org/3.9/library/re.md#regular-expression-syntax). Messages are scanned for terms that match the defined filters. Use the `^` (caret) special character to explicitly anchor the regex to the position before the first character of the string, and use `$` to anchor at the end. -No flags are used when the regular expression is compiled. Please refer to https://docs.python.org/3.9/library/re.html#re.compile[inline flag documentation] for alternative options for multiline, case-insensitive, and other matching behaviors. +No flags are used when the regular expression is compiled. Please refer to [inline flag documentation](https://docs.python.org/3.9/library/re.md#re.compile) for alternative options for multiline, case-insensitive, and other matching behaviors. -==== +:::: -[discrete] -[[aws-serverless-json-content]] -== JSON content discovery + + +## JSON content discovery [aws-serverless-json-content] The Elastic Serverless Forwarder is able to automatically discover JSON content in the payload of an input and collect the JSON objects contained in the payload. @@ -257,22 +228,26 @@ Where content is known to be plain text, you can improve overall performance by To change this configuration option, set `inputs.[].json_content_type` to one of the following values: -- **single**: indicates that the content of a single item in the input payload is a single JSON object. The content can either be on a single line or spanning multiple lines. With this setting the whole content of the payload is treated as a JSON object, with no limit on the number of lines the JSON object spans. -- **ndjson**: indicates that the content of a single item in the input payload is a valid NDJSON format. Multiple single JSON objects formatted on a single line should be separated by a newline delimiter. With this setting each line will be treated as a JSON object, which improves the parsing performance. -- **disabled**: instructs the forwarder not to attempt any automatic JSON content discovery and instead treat the content as plain text, which improves the parsing performance. +* **single**: indicates that the content of a single item in the input payload is a single JSON object. The content can either be on a single line or spanning multiple lines. With this setting the whole content of the payload is treated as a JSON object, with no limit on the number of lines the JSON object spans. +* **ndjson**: indicates that the content of a single item in the input payload is a valid NDJSON format. Multiple single JSON objects formatted on a single line should be separated by a newline delimiter. With this setting each line will be treated as a JSON object, which improves the parsing performance. +* **disabled**: instructs the forwarder not to attempt any automatic JSON content discovery and instead treat the content as plain text, which improves the parsing performance. + +::::{note} +JSON content is still stored in Elasticsearch as field type `text`. No automatic JSON expansion is performed by the forwarder; this can be achieved using the [JSON processor](elasticsearch://docs/reference/ingestion-tools/enrich-processor/json-processor.md) in an ingest pipeline in Elasticsearch. +:::: -NOTE: JSON content is still stored in Elasticsearch as field type `text`. No automatic JSON expansion is performed by the forwarder; this can be achieved using the https://www.elastic.co/guide/en/elasticsearch/reference/current/json-processor.html[JSON processor] in an ingest pipeline in Elasticsearch. -NOTE: There is no need to configure the JSON content type when <>, unless you have single JSON objects that span more than 1000 lines. +::::{note} +There is no need to configure the JSON content type when [Expanding events from JSON object lists](#expanding-events-from-json-object-lists), unless you have single JSON objects that span more than 1000 lines. +:::: -[discrete] -[[expanding-events-from-json-object-lists]] -=== Expanding events from JSON object lists + + +### Expanding events from JSON object lists [expanding-events-from-json-object-lists] You can extract a list of events to be ingested from a specific field in the JSON file. -[source, yaml] ----- +```yaml inputs: - type: "s3-sqs" id: "arn:aws:sqs:%REGION%:%ACCOUNT%:%QUEUENAME%" @@ -286,94 +261,85 @@ inputs: username: "arn:aws:secretsmanager:eu-west-1:123456789:secret:es_secrets:username" password: "arn:aws:secretsmanager:eu-west-1:123456789:secret:es_secrets:password" es_datastream_name: "logs-generic-default" ----- +``` You can define `inputs.[].expand_event_list_from_field` as a string with the value of a key in the JSON that contains a list of elements that must be sent as events instead of the encompassing JSON. To inject root fields from the JSON object into the expanded events, define `inputs.[].root_fields_to_add_to_expanded_event`. The config takes one of the following values: * the literal string "**all**" to inject all root fields (except the field you are expanding events from). For example, `root_fields_to_add_to_expanded_event: "all"` -* a list of the root fields that you want to inject. For example, `root_fields_to_add_to_expanded_event: ["owner", "logGroup", "logStream"]` +* a list of the root fields that you want to inject. For example, `root_fields_to_add_to_expanded_event: ["owner", "logGroup", "logStream"]` +::::{note} +When [routing service logs](#aws-serverless-route-service-logs), any value set for the `expand_event_list_from_field` configuration parameter will be ignored, because this will be automatically handled by the Elastic Serverless Forwarder. +:::: -NOTE: When <>, any value set for the `expand_event_list_from_field` configuration parameter will be ignored, because this will be automatically handled by the Elastic Serverless Forwarder. -[discrete] -=== Example without `root_fields_to_add_to_expanded_event` +### Example without `root_fields_to_add_to_expanded_event` [_example_without_root_fields_to_add_to_expanded_event] With the following input: -[source, json] ----- +```json {"Records":[{"key": "value #1"},{"key": "value #2"}]} {"Records":[{"key": "value #3"},{"key": "value #4"}]} ----- +``` Without setting `expand_event_list_from_field`, two events will be forwarded: -[source, json] ----- +```json {"@timestamp": "2022-06-16T04:06:03.064Z", "message": "{\"Records\":[{\"key\": \"value #1\"},{\"key\": \"value #2\"}]}"} {"@timestamp": "2022-06-16T04:06:13.888Z", "message": "{\"Records\":[{\"key\": \"value #3\"},{\"key\": \"value #4\"}]}"} ----- +``` If `expand_event_list_from_field` is set to `Records`, four events will be forwarded: -[source, json] ----- +```json {"@timestamp": "2022-06-16T04:06:21.105Z", "message": "{\"key\": \"value #1\"}"} {"@timestamp": "2022-06-16T04:06:27.204Z", "message": "{\"key\": \"value #2\"}"} {"@timestamp": "2022-06-16T04:06:31.154Z", "message": "{\"key\": \"value #3\"}"} {"@timestamp": "2022-06-16T04:06:36.189Z", "message": "{\"key\": \"value #4\"}"} ----- +``` + -[discrete] -=== Example with `root_fields_to_add_to_expanded_event` +### Example with `root_fields_to_add_to_expanded_event` [_example_with_root_fields_to_add_to_expanded_event] With the following input: -[source, json] ----- +```json {"Records":[{"key": "value #1"},{"key": "value #2"}], "field1": "value 1a", "field2": "value 2a"} {"Records":[{"key": "value #3"},{"key": "value #4"}], "field1": "value 1b", "field2": "value 2b"} ----- +``` If `expand_event_list_from_field` is set to `Records`, and `root_fields_to_add_to_expanded_event` to `all`, four events will be forwarded: -[source, json] ----- +```json {"@timestamp": "2022-06-16T04:06:21.105Z", "message": "{\"key\": \"value #1\", \"field1\": \"value 1a\", \"field2\": \"value 2a\""} {"@timestamp": "2022-06-16T04:06:27.204Z", "message": "{\"key\": \"value #2\", \"field1\": \"value 1a\", \"field2\": \"value 2a\""} {"@timestamp": "2022-06-16T04:06:31.154Z", "message": "{\"key\": \"value #3\", \"field1\": \"value 1b\", \"field2\": \"value 2b\""} {"@timestamp": "2022-06-16T04:06:36.189Z", "message": "{\"key\": \"value #4\", \"field1\": \"value 1b\", \"field2\": \"value 2b\""} ----- +``` If `expand_event_list_from_field` is set to `Records`, and `root_fields_to_add_to_expanded_event` to `["field1"]`, four events will be forwarded: -[source, json] ----- +```json {"@timestamp": "2022-06-16T04:06:21.105Z", "message": "{\"key\": \"value #1\", \"field1\": \"value 1a\""} {"@timestamp": "2022-06-16T04:06:27.204Z", "message": "{\"key\": \"value #2\", \"field1\": \"value 1a\""} {"@timestamp": "2022-06-16T04:06:31.154Z", "message": "{\"key\": \"value #3\", \"field1\": \"value 1b\""} {"@timestamp": "2022-06-16T04:06:36.189Z", "message": "{\"key\": \"value #4\", \"field1\": \"value 1b\""} ----- +``` -[discrete] -[[aws-serverless-manage-multiline-messages]] -== Manage multiline messages +## Manage multiline messages [aws-serverless-manage-multiline-messages] Forwarded content may contain messages that span multiple lines of text. For example, multiline messages are common in files that contain Java stack traces. In order to correctly handle these multiline events, you need to configure `multiline` settings for a specific input to specify which lines are part of a single event. -[discrete] -[[aws-serverless-multiline-config]] -=== Configuration options + +### Configuration options [aws-serverless-multiline-config] You can specify the following options for a specific input in the `config.yaml` file to control how the Elastic Serverless Forwarder deals with messages that span multiple lines. -[source, yaml] ----- +```yaml inputs: - type: "s3-sqs" id: "arn:aws:sqs:%REGION%:%ACCOUNT%:%QUEUENAME%" @@ -389,39 +355,43 @@ inputs: username: "arn:aws:secretsmanager:eu-west-1:123456789:secret:es_secrets:username" password: "arn:aws:secretsmanager:eu-west-1:123456789:secret:es_secrets:password" es_datastream_name: "logs-generic-default" ----- +``` The forwarder takes all the lines that do not start with `[` and combines them with the previous line that does. For example, you could use this configuration to join the following lines of a multiline message into a single event: -[source, shell] ----- +```shell [beat-logstash-some-name-832-2015.11.28] IndexNotFoundException[no such index] at org.elasticsearch.cluster.metadata.IndexNameExpressionResolver$WildcardExpressionResolver.resolve(IndexNameExpressionResolver.java:566) at org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.concreteIndices(IndexNameExpressionResolver.java:133) at org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.concreteIndices(IndexNameExpressionResolver.java:77) at org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction.checkBlock(TransportDeleteIndexAction.java:75) ----- +``` + +::::{note} +Note that you should escape the opening square bracket (`[`) in the regular expression, because it specifies a character class i.e. a set of characters that you wish to match. You also have to escape the backslash (`\`) used for escaping the opening square bracket as raw strings are not used. Thus, `^\\[` will produce the required regular expression upon compiling. +:::: -NOTE: Note that you should escape the opening square bracket (`[`) in the regular expression, because it specifies a character class i.e. a set of characters that you wish to match. You also have to escape the backslash (`\`) used for escaping the opening square bracket as raw strings are not used. Thus, `^\\[` will produce the required regular expression upon compiling. `inputs.[].multiline.type` defines which aggregation method to use. The default is `pattern`. The other options are `count`, which enables you to aggregate a constant number of lines, and `while_pattern`, which aggregates lines by pattern without matching options. -`inputs.[].multiline.pattern` differs from the patterns supported by {ls}. See https://docs.python.org/3.9/library/re.html#regular-expression-syntax[Python's 3.9 regular expression syntax] for a list of supported regexp patterns. Depending on how you configure other multiline options, lines that match the specified regular expression are considered either continuations of a previous line or the start of a new multiline event. +`inputs.[].multiline.pattern` differs from the patterns supported by {{ls}}. See [Python’s 3.9 regular expression syntax](https://docs.python.org/3.9/library/re.md#regular-expression-syntax) for a list of supported regexp patterns. Depending on how you configure other multiline options, lines that match the specified regular expression are considered either continuations of a previous line or the start of a new multiline event. `inputs.[].multiline.negate` defines whether the pattern is negated. The default is `false`. This setting works only with `pattern` and `while_pattern` types. `inputs.[].multiline.match` changes the grouping of multiple lines according to the schema below (works only with `pattern` type): -//[cols="2*/dev/null | openssl x509 -noout -fingerprint SHA1 Fingerprint=1C:46:32:75:AA:D6:F1:E2:8E:10:A3:64:44:B1:36:C9:7D:44:35:B4 ----- +``` You can use your DNS name, IP address, and port number instead of `my-deployment.es.eastus2.azure.elastic-cloud.com:443` from the above example. Copy your fingerprint value for the next step. -[discrete] -[[aaws-serverless-manage-self-signed-certificates-set-ssl-fingerprint]] -=== Set the SSL fingerprint + +### Set the SSL fingerprint [aaws-serverless-manage-self-signed-certificates-set-ssl-fingerprint] As a final step, edit your `config.yml` file to use the SSL fingerprint: -[source, yaml] ----- +```yaml inputs: - type: "s3-sqs" id: "arn:aws:sqs:eu-west-1:123456789:dev-access-logs" @@ -681,4 +635,4 @@ inputs: batch_max_actions: 500 batch_max_bytes: 10485760 ssl_assert_fingerprint: "1C:46:32:75:AA:D6:F1:E2:8E:10:A3:64:44:B1:36:C9:7D:44:35:B4" ----- +``` diff --git a/docs/reference/index.md b/docs/reference/index.md new file mode 100644 index 00000000..6a2954c7 --- /dev/null +++ b/docs/reference/index.md @@ -0,0 +1,149 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/esf/current/index.html + - https://www.elastic.co/guide/en/esf/current/aws-elastic-serverless-forwarder.html +--- + +# Elastic Serverless Forwarder for AWS [aws-elastic-serverless-forwarder] + +The Elastic Serverless Forwarder is an Amazon Web Services (AWS) Lambda function that ships logs from your AWS environment to Elastic. + +The Elastic Serverless Forwarder works with {{stack}} 7.17 and later. + +::::{important} +Using Elastic Serverless Forwarder may result in additional charges. To learn how to minimize additional charges, refer to [Preventing unexpected costs](docs-content://troubleshoot/deployments/esf/elastic-serverless-forwarder.md#preventing-unexpected-costs). +:::: + + + +## Overview [aws-serverless-forwarder-overview] + +The Elastic Serverless Forwarder can forward AWS data to cloud-hosted, self-managed Elastic environments, or [preview]{{ls}}. It supports the following inputs: + +* Amazon S3 (via SQS event notifications) +* Amazon Kinesis Data Streams +* Amazon CloudWatch Logs subscription filters +* Amazon SQS message payload + +:::{image} ../images/aws-serverless-lambda-flow.png +:alt: AWS Lambda flow +:class: screenshot +::: + +Elastic Serverless Forwarder ensures [at-least-once delivery](#aws-serverless-forwarder-at-least-once-delivery) of the forwarded message. + +When you successfully deploy the forwarder, an SQS *continuing queue* is automatically created in Lambda to ensure no data is lost. By default, the forwarder runs for a maximum of 15 minutes, so it’s possible that AWS may exit the function in the middle of processing event data. The forwarder handles this scenario by keeping track of the last offset processed. When the queue triggers a new function invocation, the forwarder will start where the last function run stopped. + +The forwarder uses a *replay queue* (also automatically created during deployment) to handle any ingestion-related exception or fail scenarios. Data in the replay queue is stored as individual events. Lambda keeps track of any failed events and writes them to the replay queue that can then be consumed by adding it as an additional SQS trigger via Lambda. + +You can use the [config.yaml](/reference/aws-deploy-elastic-serverless-forwarder.md#sample-s3-config-file) file to configure the service for each input and output type, including information such as SQS queue ARN (Amazon Resource Number) and {{es}} or {{ls}} connection details. You can create multiple input sections within the configuration file to map different inputs to specific log types. + +There is no need to define a specific input in the [config.yaml](/reference/aws-deploy-elastic-serverless-forwarder.md#sample-s3-config-file) for the continuing queue and the replay queue. + +The forwarder also supports writing directly to an index, alias, or custom data stream. This enables existing {{es}} users to re-use index templates, ingest pipelines, or dashboards that are already created and connected to other processes. + + +## Inputs [aws-serverless-forwarder-inputs] + + +### Amazon S3 (via SQS event notifications) [aws-serverless-forwarder-inputs-s3] + +The forwarder can ingest logs contained in an Amazon Simple Storage Service (S3) bucket through a Simple Queue Service (SQS) notification (`s3:ObjectCreated`) and send them to Elastic. The SQS queue serves as a trigger for the forwarder. When a new log file is written to an S3 bucket and meets the user-defined criteria (including prefix/suffix), an SQS notification is generated that triggers the Lambda function. + +You can set up separate SQS queues for each type of log (for example, `aws.vpcflow`, `aws.cloudtrail`, `aws.waf`). A single configuration file can have many input sections, pointing to different SQS queues that match specific log types. The `es_datastream_name` parameter in the config file is optional. The forwarder supports automatic routing of various AWS service logs to the corresponding data streams for further processing and storage in the {{es}} cluster. It supports automatic routing of `aws.cloudtrail`, `aws.cloudwatch_logs`, `aws.elb_logs`, `aws.firewall_logs`, `aws.vpcflow`, and `aws.waf` logs. + +For other log types, you can optionally set the `es_datastream_name` value in the configuration file according to the naming convention of the {{es}} data stream and integration. If the `es_datastream_name` is not specified, and the log cannot be matched with any of the above AWS services, then the dataset will be set to `generic` and the namespace set to `default`, pointing to the data stream name `logs-generic-default`. + +For more information on creating SQS event notifications for S3 buckets, read the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ways-to-add-notification-config-to-bucket.md). + +::::{note} +You must set a visibility timeout of `910` seconds for any SQS queues you want to use as a trigger. This is 10 seconds greater than the Elastic Serverless Forwarder Lambda timeout. +:::: + + + +### Amazon Kinesis Data Streams [aws-serverless-forwarder-inputs-kinesis] + +The forwarder can ingest logs contained in the payload of a Kinesis Data Stream record and send them to Elastic. The Kinesis Data Stream serves as a trigger for the forwarder. When a new record gets written to a Kinesis Data Stream, it triggers the Lambda function. + +You can set up separate Kinesis Data Streams for each type of log. The `es_datastream_name` parameter in the config file is mandatory. If this value is set to an {{es}} data stream, the type of log must be correctly defined with configuration parameters. A single configuration file can have many input sections, pointing to different data streams that match specific log types. + + +### Amazon CloudWatch Logs subscription filters [aws-serverless-forwarder-inputs-cloudwatch] + +The forwarder can ingest logs contained in the message payload of CloudWatch Logs events and send them to Elastic. The CloudWatch Logs service serves as a trigger for the forwarder. When a new event gets written to a CloudWatch Logs log stream, it triggers the Lambda function. + +You can set up separate CloudWatch Logs groups for each type of log. The `es_datastream_name` parameter in the config file is mandatory. If this value is set to an {{es}} data stream, the type of log must be correctly defined with configuration parameters. A single configuration file can have many input sections, pointing to different CloudWatch Logs groups that match specific log types. + + +### Amazon SQS message payload [aws-serverless-forwarder-inputs-direct] + +The forwarder can ingest logs contained within the payload of an Amazon SQS body record and send them to Elastic. The SQS queue serves as a trigger for the forwarder. When a new record gets written to an SQS queue, the Lambda function triggers. + +You can set up a separate SQS queue for each type of log. The config parameter for {{es}} output `es_datastream_name` is mandatory. If this value is set to an {{es}} data stream, the type of log must be correctly defined with configuration parameters. A single configuration file can have many input sections, pointing to different SQS queues that match specific log types. + + +## At-least-once delivery [aws-serverless-forwarder-at-least-once-delivery] + +The Elastic Serverless Forwarder ensures at-least-once delivery of the forwarded messages by using the continuing queue and replay queue. + + +### Continuing queue [aws-serverless-forwarder-at-least-once-delivery-continuing-queue] + +The Elastic Serverless Forwarder can run for a maximum amount of time of 15 minutes. Different inputs can trigger the Elastic Serverless Forwarder, with different payload sizes for each execution trigger. The size of the payload impacts the number of events to be forwarded. Configuration settings may also impact the number of events, such as [defining include/exclude filters](/reference/aws-elastic-serverless-forwarder-configuration.md#aws-serverless-define-include-exclude-filters), [expanding events from JSON object lists](/reference/aws-elastic-serverless-forwarder-configuration.md#expanding-events-from-json-object-lists), and [managing multiline messages](/reference/aws-elastic-serverless-forwarder-configuration.md#aws-serverless-manage-multiline-messages). + +The continuing queue helps ensure at-least-once delivery when the maximum amount of time (15 minutes) is not sufficient to forward all the events resulting from a single execution of the Elastic Serverless Forwarder. + +For this scenario, a grace period of two minutes is reserved at the end of the 15 minute timeout to handle any remaining events that have not been processed. At the beginning of this grace period, event forwarding is halted. The remaining time is dedicated to sending a copy of the original messages that contain the remaining events to the continuing queue. This mechanism removes the need to handle partial processing of the trigger at the input level, which is not always possible (for example in the case of [Amazon CloudWatch Logs subscription filters](#aws-serverless-forwarder-inputs-cloudwatch)) or desirable because it forces users to conform to a specific configuration of the AWS resources used as inputs. + +Each message in the continuing queue contains metadata related to the last offset processed and a reference to the original input. + +::::{note} +You can remove a specific input as a trigger of the Elastic Serverless Forwarder. However, before removing its definition from the [config.yaml](/reference/aws-deploy-elastic-serverless-forwarder.md#sample-s3-config-file), ensure that all the events generated while the input was still a trigger are fully processed, including the ones in the messages copied to the continuing queue. The handling of the messages in the continuing queue requires a lookup of the original input in the `config.yml`. +:::: + + +In the unlikely scenario that the Elastic Serverless Forwarder exceeds its maximum allocated execution time and is forcefully terminated, the continuing queue will not be properly populated with a copy of the messages left to be processed. In this scenario, all or a portion of the messages might be lost depending on the specific AWS resource used as input and its configuration. + +An AWS SQS [Dead Letter Queue](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-dead-letter-queues.md) is created for the continuing queue. + +When the Elastic Serverless Forwarder is triggered by the continuing queue, in the unlikely scenario that it exceeds its maximum allocated execution time and is forcefully terminated, the messages in the payload that triggered the Elastic Serverless Forwarder execution are not deleted from the continuing queue and another Elastic Serverless Forwarder execution is triggered. The continuing queue is configured for a number of 3 maximum receives before a message is sent to the DLQ. + + +### Replay queue [aws-serverless-forwarder-at-least-once-delivery-replay-queue] + +The Elastic Serverless Forwarder forwards events to the outputs defined for a specific input. Events to be forwarded are grouped in batches that can be configured according to the specific output. Failures can happen when forwarding events to an output. Depending on the output type, the granularity of the failure can either be for the whole batch of events, or for single events in the batch. There are multiple reasons for a failure to happen, including, but not limited to, network connectivity or the output service being unavailable or under stress. + +The replay queue helps ensure at-least-once delivery when a failure in forwarding an event happens. + +For this scenario, after a batch of events is forwarded, a copy of all the events in the batch that failed to be forwarded is sent to the replay queue. Each message sent to the replay queue contains exactly one event that failed to be forwarded. + +It is possible to enable the replay queue as a trigger of the Elastic Serverless Forwarder in order to forward the events in the queue again. + +::::{note} +Before enabling or disabling the replay queue as a trigger of the Elastic Serverless Forwarder, consider the specific reason why the forwarding failures occurred. In most cases, you should resolve the underlying issue causing the failures before trying to forward the events in the queue again. Depending on the nature and impact of the issue, forwarding the events again without solving the problem may produce new failures and events going back to the replay queue. In some scenarios, like the output service being under stress, it is recommended that you disable the replay queue as a trigger of the Elastic Serverless Forwarder, since continuing to forward the events could worsen the issue. +:::: + + +When the Elastic Serverless Forwarder is triggered by the replay queue and all events are successfully forwarded, the Elastic Serverless Forwarded execution succeeds, and the messages in the trigger payload are removed automatically from the replay queue. + +However, if any events fail again to be forwarded, all messages in the trigger payload that contain successful events are deleted, and a specific expected exception is raised. The Elastic Serverless Forwarder execution is marked as failed, and any failed messages are sent back to the replay queue. + +The messages in the replay queue contain metadata with references to the original input and the original output of the events. + +::::{note} +You can remove a specific input as a trigger of the Elastic Serverless Forwarder. However, before removing its definition from [config.yaml](/reference/aws-deploy-elastic-serverless-forwarder.md#sample-s3-config-file), ensure that all the events that failed to be ingested while the input was still a trigger are fully processed. The handling of the messages in the replay queue requires a lookup of the original input and output in the `config.yml`. +:::: + + +An AWS SQS [Dead Letter Queue (DLQ)](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-dead-letter-queues.md) is created for the replay queue. + +The same message can go back to the replay queue up to three times. After reaching the configured number of 3 maximum receives, the message will be sent to the DLQ. The same message can go back to the replay queue either because it contains an event that failed again to be forwarded, according to the planned design, or in the unlikely scenario that the Elastic Serverless Forwarder triggered by the queue exceeds its maximum allocated execution time and is forcefully terminated. In this scenario the messages will not be lost and will eventually be sent to the DQL. + + +## Get started [aws-serverless-forwarder-get-started] + +* [Deploy Elastic Serverless Forwarder](/reference/aws-deploy-elastic-serverless-forwarder.md) +* [Configuration options](/reference/aws-elastic-serverless-forwarder-configuration.md) +* [Troubleshooting](docs-content://troubleshoot/deployments/esf/elastic-serverless-forwarder.md) + diff --git a/docs/reference/toc.yml b/docs/reference/toc.yml new file mode 100644 index 00000000..7f5eddbf --- /dev/null +++ b/docs/reference/toc.yml @@ -0,0 +1,8 @@ +project: 'Elastic Serverless Forwarder for AWS reference' +toc: + - file: index.md + - file: aws-deploy-elastic-serverless-forwarder.md + - file: aws-elastic-serverless-forwarder-configuration.md + # - file: aws-serverless-troubleshooting.md + # - file: deploy-elastic-serverless-forwarder.md + # - file: configuration-options-for-elastic-serverless-forwarder.md \ No newline at end of file