From 9b95153527457dc2afef4bc01569b0f5ab3e9e02 Mon Sep 17 00:00:00 2001 From: Forest Vey Date: Thu, 16 May 2024 09:53:00 -0700 Subject: [PATCH] Add Documentation for Generating Keys and Certificates for TLS Tests (#66) * Adding documentation for how to generate keys and certificates for TLS tests. Signed-off-by: forestmvey * Adding newline to gitignore file. Signed-off-by: forestmvey * Adding version file import to avoid hard-coded file version. Signed-off-by: forestmvey --------- Signed-off-by: forestmvey --- .gitignore | 5 ++ README.md | 109 +----------------------------------- integration/tls/README.md | 42 +++++++++++++- integration/tls/tls_test.go | 31 +++++++--- 4 files changed, 73 insertions(+), 114 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2140260 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +**/.DS_Store +*.tar.gz +*.zip +integration/tls/cert +resources diff --git a/README.md b/README.md index f71a785..0941297 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,6 @@ The Prometheus Connector receives and sends time series data between Prometheus - [Developer Documentation](#developer-documentation) - [Building the Prometheus Connector from Source](#building-the-prometheus-connector-from-source) - [Building the Docker Image](#building-the-docker-image) - - [Creating Self-signed TLS Certificates](#creating-self-signed-tls-certificates) - - [Creating the Certificate Authority files](#creating-the-certificate-authority-files) - - [Creating the Server Key and Server Certificate](#creating-the-server-key-and-server-certificate) - [Troubleshooting](#troubleshooting) - [Prometheus Connector Specific Errors](#prometheus-connector-specific-errors) - [Write API Errors](#write-api-errors) @@ -96,7 +93,7 @@ To configure Prometheus to read and write to remote storage, configure the `remo > **NOTE**: As a security best practice, it is recommended to regularly [rotate IAM user access keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_RotateAccessKey). -3. It is recommended to secure the Prometheus requests with TLS encryption. This can be achieved by specifying the certificate authority file in the `tls_config` section for Prometheus' remote read and remote write configuration. To generate self-signed certificates during development see the [Creating Self-signed TLS Certificates](#creating-self-signed-tls-certificates) section. +3. With the default configuration, Prometheus uses TLS version 1.2 for `remote_read` and `remote_write` requests. It is recommended to secure the Prometheus requests with mutual TLS encryption in a production environment. This can be achieved by specifying the certificate authority file in the `tls_config` section for Prometheus' remote read and remote write configuration. To generate self-signed certificates during development see the [Creating Self-signed TLS Certificates](integration/tls/README.md#creating-self-signed-tls-certificates) section. Here is an example of `remote_write` and `remote_read` configuration with TLS, where `RootCA.pem` is within the same directory as the Prometheus configuration file: @@ -155,7 +152,7 @@ It is recommended to secure the Prometheus requests with TLS encryption. To enab 2. Ensure the certificate authority file has been specified in the `tls_config` section within Prometheus' configuration file, see [Prometheus Configuration](#prometheus-configuration) for an example. -To generate self-signed certificates during development see [Creating Self-signed TLS Certificates](#creating-self-signed-tls-certificates). +To generate self-signed certificates during development see [Creating Self-signed TLS Certificates](integration/tls/README.md#creating-self-signed-tls-certificates). For more examples on configuring the Prometheus Connector see [Configuration Options](#configuration-options). @@ -231,7 +228,7 @@ It is recommended to secure the Prometheus requests with HTTPS with TLS encrypti 2. Ensure the certificate authority file has been specified in the `tls_config` section within Prometheus' configuration file, see [Prometheus Configuration](#prometheus-configuration) for an example. -To generate self-signed certificates during development see [Creating Self-signed TLS Certificates](#creating-self-signed-tls-certificates). +To generate self-signed certificates during development see [Creating Self-signed TLS Certificates](integration/tls/README.md#creating-self-signed-tls-certificates). To configure the `web.listen-address` option when running the Prometheus Connector through a Docker image, use the `-p` flag to expose the custom endpoint. The following example listens on the custom endpoint `localhost:3080`: @@ -766,106 +763,6 @@ User-Agent: Prometheus Connector/ aws-sdk-go/ (go; **NOTE**: Self-signed certificates **should not** be used during production, they should only be used during development. - -### Creating the Certificate Authority files - -Use the following command to generate a private key and the root certificate file for the certificate authority. - -```shell -openssl req -x509 -nodes -new -sha256 -days 365 -newkey rsa:2048 -keyout RootCA.key -out RootCA.pem -``` - -This command will prompt the user to enter some information for the certificate. An example of the output is as follows: - -```shell -certificates ❯ openssl req -x509 -nodes -new -sha256 -days 365 -newkey rsa:2048 -keyout RootCA.key -out RootCA.pem -Generating a 2048 bit RSA private key -............................+++ -................+++ -writing new private key to 'RootCA.key' ------ -You are about to be asked to enter information that will be incorporated -into your certificate request. -What you are about to enter is what is called a Distinguished Name or a DN. -There are quite a few fields but you can leave some blank -For some fields there will be a default value, -If you enter '.', the field will be left blank. ------ -Country Name (2 letter code) []:US -State or Province Name (full name) []:Washington -Locality Name (eg, city) []:Seattle -Organization Name (eg, company) []:Amazon Web Services -Organizational Unit Name (eg, section) []: -Common Name (eg, fully qualified host name) []:Private-Root-CA -Email Address []: -certificates ❯ -``` - -To provide all the information directly with the `-subj` flag. An example is as follows: - -```shell -openssl req -x509 -nodes -new -sha256 -days 365 -newkey rsa:2048 -keyout RootCA.key -out RootCA.pem -subj "/C=US/ST=Washington/L=Seattle/O=Amazon Web Services/CN=Private-Root-CA" -``` - -### Creating the Server Key and Server Certificate - -Use the following command to generate a server private key and a certificate signing request: - -```shell -openssl req -days 365 -nodes -newkey rsa:2048 -keyout serverPrivateKey.key -out serverCertificateSigningRequest.csr -``` - -This command will prompt the user to enter some information for the certificate. An example of the output is as follows: - -```shell -certificates ❯ openssl req -days 365 -nodes -newkey rsa:2048 -keyout serverPrivateKey.key -out serverCertificateSigningRequest.csr -Generating a 2048 bit RSA private key -....................................................................................................................................................................................................+++ -........................+++ -writing new private key to 'serverPrivateKey.key' ------ -You are about to be asked to enter information that will be incorporated -into your certificate request. -What you are about to enter is what is called a Distinguished Name or a DN. -There are quite a few fields but you can leave some blank -For some fields there will be a default value, -If you enter '.', the field will be left blank. ------ -Country Name (2 letter code) []:US -State or Province Name (full name) []:Washington -Locality Name (eg, city) []:Seattle -Organization Name (eg, company) []:Amazon Web Services -Organizational Unit Name (eg, section) []: -Common Name (eg, fully qualified host name) []:localhost -Email Address []: -certificates ❯ -``` - -To provide all the information directly with the `-subj` flag. An example is as follows: - -```shell -openssl req -days 365 -nodes -newkey rsa:2048 -keyout serverPrivateKey.key -out serverCertificateSigningRequest.csr -subj "/C=US/ST=Washington/L=Seattle/O=Amazon Web Services/CN=localhost" -``` - -To associate the host name to the server certificate, create a `domain.ext` file with the following content: -``` -subjectAltName = DNS:localhost -``` -Store the file at the same location as the `serverCertificateSigningRequest.csr`. This `domain.ext` will be used when generating the self-signed server certificate. - -> **NOTE**: The value for DNS is set to **localhost**. This is required when running the Prometheus Connector from a Docker image or from the precompiled binaries. - -Use the following command to generate the self-signed server certificate: - -```shell -openssl x509 -req -sha256 -days 365 -in serverCertificateSigningRequest.csr -out serverCertificate.crt -CA RootCA.pem -CAkey RootCA.key -CAcreateserial -extfile domain.ext -``` - # Troubleshooting ## Information Logs diff --git a/integration/tls/README.md b/integration/tls/README.md index 3cf3ee9..e3c9180 100644 --- a/integration/tls/README.md +++ b/integration/tls/README.md @@ -3,7 +3,8 @@ ## Prerequisites Prior to running the tests in tls_test.go, ensure the following: 1. Updated the basic_auth section within [prometheus.yml](./config/prometheus.yml). -2. Download or build the Prometheus Connector Docker image and store it in a new directory named `resources` in the repository root. +2. Complete the [Creating Self-signed TLS Certificates](#creating-self-signed-tls-certificates) if this has not been previously done and store the `RootCA.pem`, `ServerCertificate.crt`, `ServerPrivateKey.key`, and `InvalidPrivateKey.key` files in the `integration/tls/cert` folder. +3. Download the [latest release artifact](https://github.com/awslabs/amazon-timestream-connector-prometheus/releases/latest) or build the Prometheus Connector Docker image and store it in a new directory named `resources` in the repository root. ## How to build and save the docker image 1. Execute the following command to build the docker image: @@ -11,6 +12,45 @@ Prior to running the tests in tls_test.go, ensure the following: 2. Execute the following command to save the docker image as a compressed file and update the `version` appropriately: `docker save timestream-prometheus-connector-docker | gzip > timestream-prometheus-connector-docker-image-.tar.gz` +## Creating Self-signed TLS Certificates + +The following steps generate self-signed TLS certificates using OpenSSL. + +> **NOTE**: Self-signed certificates **should not** be used during production, they should only be used during development. + +### Creating the Certificate Authority files + +Use the following command to generate a private key and the root certificate file for the certificate authority. + +```shell +openssl req -x509 -nodes -new -sha256 -days 1024 -newkey rsa:2048 -keyout RootCA.key -out RootCA.pem -subj "/C=US/ST=Washington/L=Seattle/O=Amazon Web Services/CN=host.docker.internal" +``` + +### Creating the Server Key + +Use the following command to generate a server private key and a certificate signing request: + +```shell +openssl req -days 365 -nodes -newkey rsa:2048 -keyout ServerPrivateKey.key -out ServerCertificateSigningRequest.csr -subj "/C=US/ST=Washington/L=Seattle/O=Amazon Web Services/CN=host.docker.internal" +``` + +### Creating the Server Certificate + +Use the following command to generate the self-signed server certificate: + +```shell +openssl x509 -req -sha256 -days 365 -in ServerCertificateSigningRequest.csr -CA RootCA.pem -CAkey RootCA.key -CAcreateserial -extfile <(printf "subjectAltName=DNS:host.docker.internal") -out ServerCertificate.crt +``` +> **NOTE**: The value for DNS is set to **DNS:host.docker.internal** to associate the host name to the server certificate. This is required when running the Prometheus Connector from a Docker image or from the precompiled binaries. + +### Creating the Invalid Private Key + +Use the following command to generate the invalid private key: + +```shell +openssl req -x509 -nodes -new -sha256 -days 365 -newkey rsa:2048 -keyout InvalidPrivateKey.key -subj "/C=US/ST=Washington/L=Seattle/O=Invalid Organization/CN=Invalid-CN" +``` + ## How to execute tests 1. Run the following command to execute the TLS tests: `go test -v ./integration/tls` diff --git a/integration/tls/tls_test.go b/integration/tls/tls_test.go index 0fdaee0..1d5ad94 100644 --- a/integration/tls/tls_test.go +++ b/integration/tls/tls_test.go @@ -33,6 +33,7 @@ import ( "testing" "time" "timestream-prometheus-connector/integration" + "timestream-prometheus-connector/timestream" ) const ( @@ -40,15 +41,17 @@ const ( prometheusConfigPath = "config/prometheus.yml" prometheusDockerImageName = "prom/prometheus" tlsRootCAPath = "cert/RootCA.pem" + tlsCertificatePath = "cert/ServerCertificate.crt" + tlsPrivateKeyPath = "cert/ServerPrivateKey.key" tlsServerCertPath = "cert" connectorDockerImageName = "timestream-prometheus-connector-docker" - connectorDockerImagePath = "../../resources/timestream-prometheus-connector-docker-image-1.0.0.tar.gz" defaultDatabaseCMD = "--default-database=tlsDB" defaultTableCMD = "--default-table=tls" - tlsCertificateCMD = "--tls-certificate=/root/cert/serverCertificate.crt" - tlsKeyCMD = "--tls-key=/root/cert/serverPrivateKey.key" - tlsUnmatchedCertificateCMD = "--tls-certificate=/root/cert/rootCA.pem" - tlsInvalidKeyFileCMD = "--tls-key=/root/cert/invalidPrivateKey.key" + tlsCertificateCMD = "--tls-certificate=/root/cert/ServerCertificate.crt" + tlsKeyCMD = "--tls-key=/root/cert/ServerPrivateKey.key" + tlsUnmatchedCertificateCMD = "--tls-certificate=/root/cert/RootCA.pem" + tlsInvalidKeyFileCMD = "--tls-key=/root/cert/InvalidPrivateKey.key" + tlsInvalidKeyPath = "cert/InvalidPrivateKey.key" region = "us-east-1" database = "tlsDB" table = "tls" @@ -78,12 +81,17 @@ func TestMain(m *testing.M) { } func TestHttpsSupport(t *testing.T) { + // Ensure required testing files exists + validateFileExists(t, tlsRootCAPath) + validateFileExists(t, tlsCertificatePath) + validateFileExists(t, tlsPrivateKeyPath) + dockerClient, ctx := integration.CreateDockerClient(t) bindString := []string{fmt.Sprintf("%s:/root/cert:ro", getAbsolutionPath(t, tlsServerCertPath))} connectorConfig := integration.ConnectorContainerConfig{ - DockerImage: connectorDockerImagePath, + DockerImage: "../../resources/timestream-prometheus-connector-docker-image-" + timestream.Version + ".tar.gz", ImageName: connectorDockerImageName, Binds: bindString, ConnectorCommands: connectorTLSCMDs, @@ -129,12 +137,15 @@ func TestHttpsSupportWithInvalidCertificate(t *testing.T) { {"test with invalid file type", connectorCMDsWithInvalidFile}, } + // Ensure invalid key file exists + validateFileExists(t, tlsInvalidKeyPath) + bindString := []string{fmt.Sprintf("%s:/root/cert:ro", getAbsolutionPath(t, tlsServerCertPath))} dockerClient, ctx := integration.CreateDockerClient(t) for _, test := range invalidTestCase { connectorConfig := integration.ConnectorContainerConfig{ - DockerImage: connectorDockerImagePath, + DockerImage: "../../resources/timestream-prometheus-connector-docker-image-" + timestream.Version + ".tar.gz", ImageName: connectorDockerImageName, Binds: bindString, ConnectorCommands: test.command, @@ -150,6 +161,12 @@ func TestHttpsSupportWithInvalidCertificate(t *testing.T) { integration.StopContainer(t, dockerClient, ctx, containerIDs) } +// Check wether a file exists. +func validateFileExists(t *testing.T, path string) { + _, err := os.Stat(path) + require.NoError(t, err) +} + // getAbsolutionPath gets the absolution path of the giving file. func getAbsolutionPath(t *testing.T, path string) string { absPath, err := filepath.Abs(path)