Bucket path style (#99)

* Remove path-style feature, add a bucket flag

* Add configuration methods, closes #96

* Add BTC and ETH addresses

* Allow specifying credentials env vars

* Run simple_crud with tests

* Add secrets to actions

Author: durch

.github/workflows/rust.yml

@@ -2,18 +2,22 @@ name: build
 
 on:
   push:
-    branches: [ master ]
+    branches: [master]
   pull_request:
-    branches: [ master ]
+    branches: [master]
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Build
-      run: cargo build --verbose
-    - name: Run tests
-      run: cargo test --verbose
+      - uses: actions/checkout@v2
+      - name: Build
+        run: cargo build --verbose
+      - name: Run tests
+        run: cargo test --verbose
+      - name: Run simple_crud
+        env:
+          EU_AWS_ACCESS_KEY_ID: ${{ secrets.EU_AWS_ACCESS_KEY_ID }}
+          EU_AWS_SECRET_ACCESS_KEY: ${{ secrets.EU_AWS_SECRET_ACCESS_KEY }}
+        run: cargo run --bin simple_crud

README.md

@@ -8,6 +8,11 @@
 
 Rust library for working with Amazon S3 or arbitrary S3 compatible APIs, fully compatible with **async/await** and `futures ^0.3`
 
+### Support further development
+
++ BTC - `3QQdtQGSMStTWEBhe65hPiAWJekXH8n26o`
++ ETH - `0x369Fd06ACc25CCfE0A28BE40018cF3aC38AcdcB6`
+
 ### Intro
 
 Modest interface towards Amazon S3, as well as S3 compatible object storage APIs such as Wasabi, Yandex or Minio.
@@ -18,6 +23,10 @@ a `PUT` presigned URL, meaning they can upload to a specific key in S3 for the d
 
 **[AWS, Yandex and Custom (Minio) Example](https://github.com/durch/rust-s3/blob/master/s3/bin/simple_crud.rs)**
 
+#### Path or subdomain style URLs and headers
+
+`Bucket` struct provides constructors for `path-style` paths, `subdomain` style is the default. `Bucket` exposes methods for configuring and accessing `path-style` configuration.
+
 #### Presign
 
 |       |                                                                                                |
@@ -85,26 +94,30 @@ while `tokio` methods are generic over `tokio::io::AsyncReadExt`.
 
 ```toml
 [dependencies]
-rust-s3 = "0.22.8"
+rust-s3 = "0.22.11"
 ```
 
-#### Disable SSL verification for endpoints, useful for custom regions
+#### Features
+
+##### Disable SSL verification for endpoints, useful for custom regions
 
 ```toml
 [dependencies]
-rust-s3 = {version = "0.22.8", features = ["no-verify-ssl"]}
+rust-s3 = {version = "0.22.11", features = ["no-verify-ssl"]}
 ```
 
-#### Fail on HTTP error responses
+##### Fail on HTTP error responses
 
 ```toml
 [dependencies]
-rust-s3 = {version = "0.22.8", features = ["fail-on-err"]}
+rust-s3 = {version = "0.22.11", features = ["fail-on-err"]}
 ```
 
-#### Use path style addressing, needed for Minio compatibility
+##### Different SSL backends
+
+Default is `reqwest/native-tls`, it is possible to switch to `reqwest/rustls-tls` which is more portable
 
 ```toml
 [dependencies]
-rust-s3 = {version = "0.22.8", features = ["path-style"]}
+rust-s3 = {version = "0.22.11", features = ["rustls-tls"]}
 ```

aws-creds/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "aws-creds"
-version = "0.22.1"
+version = "0.22.2"
 authors = ["Drazen Urch"]
 description = "Tiny Rust library for working with Amazon IAM credential,s, supports `s3` crate"
 repository = "https://github.com/durch/rust-s3"
@@ -15,7 +15,7 @@ name = "awscreds"
 path = "src/lib.rs"
 
 [dependencies]
-simpl = "0.1.0-alpha"
+simpl = "0.1.0"
 dirs = "2.0.2"
 rust-ini = "0.15.2"
 tokio = "0.2.20"

aws-creds/src/credentials.rs

@@ -2,6 +2,7 @@ use crate::{AwsCredsError, Result};
 use ini::Ini;
 use std::collections::HashMap;
 use std::env;
+use std::str::FromStr;
 
 /// AWS access credentials: access key, secret key, and optional token.
 ///
@@ -95,7 +96,7 @@ impl Credentials {
             secret_key: None,
             security_token: None,
             session_token: None,
-            _private: ()
+            _private: (),
         })
     }
 
@@ -108,8 +109,7 @@ impl Credentials {
         session_token: Option<&str>,
         profile: Option<&str>,
     ) -> Result<Credentials> {
-        
-        let security_token = if let Some(security_token) = security_token{
+        let security_token = if let Some(security_token) = security_token {
             Some(security_token.to_string())
         } else {
             None
@@ -152,14 +152,20 @@ impl Credentials {
         }
     }
 
-    pub fn from_env() -> Result<Credentials> {
-        let access_key = env::var("AWS_ACCESS_KEY_ID")?;
-        let secret_key = env::var("AWS_SECRET_ACCESS_KEY")?;
-        let security_token = match env::var("AWS_SECURITY_TOKEN") {
+    pub fn from_env_specific(
+        access_key_var: Option<&str>,
+        secret_key_var: Option<&str>,
+        security_token_var: Option<&str>,
+        session_token_var: Option<&str>,
+    ) -> Result<Credentials> {
+        let access_key = from_env_with_default(access_key_var, "AWS_ACCESS_KEY_ID")?;
+        let secret_key = from_env_with_default(secret_key_var, "AWS_SECRET_ACCESS_KEY")?;
+
+        let security_token = match from_env_with_default(security_token_var, "AWS_SECURITY_TOKEN") {
             Ok(x) => Some(x),
             Err(_) => None,
         };
-        let session_token = match env::var("AWS_SESSION_TOKEN") {
+        let session_token = match from_env_with_default(session_token_var, "AWS_SESSION_TOKEN") {
             Ok(x) => Some(x),
             Err(_) => None,
         };
@@ -172,6 +178,10 @@ impl Credentials {
         })
     }
 
+    pub fn from_env() -> Result<Credentials> {
+        Credentials::from_env_specific(None, None, None, None)
+    }
+
     async fn from_instance_metadata() -> Result<Credentials> {
         if !Credentials::is_ec2() {
             return Err(AwsCredsError::from("Not an EC2 instance"));
@@ -280,3 +290,31 @@ impl Credentials {
         })
     }
 }
+
+fn from_env_with_default(var: Option<&str>, default: &str) -> Result<String> {
+    if let Some(var) = var {
+        if let Ok(value) = env::var(var) {
+            Ok(value)
+        } else {
+            match env::var(default) {
+                Ok(value) => Ok(value),
+                Err(_) => Err(format!(
+                    "Neither {:?}, nor {} does not exist in the environment",
+                    var, default
+                )
+                .as_str()
+                .into()),
+            }
+        }
+    } else {
+        match env::var(default) {
+            Ok(value) => Ok(value),
+            Err(_) => Err(format!(
+                "Neither {:?}, nor {} does not exist in the environment",
+                var, default
+            )
+            .as_str()
+            .into()),
+        }
+    }
+}

s3/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rust-s3"
-version = "0.22.10"
+version = "0.22.11"
 authors = ["Drazen Urch", "Nick Stevens"]
 description = "Tiny Rust library for working with Amazon S3 and compatible object storage APIs"
 repository = "https://github.com/durch/rust-s3"
@@ -35,7 +35,7 @@ tokio = {version="0.2.13", features=["macros", "io-util", "stream"]}
 rand = "0.7.3"
 simpl = "0.1.0"
 aws-region = "0.21.0"
-aws-creds = "0.22.1"
+aws-creds = "0.22.2"
 log = "0.4.8"
 once_cell = "1.3.1"
 percent-encoding = "2.1.0"
@@ -44,7 +44,6 @@ percent-encoding = "2.1.0"
 default = ["native-tls"]
 no-verify-ssl = []
 fail-on-err = []
-path-style = []
 native-tls = ["reqwest/native-tls"]
 rustls-tls = ["reqwest/rustls-tls"]
 

s3/bin/simple_crud.rs

@@ -5,8 +5,8 @@ use std::str;
 
 use s3::bucket::Bucket;
 use s3::creds::Credentials;
-use s3::S3Error;
 use s3::region::Region;
+use s3::S3Error;
 
 struct Storage {
     name: String,
@@ -22,39 +22,45 @@ pub fn main() -> Result<(), S3Error> {
     let aws = Storage {
         name: "aws".into(),
         region: "eu-central-1".parse()?,
-        credentials: Credentials::from_profile(Some("rust-s3"))?,
+        // credentials: Credentials::from_profile(Some("rust-s3"))?,
+        credentials: Credentials::from_env_specific(
+            Some("EU_AWS_ACCESS_KEY_ID"),
+            Some("EU_AWS_SECRET_ACCESS_KEY"),
+            None,
+            None,
+        )?,
         bucket: "rust-s3-test".to_string(),
         location_supported: true,
     };
 
-    let aws_public = Storage {
-        name: "aws-public".into(),
-        region: "eu-central-1".parse()?,
-        credentials: Credentials::anonymous()?,
-        bucket: "rust-s3-public".to_string(),
-        location_supported: true,
-    };
-
-    let minio = Storage {
-        name: "minio".into(),
-        region: Region::Custom {
-            region: "us-east-1".into(),
-            endpoint: "https://minio.adder.black".into(),
-        },
-        credentials: Credentials::from_profile(Some("minio"))?,
-        bucket: "rust-s3".to_string(),
-        location_supported: false,
-    };
-
-    let yandex = Storage {
-        name: "yandex".into(),
-        region: "ru-central1".parse()?,
-        credentials: Credentials::from_profile(Some("yandex"))?,
-        bucket: "soundcloud".to_string(),
-        location_supported: false,
-    };
-
-    for backend in vec![aws, yandex] {
+    // let aws_public = Storage {
+    //     name: "aws-public".into(),
+    //     region: "eu-central-1".parse()?,
+    //     credentials: Credentials::anonymous()?,
+    //     bucket: "rust-s3-public".to_string(),
+    //     location_supported: true,
+    // };
+
+    // let minio = Storage {
+    //     name: "minio".into(),
+    //     region: Region::Custom {
+    //         region: "us-east-1".into(),
+    //         endpoint: "https://minio.adder.black".into(),
+    //     },
+    //     credentials: Credentials::from_profile(Some("minio"))?,
+    //     bucket: "rust-s3".to_string(),
+    //     location_supported: false,
+    // };
+
+    // let yandex = Storage {
+    //     name: "yandex".into(),
+    //     region: "ru-central1".parse()?,
+    //     credentials: Credentials::from_profile(Some("yandex"))?,
+    //     bucket: "soundcloud".to_string(),
+    //     location_supported: false,
+    // };
+
+    for backend in vec![aws] {
         println!("Running {}", backend.name);
         // Create Bucket in REGION for BUCKET
         let bucket = Bucket::new(&backend.bucket, backend.region, backend.credentials)?;
@@ -75,7 +81,8 @@ pub fn main() -> Result<(), S3Error> {
 
         // Put a "test_file" with the contents of MESSAGE at the root of the
         // bucket.
-        let (_, code) = bucket.put_object_blocking("test_file", MESSAGE.as_bytes(), "text/plain")?;
+        let (_, code) =
+            bucket.put_object_blocking("test_file", MESSAGE.as_bytes(), "text/plain")?;
         // println!("{}", bucket.presign_get("test_file", 604801)?);
         assert_eq!(200, code);