Merge pull request #86 from EMCECS/test-failures-fix

resolve tagging failures in S3Encryption* tests

Author: twincitiesguy

src/main/java/com/emc/object/s3/jersey/S3EncryptionClient.java

@@ -183,6 +183,13 @@ public boolean rekey(String bucketName, String key) {
         }
     }
 
+    /**
+     * Encrypted version of {@link S3JerseyClient#putObject(PutObjectRequest)}.
+     * <p>
+     * Note: this method will write the encrypted object first, then update the metadata to finalize encryption
+     * properties (including original SHA1 and metadata signature). For version-enabled buckets, this will create
+     * 2 versions.
+     */
     @Override
     public PutObjectResult putObject(PutObjectRequest request) {
         if (request.getRange() != null)
@@ -202,8 +209,11 @@ public PutObjectResult putObject(PutObjectRequest request) {
         // encryption filter will modify userMeta with encryption metadata *after* the object is transferred
         // we must send a separate metadata update or the object will be unreadable
         // TODO: should this be atomic?  how do we handle rollback?
+        // TODO: also, for version-enabled buckets, should we delete the intermediate version??
+        //       ... and if so, what to do if retention is enabled?
         CopyObjectRequest metadataUpdate = new CopyObjectRequest(request.getBucketName(), request.getKey(),
                 request.getBucketName(), request.getKey()).withAcl(request.getAcl())
+                .withObjectTagging(request.getObjectTagging())
                 .withObjectMetadata(request.getObjectMetadata()).withIfMatch(result.getETag());
         return super.copyObject(metadataUpdate);
     }

src/main/java/com/emc/object/s3/request/CopyObjectRequest.java

@@ -31,6 +31,7 @@
 import com.emc.object.s3.S3ObjectMetadata;
 import com.emc.object.s3.bean.AccessControlList;
 import com.emc.object.s3.bean.CannedAcl;
+import com.emc.object.s3.bean.ObjectTagging;
 import com.emc.object.util.RestUtil;
 
 import java.util.Date;
@@ -55,6 +56,8 @@ public class CopyObjectRequest extends S3ObjectRequest {
     private CannedAcl cannedAcl;
 
     private String copyMode;
+    private ObjectTagging objectTagging;
+
 
     public CopyObjectRequest(String sourceBucketName, String sourceKey, String bucketName, String key) {
         super(Method.PUT, bucketName, key, null);
@@ -87,6 +90,8 @@ public Map<String, List<Object>> getHeaders() {
         if (cannedAcl != null) RestUtil.putSingle(headers, S3Constants.AMZ_ACL, cannedAcl.getHeaderValue());
         if (copyMode != null)
             RestUtil.putSingle(headers, RestUtil.EMC_COPY_MODE, copyMode);
+        if (objectTagging != null)
+            RestUtil.putSingle(headers, S3Constants.AMZ_TAGGING, RestUtil.generateRawQueryString(objectTagging.toStringMap()));
         return headers;
     }
 
@@ -186,6 +191,14 @@ public void setCopyMode(String copyMode) {
         this.copyMode = copyMode;
     }
 
+    public ObjectTagging getObjectTagging() {
+        return objectTagging;
+    }
+
+    public void setObjectTagging(ObjectTagging objectTagging) {
+        this.objectTagging = objectTagging;
+    }
+
     public CopyObjectRequest withSourceVersionId(String sourceVersionId) {
         setSourceVersionId(sourceVersionId);
         return this;
@@ -230,4 +243,9 @@ public CopyObjectRequest withCopyMode(String copyMode) {
         setCopyMode(copyMode);
         return this;
     }
+
+    public CopyObjectRequest withObjectTagging(ObjectTagging objectTagging) {
+        setObjectTagging(objectTagging);
+        return this;
+    }
 }

src/test/java/com/emc/object/s3/AbstractS3ClientTest.java

@@ -29,20 +29,22 @@
 import com.emc.object.AbstractClientTest;
 import com.emc.object.ObjectConfig;
 import com.emc.object.Protocol;
-import com.emc.object.s3.bean.AbstractVersion;
-import com.emc.object.s3.bean.EncodingType;
-import com.emc.object.s3.bean.S3Object;
+import com.emc.object.s3.bean.*;
 import com.emc.object.s3.jersey.S3JerseyClient;
+import com.emc.object.s3.request.DeleteObjectRequest;
 import com.emc.object.s3.request.ListObjectsRequest;
 import com.emc.object.s3.request.ListVersionsRequest;
+import com.emc.object.s3.request.SetObjectLegalHoldRequest;
 import com.emc.object.util.TestProperties;
 import com.emc.rest.smart.LoadBalancer;
 import com.emc.rest.smart.ecs.Vdc;
 import com.emc.util.TestConfig;
 import org.junit.After;
+import org.junit.Before;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
 import java.net.URI;
 import java.util.Arrays;
 import java.util.Properties;
@@ -55,6 +57,8 @@ public abstract class AbstractS3ClientTest extends AbstractClientTest {
      * may be null
      */
     protected String ecsVersion;
+    protected boolean isIamUser = false;
+    protected CanonicalUser bucketOwner;
 
     protected abstract S3Client createS3Client() throws Exception;
 
@@ -67,6 +71,12 @@ protected final void initClient() throws Exception {
         }
     }
 
+    @Before
+    public void checkIamUser() throws IOException {
+        Properties props = TestConfig.getProperties();
+        this.isIamUser = Boolean.parseBoolean(props.getProperty(TestProperties.S3_IAM_USER));
+    }
+
     @After
     public void dumpLBStats() {
         if (client != null) {
@@ -83,14 +93,22 @@ public void shutdownClient() {
     @Override
     protected void createBucket(String bucketName) throws Exception {
         client.createBucket(bucketName);
+        this.bucketOwner = client.getBucketAcl(bucketName).getOwner();
     }
 
     @Override
     protected void cleanUpBucket(String bucketName) {
         if (client != null && client.bucketExists(bucketName)) {
+            boolean objectLockEnabled = isIamUser && client.getObjectLockConfiguration(bucketName) != null;
             if (client.getBucketVersioning(bucketName).getStatus() != null) {
                 for (AbstractVersion version : client.listVersions(new ListVersionsRequest(bucketName).withEncodingType(EncodingType.url)).getVersions()) {
-                    client.deleteVersion(bucketName, version.getKey(), version.getVersionId());
+                    DeleteObjectRequest deleteRequest = new DeleteObjectRequest(bucketName, version.getKey()).withVersionId(version.getVersionId());
+                    if (objectLockEnabled) {
+                        client.setObjectLegalHold(new SetObjectLegalHoldRequest(bucketName, version.getKey()).withVersionId(version.getVersionId())
+                                .withLegalHold(new ObjectLockLegalHold().withStatus(ObjectLockLegalHold.Status.OFF)));
+                        deleteRequest.withBypassGovernanceRetention(true);
+                    }
+                    client.deleteObject(deleteRequest);
                 }
             } else {
                 for (S3Object object : client.listObjects(new ListObjectsRequest(bucketName).withEncodingType(EncodingType.url)).getObjects()) {

src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java

@@ -35,6 +35,9 @@
 import com.emc.object.s3.jersey.FaultInjectionFilter;
 import com.emc.object.s3.jersey.S3EncryptionClient;
 import com.emc.object.s3.jersey.S3JerseyClient;
+import com.emc.object.s3.request.DeleteObjectRequest;
+import com.emc.object.s3.request.GetObjectRequest;
+import com.emc.object.s3.request.GetObjectTaggingRequest;
 import com.emc.object.s3.request.PutObjectRequest;
 import com.emc.util.RandomInputStream;
 import org.apache.commons.codec.digest.DigestUtils;
@@ -52,6 +55,8 @@
 import java.security.KeyPair;
 import java.security.interfaces.RSAPrivateKey;
 import java.security.interfaces.RSAPublicKey;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Properties;
 
 import static org.junit.Assert.assertEquals;
@@ -557,6 +562,69 @@ public void testExtendObjectRetentionPeriod() {
     public void testPreSignedUrlHeaderOverrides() throws Exception {
     }
 
+    @Ignore
+    @Override
+    public void testSingleMultipartUploadWithRetention() {
+    }
+
+    @Ignore
+    @Override
+    public void testCopyObjectWithLegalHoldON() {
+    }
+
+    @Override
+    public void testGetPutDeleteObjectWithTagging() {
+        // set up env
+        String bucketName = getTestBucket(), key = "test-object-tagging";
+        client.setBucketVersioning(bucketName, new VersioningConfiguration().withStatus(VersioningConfiguration.Status.Enabled));
+
+        // write version 1
+        client.putObject(new PutObjectRequest(bucketName, key, "Hello Version 1 !")
+                .withObjectTagging(new ObjectTagging().withTagSet(Arrays.asList(new ObjectTag("k11", "v11"), new ObjectTag("k12", "v12")))));
+        // write version 2
+        client.putObject(new PutObjectRequest(bucketName, key, "Hello Version 2 !"));
+
+        // NOTE: encryption client creates 2 versions per PUT, due to secondary metadata update operation
+        List<AbstractVersion> versions = client.listVersions(bucketName, key).getVersions();
+        String versionId1a = versions.get(3).getVersionId();
+        String versionId1b = versions.get(2).getVersionId();
+        String versionId2a = versions.get(1).getVersionId();
+        String versionId2b = versions.get(0).getVersionId();
+
+        // Only the particular version of the object should get deleted and no other versions of object should be affected
+        // NOTE: have to delete both versions created by the encryption client
+        client.deleteObject(new DeleteObjectRequest(bucketName, key).withVersionId(versionId2a));
+        client.deleteObject(new DeleteObjectRequest(bucketName, key).withVersionId(versionId2b));
+        // NOTE: actually both versions that the encryption client creates should have tagging set
+        //       but to test, we must use rclient (raw client) because encryption client cannot read the intermediate version
+        Assert.assertEquals(2,
+                rclient.getObject(new GetObjectRequest(bucketName, key).withVersionId(versionId1a), String.class).getObjectMetadata().getTaggingCount());
+        Assert.assertEquals(2,
+                client.getObject(new GetObjectRequest(bucketName, key).withVersionId(versionId1b), String.class).getObjectMetadata().getTaggingCount());
+
+        // Object and associated multiple tags should get deleted
+        // NOTE: have to delete both versions created by the encryption client
+        client.deleteObject(new DeleteObjectRequest(bucketName, key).withVersionId(versionId1a));
+        client.deleteObject(new DeleteObjectRequest(bucketName, key).withVersionId(versionId1b));
+        try {
+            client.getObjectTagging(new GetObjectTaggingRequest(bucketName, key).withVersionId(versionId1b));
+            Assert.fail("Fail was expected. Can NOT get tags from a deleted object");
+        } catch (S3Exception e) {
+            Assert.assertEquals(404, e.getHttpCode());
+            Assert.assertEquals("NoSuchKey", e.getErrorCode());
+        }
+    }
+
+    @Ignore
+    @Override
+    public void testCopyObjectWithTaggingAndMeta() {
+    }
+
+    @Ignore
+    @Override
+    public void testMultipartUploadWithTagging() {
+    }
+
     private class ErrorStream extends FilterInputStream {
         private int callCount = 0;