The operation of this implementation is controlled by three conditions:
+ *These three variables work together as follows:
+ * + *In summary:
+ * + *| Original Off-Heap | OneArena | OffHeap | Subsequent Allocations |
|---|---|---|---|
| false | N/A | N/A | All on-heap |
| true | N/A | false | All on-heap |
| true | true | true | All off-heap in original Arena |
| true | false | true | All off-heap in separate Arenas |
Austin Appleby's C++ - * - * MurmurHash3_x64_128(...), final revision 150, - * which is in the Public Domain, was the inspiration for this implementation in Java.
- * - *This implementation of the MurmurHash3 allows hashing of a block of on-heap Memory defined by an offset - * and length. The calling API also allows the user to supply the small output array of two longs, - * so that the entire hash function is static and free of object allocations.
- * - *This implementation produces exactly the same hash result as the - * MurmurHash3 function in datasketches-java given compatible inputs.
- * - *This version 4 of the implementation leverages the java.lang.foreign package of JDK-21 in place of - * the Unsafe class. - * - * @author Lee Rhodes - */ -public final class MurmurHash3 { - - private MurmurHash3() { } - - //Provided for backward compatibility - - /** - * Returns a 128-bit hash of the input. - * Provided for compatibility with older version of MurmurHash3, - * but empty or null input now throws IllegalArgumentException. - * @param in long array - * @param seed A long valued seed. - * @return the hash - */ - public static long[] hash( - final long[] in, - final long seed) { - return MurmurHash3v4.hash(in, seed); - } - - /** - * Returns a 128-bit hash of the input. - * Provided for compatibility with older version of MurmurHash3, - * but empty or null input now throws IllegalArgumentException. - * @param in int array - * @param seed A long valued seed. - * @return the hash - */ - public static long[] hash( - final int[] in, - final long seed) { - return MurmurHash3v4.hash(in, seed); - } - - /** - * Returns a 128-bit hash of the input. - * Provided for compatibility with older version of MurmurHash3, - * but empty or null input now throws IllegalArgumentException. - * @param in char array - * @param seed A long valued seed. - * @return the hash - */ - public static long[] hash( - final char[] in, - final long seed) { - return MurmurHash3v4.hash(in, seed); - } - - /** - * Returns a 128-bit hash of the input. - * Provided for compatibility with older version of MurmurHash3, - * but empty or null input now throws IllegalArgumentException. - * @param in byte array - * @param seed A long valued seed. - * @return the hash - */ - public static long[] hash( - final byte[] in, - final long seed) { - return MurmurHash3v4.hash(in, seed); - } - - //Single primitive inputs - - /** - * Returns a 128-bit hash of the input. - * Note the entropy of the resulting hash cannot be more than 64 bits. - * @param in a long - * @param seed A long valued seed. - * @param hashOut A long array of size 2 - * @return the hash - */ - public static long[] hash( - final long in, - final long seed, - final long[] hashOut) { - return MurmurHash3v4.hash(in, seed, hashOut); - } - - /** - * Returns a 128-bit hash of the input. - * Note the entropy of the resulting hash cannot be more than 64 bits. - * @param in a double - * @param seed A long valued seed. - * @param hashOut A long array of size 2 - * @return the hash - */ - public static long[] hash( - final double in, - final long seed, - final long[] hashOut) { - return MurmurHash3v4.hash(in, seed, hashOut); - } - - /** - * Returns a 128-bit hash of the input. - * An empty or null input throws IllegalArgumentException. - * @param in a String - * @param seed A long valued seed. - * @param hashOut A long array of size 2 - * @return the hash - */ - public static long[] hash( - final String in, - final long seed, - final long[] hashOut) { - return MurmurHash3v4.hash(in, seed, hashOut); - } - - //The main API calls - - /** - * Returns a 128-bit hash of the input as a long array of size 2. - * - * @param mem The input on-heap Memory. Must be non-null and non-empty. - * @param offsetBytes the starting point within Memory. - * @param lengthBytes the total number of bytes to be hashed. - * @param seed A long valued seed. - * @param hashOut the size 2 long array for the resulting 128-bit hash - * @return the hash. - */ - public static long[] hash( - final Memory mem, - final long offsetBytes, - final long lengthBytes, - final long seed, - final long[] hashOut) { - return MurmurHash3v4.hash(mem, offsetBytes, lengthBytes, seed, hashOut); - } - - /** - * Returns a 128-bit hash of the input as a long array of size 2. - * - * @param seg The input MemorySegment. Must be non-null and non-empty. - * @param offsetBytes the starting point within Memory. - * @param lengthBytes the total number of bytes to be hashed. - * @param seed A long valued seed. - * @param hashOut the size 2 long array for the resulting 128-bit hash - * @return the hash. - */ - public static long[] hash( - final MemorySegment seg, - final long offsetBytes, - final long lengthBytes, - final long seed, - final long[] hashOut) { - return MurmurHash3v4.hash(seg, offsetBytes, lengthBytes, seed, hashOut); - } - -} diff --git a/src/main/java/org/apache/datasketches/memory/Resource.java b/src/main/java/org/apache/datasketches/memory/Resource.java index 701b31ec..986c471f 100644 --- a/src/main/java/org/apache/datasketches/memory/Resource.java +++ b/src/main/java/org/apache/datasketches/memory/Resource.java @@ -36,8 +36,9 @@ public interface Resource { /** * The default MemoryRequestServer used primarily by test. + * Do not allocate requested memory off-heap. */ - static final MemoryRequestServer defaultMemReqSvr = new DefaultMemoryRequestServer(); + static final MemoryRequestServer defaultMemReqSvr = new DefaultMemoryRequestServer(8, ByteOrder.nativeOrder(), false, false); /** * Gets the {@link MemoryRequestServer} to request additional memory @@ -145,7 +146,7 @@ boolean equalTo( /** * Returns the arena used to create this resource and possibly other resources. * Be careful when you close the returned Arena, you may be closing other resources as well. - * @return the arena used to create this resource and possibly other resources. + * @return the arena used to create this resource and possibly other resources. It may be null. */ Arena getArena(); @@ -156,11 +157,18 @@ boolean equalTo( long getCapacity(); /** - * Gets the relative base offset of this with respect to that, defined as: this - that. - * This method is only valid for native (off-heap) allocated resources. + * Gets the MemorySegment that backs this resource as a read-only MemorySegment. + * @return the MemorySegment that back this resource as a read-only MemorySegment. + */ + MemorySegment getMemorySegment(); + + /** + * Gets the relative base offset of this resource with respect to that resource, + * defined as: this - that. * @param that the given resource. * @return this - that offset - * @throws IllegalArgumentException if one of the resources is on-heap. + * @throws UnsupportedOperationException if the two resources cannot be compared, e.g. because they are of + * different kinds, or because they are backed by different Java arrays. */ long getRelativeOffset(Resource that); @@ -253,9 +261,17 @@ boolean equalTo( boolean isRegion(); /** - * Returns true if the underlying resource is the same underlying resource as that. - * @param that the other Resource object - * @return a long value representing the ordering and size of overlap between this and that + * Returns true if the underlying resource is the same resource as that. + * + *
Two resources are considered the same if one were to write a value at offset A in one resource + * and that same value appears in the other resource at the same offset A. In other words, + * if two regions (or slices) are derived from the same underlying resource they both must have the same + * starting offset with respect to the resource and the same size in order to be considered to be the same resource.
+ * + *Note: for on-heap resources neither this nor that can be read-only.
+ * + * @param that the other Resource. + * @return true if the underlying resource is the same underlying resource as that. */ boolean isSameResource(Resource that); @@ -265,17 +281,6 @@ boolean equalTo( */ void load(); - /** - * Returns a positive number if this overlaps that and this base address is ≤ that - * base address. - * Returns a negative number if this overlaps that and this base address is > that - * base address. - * Returns a zero if there is no overlap or if one or both objects are null, not active or on heap. - * @param that the other Resource object - * @return a long value representing the ordering and size of overlap between this and that. - */ - long nativeOverlap(Resource that); - /** * Finds the first byte mismatch with that. * @param that the other Resource @@ -302,6 +307,17 @@ boolean equalTo( */ long mismatch(Resource src, long srcFromOffset, long srcToOffset, Resource dst, long dstFromOffset, long dstToOffset); + /** + * Returns a positive number if this overlaps that and this base address is ≤ that + * base address. + * Returns a negative number if this overlaps that and this base address is > that + * base address. + * Returns a zero if there is no overlap or if one or both objects are null, not active or on heap. + * @param that the other Resource object + * @return a long value representing the ordering and size of overlap between this and that. + */ + long nativeOverlap(Resource that); + /** * Returns the resource scope associated with this memory segment. * @return the resource scope associated with this memory segment. @@ -322,6 +338,7 @@ boolean equalTo( * @param arena the given arena. * If the desired result is to be off-heap, the arena must not be null. * Otherwise, the result will be on-heap. + * Warning: This class is not thread-safe. Specifying an Arena that allows multiple threads is not recommended. * @param alignment requested segment alignment. Typically 1, 2, 4 or 8. * @return a copy of the underlying MemorySegment in the given arena. */ diff --git a/src/main/java/org/apache/datasketches/memory/WritableMemory.java b/src/main/java/org/apache/datasketches/memory/WritableMemory.java index 23d0ba00..69656ac1 100644 --- a/src/main/java/org/apache/datasketches/memory/WritableMemory.java +++ b/src/main/java/org/apache/datasketches/memory/WritableMemory.java @@ -117,11 +117,10 @@ static WritableMemory writableMap( * Allocates and provides access to capacityBytes directly in native (off-heap) memory. * The allocated memory will be 8-byte aligned. * Native byte order is assumed. - * A new DefaultMemoryRequestServer() is created. + * A new DefaultMemoryRequestServer() is created, which allocates on-heap. * - *NOTE: Native/Direct memory acquired may have garbage in it. - * It is the responsibility of the using application to clear this memory, if required, - * and to call close() when done.
+ *NOTE:It is the responsibility of the using application to call + * WritableMemory::getArena().close() when done.
* @param capacityBytes the size of the desired memory in bytes. * @param arena the given arena to manage the new off-heap WritableMemory. It must be non-null. * Warning: This class is not thread-safe. Specifying an Arena that allows multiple threads is not recommended. @@ -132,6 +131,25 @@ static WritableMemory allocateDirect(long capacityBytes, Arena arena) { return allocateDirect(capacityBytes, 8, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer(), arena); } + /** + * Allocates and provides access to capacityBytes directly in native (off-heap) memory. + * The allocated memory will be 8-byte aligned. + * Native byte order is assumed. + * + *NOTE:It is the responsibility of the using application to call + * WritableMemory::getArena().close() when done.
+ * @param capacityBytes the size of the desired memory in bytes. + * @param memReqSvr A user-specified MemoryRequestServer, which may be null. + * This is a callback mechanism for a user client of direct memory to request more memory. + * @param arena the given arena to manage the new off-heap WritableMemory. It must be non-null. + * Warning: This class is not thread-safe. Specifying an Arena that allows multiple threads is not recommended. + * + * @return a WritableMemory for this off-heap resource. + */ + static WritableMemory allocateDirect(long capacityBytes, MemoryRequestServer memReqSvr, Arena arena) { + return allocateDirect(capacityBytes, 8, ByteOrder.nativeOrder(), memReqSvr, arena); + } + /** * Allocates and provides access to capacityBytes directly in native (off-heap) memory. * The allocated memory will be aligned to the given alignmentBytes. @@ -340,7 +358,7 @@ static WritableMemory writableWrap( ByteOrder byteOrder, MemoryRequestServer memReqSvr) { final MemorySegment slice = MemorySegment.ofArray(array).asSlice(offsetBytes, lengthBytes); - return WritableMemoryImpl.wrapSegmentAsArray(slice, byteOrder, memReqSvr); + return WritableMemoryImpl.wrapSegment(slice, byteOrder, memReqSvr); } //intentionally removed writableWrap(boolean[]) @@ -352,7 +370,7 @@ static WritableMemory writableWrap( */ static WritableMemory writableWrap(char[] array) { final MemorySegment seg = MemorySegment.ofArray(array); - return WritableMemoryImpl.wrapSegmentAsArray(seg, ByteOrder.nativeOrder(), null); + return WritableMemoryImpl.wrapSegment(seg, ByteOrder.nativeOrder()); } /** @@ -362,7 +380,7 @@ static WritableMemory writableWrap(char[] array) { */ static WritableMemory writableWrap(short[] array) { final MemorySegment seg = MemorySegment.ofArray(array); - return WritableMemoryImpl.wrapSegmentAsArray(seg, ByteOrder.nativeOrder(), null); + return WritableMemoryImpl.wrapSegment(seg, ByteOrder.nativeOrder()); } /** @@ -372,7 +390,7 @@ static WritableMemory writableWrap(short[] array) { */ static WritableMemory writableWrap(int[] array) { final MemorySegment seg = MemorySegment.ofArray(array); - return WritableMemoryImpl.wrapSegmentAsArray(seg, ByteOrder.nativeOrder(), null); + return WritableMemoryImpl.wrapSegment(seg, ByteOrder.nativeOrder()); } /** @@ -382,7 +400,7 @@ static WritableMemory writableWrap(int[] array) { */ static WritableMemory writableWrap(long[] array) { final MemorySegment seg = MemorySegment.ofArray(array); - return WritableMemoryImpl.wrapSegmentAsArray(seg, ByteOrder.nativeOrder(), null); + return WritableMemoryImpl.wrapSegment(seg, ByteOrder.nativeOrder()); } /** @@ -392,7 +410,7 @@ static WritableMemory writableWrap(long[] array) { */ static WritableMemory writableWrap(float[] array) { final MemorySegment seg = MemorySegment.ofArray(array); - return WritableMemoryImpl.wrapSegmentAsArray(seg, ByteOrder.nativeOrder(), null); + return WritableMemoryImpl.wrapSegment(seg, ByteOrder.nativeOrder()); } /** @@ -402,7 +420,7 @@ static WritableMemory writableWrap(float[] array) { */ static WritableMemory writableWrap(double[] array) { final MemorySegment seg = MemorySegment.ofArray(array); - return WritableMemoryImpl.wrapSegmentAsArray(seg, ByteOrder.nativeOrder(), null); + return WritableMemoryImpl.wrapSegment(seg, ByteOrder.nativeOrder()); } //END OF CONSTRUCTOR-TYPE METHODS diff --git a/src/main/java/org/apache/datasketches/memory/internal/MurmurHash3v4.java b/src/main/java/org/apache/datasketches/memory/internal/MurmurHash3v4.java deleted file mode 100644 index 3c64961b..00000000 --- a/src/main/java/org/apache/datasketches/memory/internal/MurmurHash3v4.java +++ /dev/null @@ -1,386 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.datasketches.memory.internal; - -import static java.nio.charset.StandardCharsets.UTF_8; - -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.util.Objects; - -import org.apache.datasketches.memory.Memory; - -/** - * The MurmurHash3 is a fast, non-cryptographic, 128-bit hash function that has - * excellent avalanche and 2-way bit independence properties. - * - *Austin Appleby's C++ - * - * MurmurHash3_x64_128(...), final revision 150, - * which is in the Public Domain, was the inspiration for this implementation in Java.
- * - *This implementation of the MurmurHash3 allows hashing of a block of on-heap Memory defined by an offset - * and length. The calling API also allows the user to supply the small output array of two longs, - * so that the entire hash function is static and free of object allocations.
- * - *This implementation produces exactly the same hash result as the - * MurmurHash3 function in datasketches-java given compatible inputs.
- * - *This version 4 of the implementation leverages the java.lang.foreign package of JDK-21 in place of - * the Unsafe class. - * - * @author Lee Rhodes - */ -public final class MurmurHash3v4 { - private static final long C1 = 0x87c37b91114253d5L; - private static final long C2 = 0x4cf5ad432745937fL; - - /** - * Returns a 128-bit hash of the input. - * Provided for compatibility with older version of MurmurHash3, - * but empty or null input now throws IllegalArgumentException. - * @param in long array - * @param seed A long valued seed. - * @return the hash - * @throws IllegalArgumentException if input is empty or null - */ - public static long[] hash(final long[] in, final long seed) { - if ((in == null) || (in.length == 0)) { - throw new IllegalArgumentException("Input in is empty or null."); - } - return hash(MemorySegment.ofArray(in), 0L, in.length << 3, seed, new long[2]); - } - - /** - * Returns a 128-bit hash of the input. - * Provided for compatibility with older version of MurmurHash3, - * but empty or null input now throws IllegalArgumentException. - * @param in int array - * @param seed A long valued seed. - * @return the hash - * @throws IllegalArgumentException if input is empty or null - */ - public static long[] hash(final int[] in, final long seed) { - if ((in == null) || (in.length == 0)) { - throw new IllegalArgumentException("Input in is empty or null."); - } - return hash(MemorySegment.ofArray(in), 0L, in.length << 2, seed, new long[2]); - } - - /** - * Returns a 128-bit hash of the input. - * Provided for compatibility with older version of MurmurHash3, - * but empty or null input now throws IllegalArgumentException. - * @param in char array - * @param seed A long valued seed. - * @return the hash - * @throws IllegalArgumentException if input is empty or null - */ - public static long[] hash(final char[] in, final long seed) { - if ((in == null) || (in.length == 0)) { - throw new IllegalArgumentException("Input in is empty or null."); - } - return hash(MemorySegment.ofArray(in), 0L, in.length << 1, seed, new long[2]); - } - - /** - * Returns a 128-bit hash of the input. - * Provided for compatibility with older version of MurmurHash3, - * but empty or null input now throws IllegalArgumentException. - * @param in byte array - * @param seed A long valued seed. - * @return the hash - * @throws IllegalArgumentException if input is empty or null - */ - public static long[] hash(final byte[] in, final long seed) { - if ((in == null) || (in.length == 0)) { - throw new IllegalArgumentException("Input in is empty or null."); - } - return hash(MemorySegment.ofArray(in), 0L, in.length, seed, new long[2]); - } - - //Single primitive inputs - - /** - * Returns a 128-bit hash of the input. - * Note the entropy of the resulting hash cannot be more than 64 bits. - * @param in a long - * @param seed A long valued seed. - * @param hashOut A long array of size 2 - * @return the hash - */ - public static long[] hash(final long in, final long seed, final long[] hashOut) { - final long h1 = seed ^ mixK1(in); - final long h2 = seed; - return finalMix128(h1, h2, 8, hashOut); - } - - /** - * Returns a 128-bit hash of the input. - * Note the entropy of the resulting hash cannot be more than 64 bits. - * @param in a double - * @param seed A long valued seed. - * @param hashOut A long array of size 2 - * @return the hash - */ - public static long[] hash(final double in, final long seed, final long[] hashOut) { - final double d = (in == 0.0) ? 0.0 : in; // canonicalize -0.0, 0.0 - final long k1 = Double.doubleToLongBits(d); // canonicalize all NaN forms - final long h1 = seed ^ mixK1(k1); - final long h2 = seed; - return finalMix128(h1, h2, 8, hashOut); - } - - /** - * Returns a 128-bit hash of the input. - * An empty or null input throws IllegalArgumentException. - * @param in a String - * @param seed A long valued seed. - * @param hashOut A long array of size 2 - * @return the hash - * @throws IllegalArgumentException if input is empty or null - */ - public static long[] hash(final String in, final long seed, final long[] hashOut) { - if ((in == null) || (in.length() == 0)) { - throw new IllegalArgumentException("Input in is empty or null."); - } - final byte[] byteArr = in.getBytes(UTF_8); - return hash(MemorySegment.ofArray(byteArr), 0L, byteArr.length, seed, hashOut); - } - - //The main API calls - - /** - * Returns a 128-bit hash of the input as a long array of size 2. - * - * @param mem The input Memory. Must be non-null and non-empty, - * otherwise throws IllegalArgumentException. - * @param offsetBytes the starting point within Memory. - * @param lengthBytes the total number of bytes to be hashed. - * @param seed A long valued seed. - * @param hashOut the size 2 long array for the resulting 128-bit hash - * @return the hash. - */ - public static long[] hash(final Memory mem, final long offsetBytes, final long lengthBytes, - final long seed, final long[] hashOut) { - Objects.requireNonNull(mem, "Input Memory is null"); - final MemorySegment seg = ((ResourceImpl)mem).seg; - return hash(seg, offsetBytes, lengthBytes, seed, hashOut); - } - - /** - * Returns a 128-bit hash of the input as a long array of size 2. - * - * @param seg The input MemorySegment. Must be non-null and non-empty, - * otherwise throws IllegalArgumentException. - * @param offsetBytes the starting point within Memory. - * @param lengthBytes the total number of bytes to be hashed. - * @param seed A long valued seed. - * @param hashOut the size 2 long array for the resulting 128-bit hash - * @return the hash. - * @throws IllegalArgumentException if input MemorySegment is empty - */ - public static long[] hash(final MemorySegment seg, final long offsetBytes, final long lengthBytes, - final long seed, final long[] hashOut) { - Objects.requireNonNull(seg, "Input MemorySegment is null"); - if (seg.byteSize() == 0L) { throw new IllegalArgumentException("Input MemorySegment is empty."); } - - long cumOff = offsetBytes; - - long h1 = seed; - long h2 = seed; - long rem = lengthBytes; - - // Process the 128-bit blocks (the body) into the hash - while (rem >= 16L) { - final long k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); //0, 16, 32, ... - final long k2 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff + 8); //8, 24, 40, ... - cumOff += 16L; - rem -= 16L; - - h1 ^= mixK1(k1); - h1 = Long.rotateLeft(h1, 27); - h1 += h2; - h1 = (h1 * 5) + 0x52dce729L; - - h2 ^= mixK2(k2); - h2 = Long.rotateLeft(h2, 31); - h2 += h1; - h2 = (h2 * 5) + 0x38495ab5L; - } - - // Get the tail (if any): 1 to 15 bytes - if (rem > 0L) { - long k1 = 0; - long k2 = 0; - switch ((int) rem) { - case 15: { - k2 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 14) & 0xFFL) << 48; - } - //$FALL-THROUGH$ - case 14: { - k2 ^= (seg.get(ValueLayout.JAVA_SHORT_UNALIGNED, cumOff + 12) & 0xFFFFL) << 32; - k2 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff + 8) & 0xFFFFFFFFL; - k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); - break; - } - - case 13: { - k2 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 12) & 0xFFFFL) << 32; - } - //$FALL-THROUGH$ - case 12: { - k2 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff + 8) & 0xFFFFFFFFL; - k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); - break; - } - - case 11: { - k2 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 10) & 0xFFL) << 16; - } - //$FALL-THROUGH$ - case 10: { - k2 ^= seg.get(ValueLayout.JAVA_SHORT_UNALIGNED, cumOff + 8) & 0xFFFFL; - k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); - break; - } - - case 9: { - k2 ^= seg.get(ValueLayout.JAVA_BYTE, cumOff + 8) & 0xFFL; - } - //$FALL-THROUGH$ - case 8: { - k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); - break; - } - - case 7: { - k1 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 6) & 0xFFL) << 48; - } - //$FALL-THROUGH$ - case 6: { - k1 ^= (seg.get(ValueLayout.JAVA_SHORT_UNALIGNED, cumOff + 4) & 0xFFFFL) << 32; - k1 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff) & 0xFFFFFFFFL; - break; - } - - case 5: { - k1 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 4) & 0xFFL) << 32; - } - //$FALL-THROUGH$ - case 4: { - k1 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff) & 0xFFFFFFFFL; - break; - } - - case 3: { - k1 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 2) & 0xFFL) << 16; - } - //$FALL-THROUGH$ - case 2: { - k1 ^= seg.get(ValueLayout.JAVA_SHORT_UNALIGNED, cumOff) & 0xFFFFL; - break; - } - - case 1: { - k1 ^= seg.get(ValueLayout.JAVA_BYTE, cumOff) & 0xFFL; - break; - } - default: break; //can't happen - } - - h1 ^= mixK1(k1); - h2 ^= mixK2(k2); - } - return finalMix128(h1, h2, lengthBytes, hashOut); - } - - //--Helper methods---------------------------------------------------- - - /** - * Self mix of k1 - * - * @param k1 input argument - * @return mix - */ - private static long mixK1(long k1) { - k1 *= C1; - k1 = Long.rotateLeft(k1, 31); - k1 *= C2; - return k1; - } - - /** - * Self mix of k2 - * - * @param k2 input argument - * @return mix - */ - private static long mixK2(long k2) { - k2 *= C2; - k2 = Long.rotateLeft(k2, 33); - k2 *= C1; - return k2; - } - - /** - * Final self mix of h*. - * - * @param h input to final mix - * @return mix - */ - private static long finalMix64(long h) { - h ^= h >>> 33; - h *= 0xff51afd7ed558ccdL; - h ^= h >>> 33; - h *= 0xc4ceb9fe1a85ec53L; - h ^= h >>> 33; - return h; - } - - /** - * Finalization: Add the length into the hash and mix - * @param h1 intermediate hash - * @param h2 intermediate hash - * @param lengthBytes the length in bytes - * @param hashOut the output array of 2 longs - * @return hashOut - */ - private static long[] finalMix128(long h1, long h2, final long lengthBytes, final long[] hashOut) { - h1 ^= lengthBytes; - h2 ^= lengthBytes; - - h1 += h2; - h2 += h1; - - h1 = finalMix64(h1); - h2 = finalMix64(h2); - - h1 += h2; - h2 += h1; - - hashOut[0] = h1; - hashOut[1] = h2; - return hashOut; - } - - private MurmurHash3v4() { } - -} diff --git a/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java b/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java index 459af659..f2e271f2 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/ResourceImpl.java @@ -106,7 +106,7 @@ abstract class ResourceImpl implements Resource { /** * Root constructor. - * @param seg the given, one and only one MemorySegment + * @param seg the given, one and only MemorySegment * @param typeId the given typeId * @param memReqSvr the given MemoryRequestServer, or null. * @param arena the given Arena, or null if an on-heap MemorySegment. @@ -379,6 +379,11 @@ public final long getCapacity() { return seg.byteSize(); } + @Override + public MemorySegment getMemorySegment() { + return seg.asReadOnly(); + } + @Override public final long getRelativeOffset(final Resource that) { final ResourceImpl that2 = (ResourceImpl) that; @@ -454,9 +459,22 @@ public final boolean isRegion() { @Override public final boolean isSameResource(final Resource that) { Objects.requireNonNull(that); - final ResourceImpl that2 = (ResourceImpl) that; - return this.seg.address() == that2.seg.address() - && this.seg.byteSize() == that2.seg.byteSize(); + final MemorySegment thisSeg = this.seg; + final MemorySegment thatSeg = ((ResourceImpl)that).seg; + final boolean thisNative = thisSeg.isNative(); + final boolean thatNative = thatSeg.isNative(); + if (thisNative != thatNative) { return false; } + if (thisNative && thatNative) { //off-heap + return thisSeg.address() == thatSeg.address() + && thisSeg.byteSize() == thatSeg.byteSize(); + } else { //on heap + if (thisSeg.isReadOnly() || thatSeg.isReadOnly()) { + throw new IllegalArgumentException("Cannot determine 'isSameResource(..)' on heap if either resource is Read-only."); + } + return (thisSeg.heapBase().get() == thatSeg.heapBase().get()) + && (thisSeg.address() == thatSeg.address()) + && thisSeg.byteSize() == thatSeg.byteSize(); + } } @Override diff --git a/src/main/java/org/apache/datasketches/memory/internal/WritableMemoryImpl.java b/src/main/java/org/apache/datasketches/memory/internal/WritableMemoryImpl.java index 4604e092..339c434b 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/WritableMemoryImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/WritableMemoryImpl.java @@ -57,16 +57,35 @@ public abstract class WritableMemoryImpl extends ResourceImpl implements Writabl super(seg, typeId, memReqSvr, arena); } - //WRAP HEAP ARRAY RESOURCE + //WRAP SEGMENT RESOURCE /** - * Wrap a MemorySegment as an array + * Wrap a MemorySegment. + * @param seg the given MemorySegment. It must be non-null. + * @param byteOrder the given ByteOrder. It must be non-null. + * @return a WritableMemory. + */ + public static WritableMemory wrapSegment( + final MemorySegment seg, + final ByteOrder byteOrder) { + Objects.requireNonNull(byteOrder, "byteOrder must be non-null"); + int type = MEMORY + | (seg.isReadOnly() ? READONLY : 0); + if (byteOrder == NON_NATIVE_BYTE_ORDER) { + type |= NONNATIVE_BO; + return new NonNativeWritableMemoryImpl(seg, type, null, null); + } + return new NativeWritableMemoryImpl(seg, type, null, null); + } + + /** + * Wrap a MemorySegment. * @param seg the given MemorySegment. It must be non-null. * @param byteOrder the given ByteOrder. It must be non-null. * @param memReqSvr the given MemoryRequestServer. It may be null. * @return a WritableMemory. */ - public static WritableMemory wrapSegmentAsArray( + public static WritableMemory wrapSegment( final MemorySegment seg, final ByteOrder byteOrder, final MemoryRequestServer memReqSvr) { diff --git a/src/test/java/org/apache/datasketches/memory/internal/AllocateDirectMemoryTest.java b/src/test/java/org/apache/datasketches/memory/internal/AllocateDirectMemoryTest.java index dd0049e2..135d236f 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/AllocateDirectMemoryTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/AllocateDirectMemoryTest.java @@ -27,6 +27,7 @@ import java.lang.foreign.Arena; import java.nio.ByteOrder; +import org.apache.datasketches.memory.DefaultMemoryRequestServer; import org.apache.datasketches.memory.MemoryRequestServer; import org.apache.datasketches.memory.Resource; import org.apache.datasketches.memory.WritableMemory; @@ -56,28 +57,76 @@ public void simpleAllocateDirect() { @Test public void checkDefaultMemoryRequestServer() { + boolean oneArena = false; + boolean offHeap = false; + checkDefaultMemoryRequestServerVariations(false, false, false); + checkDefaultMemoryRequestServerVariations(false, false, true); + checkDefaultMemoryRequestServerVariations(false, true, false); + checkDefaultMemoryRequestServerVariations(false, true, true); + checkDefaultMemoryRequestServerVariations(true, false, false); + checkDefaultMemoryRequestServerVariations(true, false, true); + checkDefaultMemoryRequestServerVariations(true, true, false); + checkDefaultMemoryRequestServerVariations(true, true, true); + } + + private void checkDefaultMemoryRequestServerVariations(boolean origArena, boolean oneArena, boolean offHeap) { int longs1 = 32; int bytes1 = longs1 << 3; - try (Arena arena = Arena.ofConfined()) { - WritableMemory origWmem = WritableMemory.allocateDirect(bytes1, 8, ByteOrder.LITTLE_ENDIAN, Resource.defaultMemReqSvr, arena); + WritableMemory origWmem, newWmem; + + if (origArena) { + MemoryRequestServer dmrs = new DefaultMemoryRequestServer(8, ByteOrder.nativeOrder(), oneArena, offHeap); + try (Arena arena = Arena.ofConfined()) { + origWmem = WritableMemory.allocateDirect(bytes1, 8, ByteOrder.LITTLE_ENDIAN, dmrs, arena); + assertTrue(origWmem.isDirect()); + for (int i = 0; i < longs1; i++) { //puts data in origWmem + origWmem.putLong(i << 3, i); + assertEquals(origWmem.getLong(i << 3), i); + } + println(origWmem.toString("Test", 0, longs1 << 3, true)); + + int longs2 = 2 * longs1; + int bytes2 = longs2 << 3; + MemoryRequestServer myMemReqSvr = origWmem.getMemoryRequestServer(); + + newWmem = myMemReqSvr.request(origWmem, bytes2); + assertTrue( (offHeap && origArena) ? newWmem.isDirect() : newWmem.isHeap() ); + for (int i = 0; i < longs2; i++) { + newWmem.putLong(i << 3, i); + assertEquals(newWmem.getLong(i << 3), i); + } + println(newWmem.toString("Test", 0, longs2 << 3, true)); + if (oneArena && offHeap) { assertTrue((newWmem.getArena() == origWmem.getArena()) && origWmem != null); } + if (oneArena && !offHeap) { assertTrue((newWmem.getArena() == null) && origWmem != null); } + if (!oneArena && offHeap) { assertTrue((newWmem.getArena() != origWmem.getArena()) && origWmem != null); } + } //allow the TWR to close the origWmem resource + assertFalse(origWmem.getArena().scope().isAlive()); + if (!oneArena && offHeap) { + newWmem.getArena().close(); + assertFalse(newWmem.getArena().scope().isAlive()); + } + + } else { + MemoryRequestServer dmrs = new DefaultMemoryRequestServer(8, ByteOrder.nativeOrder(), oneArena, offHeap); + origWmem = WritableMemory.allocate(bytes1,ByteOrder.LITTLE_ENDIAN, dmrs); for (int i = 0; i < longs1; i++) { //puts data in origWmem origWmem.putLong(i << 3, i); assertEquals(origWmem.getLong(i << 3), i); } - println(origWmem.toString("Test", 0, 32 * 8, true)); + println(origWmem.toString("Test", 0, longs1 << 3, true)); - int longs2 = 64; + int longs2 = 2 * longs1; int bytes2 = longs2 << 3; - origWmem.setMemoryRequestServer(Resource.defaultMemReqSvr); MemoryRequestServer myMemReqSvr = origWmem.getMemoryRequestServer(); - WritableMemory newWmem = myMemReqSvr.request(bytes2, 8, ByteOrder.LITTLE_ENDIAN, null); //null -> on-heap - assertTrue(newWmem.isHeap()); + newWmem = myMemReqSvr.request(origWmem, bytes2); + assertTrue( (offHeap && origArena) ? newWmem.isDirect() : newWmem.isHeap() ); for (int i = 0; i < longs2; i++) { newWmem.putLong(i << 3, i); assertEquals(newWmem.getLong(i << 3), i); } - } //allow the TWR to close all resources + println(newWmem.toString("Test", 0, longs2 << 3, true)); + } } @Test diff --git a/src/test/java/org/apache/datasketches/memory/internal/DruidIssue11544Test.java b/src/test/java/org/apache/datasketches/memory/internal/DruidIssue11544Test.java index a9e4d631..ea116a82 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/DruidIssue11544Test.java +++ b/src/test/java/org/apache/datasketches/memory/internal/DruidIssue11544Test.java @@ -59,26 +59,26 @@ public void withByteBuffer() { //Wrap bb into WritableMemory WritableMemory mem1 = WritableMemory.writableWrap(bb); - //ByteBuffers are automatically assigned an implicit shared scope (non-closeable) + //ByteBuffers are not directly closeable. They are closed by the GC. assertTrue(mem1.isDirect()); //confirm mem1 is off-heap - + assertTrue(mem1.getArena() == null); //and Arena is null //Request Bigger Memory on heap int size2 = size1 * 2; - WritableMemory mem2 = myMemReqSvr.request(size2, 8, ByteOrder.LITTLE_ENDIAN, null); + WritableMemory mem2 = myMemReqSvr.request(mem1, size2); - //Confirm that mem2 is on the heap (the default) and 2X size1 + //Confirm that mem2 is on the heap and 2X size1 assertFalse(mem2.isDirect()); assertEquals(mem2.getCapacity(), size2); //Move data to new memory mem1.copyTo(0, mem2, 0, size1); - assertTrue(mem1.isAlive()); + assertTrue(mem1.isAlive()); //because mem1 seg is holding a reference to it. assertTrue(mem2.isAlive()); //Now we are on the heap and need to grow again: int size3 = size2 * 2; - WritableMemory mem3 = myMemReqSvr.request(size3, 8, ByteOrder.LITTLE_ENDIAN, null); + WritableMemory mem3 = myMemReqSvr.request(mem2, size3); //Confirm that mem3 is still on the heap and 2X of size2 assertFalse(mem3.isDirect()); diff --git a/src/test/java/org/apache/datasketches/memory/internal/ExampleMemoryRequestServerTest.java b/src/test/java/org/apache/datasketches/memory/internal/ExampleMemoryRequestServerTest.java index 55dd7884..d24005f0 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/ExampleMemoryRequestServerTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/ExampleMemoryRequestServerTest.java @@ -24,6 +24,7 @@ import org.apache.datasketches.memory.DefaultMemoryRequestServer; import org.apache.datasketches.memory.MemoryRequestServer; +import org.apache.datasketches.memory.Resource; import org.apache.datasketches.memory.WritableMemory; import org.testng.annotations.Test; @@ -35,7 +36,6 @@ * @author Lee Rhodes */ public class ExampleMemoryRequestServerTest { - private static long alignmentBytes = 8; /** * This version is without a TWR block. All of the memory allocations are done through the MemoryRequestServer @@ -48,15 +48,13 @@ public void checkExampleMemoryRequestServer1() { long workingMemBytes = 8; Arena arena = Arena.ofConfined(); - //Configure the default memReqSvr to create new memory off-heap and copy data from old to new - MemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); + //Configure the memReqSvr to create new subsequent allocations off-heap, each with a new Arena. + MemoryRequestServer myMemReqSvr = new DefaultMemoryRequestServer(8, ByteOrder.nativeOrder(), false, true); //Create the initial working memory for the client WritableMemory workingMem = WritableMemory.allocateDirect( workingMemBytes, - alignmentBytes, - ByteOrder.nativeOrder(), - memReqSvr, + myMemReqSvr, arena); MemoryHungryClient client = new MemoryHungryClient(workingMem); @@ -95,14 +93,14 @@ void process() { oldWorkingCap = newWorkingCap; newWorkingCap = 2 * oldWorkingCap; Arena arena = Arena.ofConfined(); // new confined scope for each iteration - newMem = memReqSvr.request(newWorkingCap, alignmentBytes, ByteOrder.LITTLE_ENDIAN, arena); + newMem = memReqSvr.request(workingMem, newWorkingCap); //done with old memory, close it - memReqSvr.requestClose(workingMem.getArena()); + memReqSvr.requestClose(workingMem); workingMem = newMem; itr++; } - + //close the last allocation workingMem.getArena().close(); } } diff --git a/src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v3Test.java b/src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v3Test.java deleted file mode 100644 index 68069003..00000000 --- a/src/test/java/org/apache/datasketches/memory/internal/MurmurHash3v3Test.java +++ /dev/null @@ -1,419 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.datasketches.memory.internal; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.datasketches.memory.MurmurHash3.*; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; - -import java.lang.foreign.Arena; -import java.lang.foreign.MemorySegment; -import java.nio.ByteOrder; - -import org.apache.datasketches.memory.Resource; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.MemoryRequestServer; -import org.apache.datasketches.memory.WritableMemory; -import org.testng.Assert; -import org.testng.annotations.Test; - -/** - * Tests the MurmurHash3 against specific, known hash results given known - * inputs obtained from the public domain C++ version 150. - * - * @author Lee Rhodes - */ -public class MurmurHash3v3Test { - private static final MemoryRequestServer memReqSvr = Resource.defaultMemReqSvr; - - @Test - public void checkByteArrRemainderGT8() { //byte[], remainder > 8 - String keyStr = "The quick brown fox jumps over the lazy dog"; - byte[] key = keyStr.getBytes(UTF_8); - long[] result = hash(key, 0); - //Should be: - long h1 = 0xe34bbc7bbc071b6cL; - long h2 = 0x7a433ca9c49a9347L; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - @Test - public void checkByteArrRemainderGT8withSegment() { //byte[], remainder > 8 - String keyStr = "The quick brown fox jumps over the lazy dog"; - byte[] key = keyStr.getBytes(UTF_8); - long[] out = new long[2]; - MemorySegment seg = MemorySegment.ofArray(key); - long[] result = hash(seg, 0, seg.byteSize(), 0, out); - //Should be: - long h1 = 0xe34bbc7bbc071b6cL; - long h2 = 0x7a433ca9c49a9347L; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - @Test - public void checkByteArrChange1bit() { //byte[], change one bit - String keyStr = "The quick brown fox jumps over the lazy eog"; - byte[] key = keyStr.getBytes(UTF_8); - long[] result = hash(key, 0); - //Should be: - long h1 = 0x362108102c62d1c9L; - long h2 = 0x3285cd100292b305L; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - @Test - public void checkByteArrRemainderLt8() { //byte[], test a remainder < 8 - String keyStr = "The quick brown fox jumps over the lazy dogdogdog"; - byte[] key = keyStr.getBytes(UTF_8); - long[] result = hash(key, 0); - //Should be; - long h1 = 0x9c8205300e612fc4L; - long h2 = 0xcbc0af6136aa3df9L; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - @Test - public void checkByteArrReaminderEQ8() { //byte[], test a remainder = 8 - String keyStr = "The quick brown fox jumps over the lazy1"; - byte[] key = keyStr.getBytes(UTF_8); - long[] result = hash(key, 0); - //Should be: - long h1 = 0xe3301a827e5cdfe3L; - long h2 = 0xbdbf05f8da0f0392L; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - /** - * This test should have the exact same output as Test4 - */ - @Test - public void checkLongArrRemainderEQ8() { //long[], test a remainder = 8 - String keyStr = "The quick brown fox jumps over the lazy1"; - long[] key = stringToLongs(keyStr); - long[] result = hash(key, 0); - //Should be: - long h1 = 0xe3301a827e5cdfe3L; - long h2 = 0xbdbf05f8da0f0392L; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - /** - * This test should have the exact same output as Test4 - */ - @Test - public void checkIntArrRemainderEQ8() { //int[], test a remainder = 8 - String keyStr = "The quick brown fox jumps over the lazy1"; //40B - int[] key = stringToInts(keyStr); - long[] result = hash(key, 0); - //Should be: - long h1 = 0xe3301a827e5cdfe3L; - long h2 = 0xbdbf05f8da0f0392L; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - @Test - public void checkIntArrRemainderEQ0() { //int[], test a remainder = 0 - String keyStr = "The quick brown fox jumps over t"; //32B - int[] key = stringToInts(keyStr); - long[] result = hash(key, 0); - //Should be: - long h1 = 0xdf6af91bb29bdacfL; - long h2 = 0x91a341c58df1f3a6L; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - /** - * Tests an odd remainder of int[]. - */ - @Test - public void checkIntArrOddRemainder() { //int[], odd remainder - String keyStr = "The quick brown fox jumps over the lazy dog"; //43B - int[] key = stringToInts(keyStr); - long[] result = hash(key, 0); - //Should be: - long h1 = 0x1eb232b0087543f5L; - long h2 = 0xfc4c1383c3ace40fL; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - /** - * Tests an odd remainder of int[]. - */ - @Test - public void checkCharArrOddRemainder() { //char[], odd remainder - String keyStr = "The quick brown fox jumps over the lazy dog.."; //45B - char[] key = keyStr.toCharArray(); - long[] result = hash(key, 0); - //Should be: - long h1 = 0xca77b498ea9ed953L; - long h2 = 0x8b8f8ec3a8f4657eL; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - /** - * Tests an odd remainder of int[]. - */ - @Test - public void checkCharArrRemainderEQ0() { //char[], remainder of 0 - String keyStr = "The quick brown fox jumps over the lazy "; //40B - char[] key = keyStr.toCharArray(); - long[] result = hash(key, 0); - //Should be: - long h1 = 0x51b15e9d0887f9f1L; - long h2 = 0x8106d226786511ebL; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - @Test - public void checkByteArrAllOnesZeros() { //byte[], test a ones byte and a zeros byte - byte[] key = { - 0x54, 0x68, 0x65, 0x20, 0x71, 0x75, 0x69, 0x63, 0x6b, 0x20, 0x62, 0x72, 0x6f, 0x77, 0x6e, - 0x20, 0x66, 0x6f, 0x78, 0x20, 0x6a, 0x75, 0x6d, 0x70, 0x73, 0x20, 0x6f, 0x76, 0x65, - 0x72, 0x20, 0x74, 0x68, 0x65, 0x20, 0x6c, 0x61, 0x7a, 0x79, 0x20, 0x64, 0x6f, 0x67, - (byte) 0xff, 0x64, 0x6f, 0x67, 0x00 - }; - long[] result = MurmurHash3v4.hash(key, 0); - //Should be: - long h1 = 0xe88abda785929c9eL; - long h2 = 0x96b98587cacc83d6L; - Assert.assertEquals(result[0], h1); - Assert.assertEquals(result[1], h2); - } - - /** - * This test demonstrates that the hash of byte[], char[], int[], or long[] will produce the - * same hash result if, and only if, all the arrays have the same exact length in bytes, and if - * the contents of the values in the arrays have the same byte endianness and overall order. - */ - @Test - public void checkCrossTypeHashConsistency() { - long[] out; - println("Bytes"); - byte[] bArr = {1,2,3,4,5,6,7,8, 9,10,11,12,13,14,15,16, 17,18,19,20,21,22,23,24}; - long[] out1 = hash(bArr, 0L); - println(longToHexBytes(out1[0])); - println(longToHexBytes(out1[1])); - - println("Chars"); - char[] cArr = {0X0201, 0X0403, 0X0605, 0X0807, 0X0a09, 0X0c0b, 0X0e0d, 0X100f, - 0X1211, 0X1413, 0X1615, 0X1817}; - out = hash(cArr, 0L); - Assert.assertEquals(out, out1); - println(longToHexBytes(out[0])); - println(longToHexBytes(out[1])); - - println("Ints"); - int[] iArr = {0X04030201, 0X08070605, 0X0c0b0a09, 0X100f0e0d, 0X14131211, 0X18171615}; - out = hash(iArr, 0L); - Assert.assertEquals(out, out1); - println(longToHexBytes(out[0])); - println(longToHexBytes(out[1])); - - println("Longs"); - long[] lArr = {0X0807060504030201L, 0X100f0e0d0c0b0a09L, 0X1817161514131211L}; - out = hash(lArr, 0L); - Assert.assertEquals(out, out1); - println(longToHexBytes(out[0])); - println(longToHexBytes(out[1])); - } - - @Test - public void checkEmptyOrNullExceptions() { - try { - long[] arr = null; - hash(arr, 1L); - fail(); - } - catch (final IllegalArgumentException e) { } - try { - int[] arr = null; hash(arr, 1L); fail(); - } catch (final IllegalArgumentException e) { } - try { - char[] arr = null; hash(arr, 1L); fail(); - } catch (final IllegalArgumentException e) { } - try { - byte[] arr = null; hash(arr, 1L); fail(); - } catch (final IllegalArgumentException e) { } - - long[] out = new long[2]; - try { - String in = null; hash(in, 1L, out); fail(); - } catch (final IllegalArgumentException e) { } - try { - Memory mem = Memory.wrap(new byte[0]); - hash(mem, 0L, 4L, 1L, out); - } catch (final IllegalArgumentException e) { } - try (Arena arena = Arena.ofConfined()) { - Memory mem = WritableMemory.allocateDirect(8, 1, ByteOrder.nativeOrder(), memReqSvr, arena); - out = hash(mem, 0L, 4L, 1L, out); - } - assertTrue((out[0] != 0) && (out[1] != 0)); - } - - @Test - public void checkHashTails() { - long[] out = new long[2]; - WritableMemory mem = WritableMemory.allocate(32); - mem.fill((byte)85); - - for (int i = 16; i <= 32; i++) { - out = hash(mem, 0, i, 1L, out); - } - } - - @Test - public void checkSinglePrimitives() { - long[] out = new long[2]; - out = hash(1L, 1L, out); - out = hash(0.0, 1L, out); - out = hash("123", 1L, out); - } - - //Helper methods - - private static long[] stringToLongs(String in) { - byte[] bArr = in.getBytes(UTF_8); - int inLen = bArr.length; - int outLen = (inLen / 8) + (((inLen % 8) != 0) ? 1 : 0); - long[] out = new long[outLen]; - - for (int i = 0; i < (outLen - 1); i++ ) { - for (int j = 0; j < 8; j++ ) { - out[i] |= ((bArr[(i * 8) + j] & 0xFFL) << (j * 8)); - } - } - int inTail = 8 * (outLen - 1); - int rem = inLen - inTail; - for (int j = 0; j < rem; j++ ) { - out[outLen - 1] |= ((bArr[inTail + j] & 0xFFL) << (j * 8)); - } - return out; - } - - private static int[] stringToInts(String in) { - byte[] bArr = in.getBytes(UTF_8); - int inLen = bArr.length; - int outLen = (inLen / 4) + (((inLen % 4) != 0) ? 1 : 0); - int[] out = new int[outLen]; - - for (int i = 0; i < (outLen - 1); i++ ) { - for (int j = 0; j < 4; j++ ) { - out[i] |= ((bArr[(i * 4) + j] & 0xFFL) << (j * 8)); - } - } - int inTail = 4 * (outLen - 1); - int rem = inLen - inTail; - for (int j = 0; j < rem; j++ ) { - out[outLen - 1] |= ((bArr[inTail + j] & 0xFFL) << (j * 8)); - } - return out; - } - - /** - * Returns a string of spaced hex bytes in Big-Endian order. - * @param v the given long - * @return string of spaced hex bytes in Big-Endian order. - */ - private static String longToHexBytes(final long v) { - final long mask = 0XFFL; - final StringBuilder sb = new StringBuilder(); - for (int i = 8; i-- > 0; ) { - final String s = Long.toHexString((v >>> (i * 8)) & mask); - sb.append(zeroPad(s, 2)).append(" "); - } - return sb.toString(); - } - - /** - * Prepend the given string with zeros. If the given string is equal or greater than the given - * field length, it will be returned without modification. - * @param s the given string - * @param fieldLength desired total field length including the given string - * @return the given string prepended with zeros. - */ - private static final String zeroPad(final String s, final int fieldLength) { - return characterPad(s, fieldLength, '0', false); - } - - /** - * Prepend or postpend the given string with the given character to fill the given field length. - * If the given string is equal or greater than the given field length, it will be returned - * without modification. - * @param s the given string - * @param fieldLength the desired field length - * @param padChar the desired pad character - * @param postpend if true append the padCharacters to the end of the string. - * @return prepended or postpended given string with the given character to fill the given field - * length. - */ - private static final String characterPad(final String s, final int fieldLength, final char padChar, - final boolean postpend) { - final char[] chArr = s.toCharArray(); - final int sLen = chArr.length; - if (sLen < fieldLength) { - final char[] out = new char[fieldLength]; - final int blanks = fieldLength - sLen; - - if (postpend) { - for (int i = 0; i < sLen; i++) { - out[i] = chArr[i]; - } - for (int i = sLen; i < fieldLength; i++) { - out[i] = padChar; - } - } else { //prepend - for (int i = 0; i < blanks; i++) { - out[i] = padChar; - } - for (int i = blanks; i < fieldLength; i++) { - out[i] = chArr[i - blanks]; - } - } - - return String.valueOf(out); - } - return s; - } - - @Test - public void printlnTest() { - println("PRINTING: "+this.getClass().getName()); - } - - /** - * @param s value to print - */ - static void println(String s) { - //System.out.println(s); //disable here - } - -}