diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2c447056..0008f514 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,5 +45,7 @@ jobs: if: ${{ failure() }} uses: actions/upload-artifact@v4 with: - path: build/reports/tests/test - name: Test Report \ No newline at end of file + name: test-report-${{ matrix.os }}-${{ matrix.java }} + path: | + **/build/reports/tests/test/** + **/build/test-results/test/*.xml \ No newline at end of file diff --git a/build.gradle b/build.gradle index 031e85d9..776cd6c4 100644 --- a/build.gradle +++ b/build.gradle @@ -39,6 +39,8 @@ ext.leafProjects = leafProjects // Publish as external variable ext { sonaUser = findProperty('SONATYPE_NEXUS_USERNAME') ?: System.getenv('SONATYPE_NEXUS_USERNAME') ?: "FakeUser" sonaPass = findProperty('SONATYPE_NEXUS_PASSWORD') ?: System.getenv('SONATYPE_NEXUS_PASSWORD') ?: "FakePass" + + testTags = null // Default test tags } // Default test categories @@ -69,6 +71,9 @@ configure(leafProjects) { // Defines versions of dependencies to be used by subprojects // https://github.com/spring-gradle-plugins/dependency-management-plugin#dependency-management-dsl dependencyManagement { + imports { + mavenBom 'org.junit:junit-bom:5.14.0' + } dependencies { dependency ('com.epam.deltix:thread-affinity:1.0.4') @@ -97,6 +102,11 @@ configure(leafProjects) { entry 'jmh-core' entry 'jmh-generator-annprocess' } + + dependency 'org.jetbrains:annotations:24.1.0' + + dependency "junit:junit:4.13.2" + dependency "org.mockito:mockito-core:5.15.2" } } @@ -120,11 +130,29 @@ configure(leafProjects) { compileOnly 'com.google.code.findbugs:jsr305' compileOnly 'com.google.code.findbugs:annotations' - testImplementation 'junit:junit:4.12' + testCompileOnly 'junit:junit' + + testImplementation "org.mockito:mockito-core" + + testImplementation 'org.junit.jupiter:junit-jupiter-api' + testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' + testRuntimeOnly 'org.junit.vintage:junit-vintage-engine' + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' spotbugsPlugins 'com.h3xstream.findsecbugs:findsecbugs-plugin:1.9.0' } + test { + useJUnitPlatform { + if (testTags != null) { + setIncludeTags testTags.split(',') as Set + } + } + + // sync logging to not mix up messages from different tests + jvmArgs '-Dgflog.sync=true', '-Dgflog.console.appender.wrap=true'/*, '--add-opens', 'java.base/sun.nio.ch=ALL-UNNAMED'*/ + } + spotbugs { toolVersion = "3.1.12" effort = "max" diff --git a/collections/src/main/java/com/epam/deltix/util/collections/ByteQueue.java b/collections/src/main/java/com/epam/deltix/util/collections/ByteQueue.java index 6fe428d1..36223a8c 100644 --- a/collections/src/main/java/com/epam/deltix/util/collections/ByteQueue.java +++ b/collections/src/main/java/com/epam/deltix/util/collections/ByteQueue.java @@ -14,6 +14,7 @@ * License for the specific language governing permissions and limitations under * the License. */ + package com.epam.deltix.util.collections; import java.io.IOException; @@ -26,8 +27,8 @@ public final class ByteQueue implements ByteContainer { private int capacity; private byte [] buffer; private int size = 0; - private int head = 0; - private int tail = 0; + private int head = 0; // Where data gets consumed + private int tail = 0; // Where data gets added public ByteQueue (int capacity) { this.capacity = capacity; @@ -55,36 +56,6 @@ public void offer (byte value) { tail = 0; } -// public void insert (byte [] src, int offset, int length) { -// if (capacity - size < length) { -// // increase buffer -// if (tail > head) { -// byte[] temp = new byte[capacity + length]; -// System.arraycopy (buffer, head, temp, length + head, size); -// buffer = temp; -// } else { -// byte[] temp = new byte[capacity + length]; -// System.arraycopy (buffer, 0, temp, 0, tail); -// System.arraycopy (buffer, head, temp, capacity - head + length, capacity - head); -// buffer = temp; -// } -// capacity += length; -// } -// -// if (head > length) { -// System.arraycopy (src, offset, buffer, head - length, length); -// head -= length; -// } else { -// int remains = length - head; -// System.arraycopy (src, offset, buffer, capacity - remains, remains); -// System.arraycopy (src, offset + remains, buffer, 0, head); -// -// head = capacity - remains; -// } -// size += length; -// -// } - public void offer (byte [] src, int offset, int length) { assert size + length <= capacity : "size: " + size + "; length: " + length + "; capacity: " + capacity; @@ -93,16 +64,19 @@ public void offer (byte [] src, int offset, int length) { int excess = end - capacity; if (excess > 0) { + // Wrap over array end. + // Length of the first part. int n = capacity - tail; - System.arraycopy (src, offset, buffer, tail, n); - + + // Length of the second part. tail = length - n; System.arraycopy (src, offset + n, buffer, 0, tail); } else { - System.arraycopy (src, offset, buffer, tail, length); + // Single chunk copy + System.arraycopy (src, offset, buffer, tail, length); tail = excess == 0 ? 0 : end; } @@ -256,7 +230,7 @@ public void truncate(int length) { assert length < size; int newTail = tail - length; if (newTail < 0) - newTail =+ capacity; + newTail += capacity; size -= length; tail = newTail; @@ -267,13 +241,17 @@ public boolean setCapacity (int value) { return false; assert(value > capacity); - capacity = value; byte[] previous = buffer; - buffer = new byte[capacity]; + // Allocate new buffer before changing any other fields to avoid state corruption if allocation fails with OOM. + buffer = new byte[value]; + capacity = value; + if (tail > head) { + // No wrap System.arraycopy (previous, head, buffer, 0, size); } else { + // Wrapped System.arraycopy (previous, head, buffer, 0, previous.length - head); System.arraycopy (previous, 0, buffer, previous.length - head, tail); } @@ -283,4 +261,4 @@ public boolean setCapacity (int value) { return true; } -} \ No newline at end of file +} diff --git a/collections/src/main/templates/RingedList.vpp b/collections/src/main/templates/RingedList.vpp new file mode 100644 index 00000000..20cd7589 --- /dev/null +++ b/collections/src/main/templates/RingedList.vpp @@ -0,0 +1,193 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + #if( $name == "Object" ) +#set( $type = "ObjectType" ) +#end +package deltix.util.collections.generated; + +import java.util.AbstractList; +import java.util.Arrays; + +public class ${name}RingedList extends AbstractList<${name}> implements ${name}List { + + private static final int MAX_ARRAY_LENGTH = Integer.MAX_VALUE - 8; + + protected ${type}[] array; + + private int first, size; + + public ${name}RingedList() { + this(0x10); + } + + public ${name}RingedList(int capacity) { + if (capacity < 0) { + throw new IllegalArgumentException("Initial capacity (" + capacity + ") is negative"); + } + + array = new ${type}[capacity]; + first = size = 0; + } + + @Override + public int size() { + return size; + } + + @Override + public ${name} get(int index) { + return get${name}(index); + } + + @Override + public ${type} get${name}(int index) { + if (index > size) { + throw new IndexOutOfBoundsException(); + } + + return get${name}NoRangeCheck(index); + } + + @Override + public ${type} get${name}NoRangeCheck(int index) { + index += first; + index = index >= array.length ? index - array.length : index; + return array[index]; + } + + @Override + public boolean add(${type} x) { + ensureFreeSpace(1); + + int index = first + size++; + index = index >= array.length ? index - array.length : index; + array[index] = x; + return true; + } + + public void set(int index, ${type} element) { + if (index > size) { + throw new IndexOutOfBoundsException(); + } + + index += first; + index = index >= array.length ? index - array.length : index; + array[index] = element; + } + + public ${type} first() { + return array[first]; + } + + public ${type} last() { + return get(size - 1); + } + + public ${type} pop() { + ${type} res = array[first++]; + first = first >= array.length ? first - array.length : first; + --size; + + return res; + } + + @Override + public boolean contains(${type} elem) { + return indexOf(elem) >= 0; + } + + @Override + public int indexOf(${type} elem) { + throw new UnsupportedOperationException(); + } + + @Override + public int lastIndexOf(${type} elem) { + throw new UnsupportedOperationException(); + } + + @Override + public ${type}[] to${name_abbr}Array() { + ${type}[] result = new ${type}[size]; + toArray(result, 0); + return result; + } + + public void toArray(${type}[] data, int offset) { + if (size > data.length - offset) { + throw new IllegalArgumentException(); + } + + for (int i = 0, j = first; i < size; ++i) { + data[i] = array[j++]; + j = j == array.length ? 0 : j; + } + } + + private void ensureFreeSpace(int required) { + long requiredCapacity = (long) size + required; + ensureCapacity(requiredCapacity); + } + + public void ensureCapacity(long minCapacity) { + if (minCapacity > array.length) { + extend(minCapacity); + } + } + + private void extend(long requiredCapacity) { + if (requiredCapacity > MAX_ARRAY_LENGTH) { + throw new OutOfMemoryError("required capacity=" + requiredCapacity + " exceeds max"); + } + + int newCapacity = (int) Math.min(MAX_ARRAY_LENGTH, requiredCapacity + (requiredCapacity >>> 1)); // 50% for growth + + ${type}[] newArray = new ${type}[newCapacity]; + + System.arraycopy(array, first, newArray, 0, array.length - first); + if (first > 0) { + System.arraycopy(array, 0, newArray, array.length - first, first); + } + first = 0; + array = newArray; + } + + @Override + public void clear() { + first = size= 0; + } + + @Override + public void setSize(int newSize) { + ensureCapacity(newSize); + size = newSize; + } + +#if ($type != "boolean") + @Override + public void sort() { + array = to${name_abbr}Array(); + first = 0; + Arrays.sort(array, 0, size); + } +#end + +} + +#if( $name == "Object") +#set( $type = "Object" ) +#end \ No newline at end of file diff --git a/collections/src/test/java/com/epam/deltix/util/collections/Test_ByteQueue.java b/collections/src/test/java/com/epam/deltix/util/collections/Test_ByteQueue.java new file mode 100644 index 00000000..0cbf8803 --- /dev/null +++ b/collections/src/test/java/com/epam/deltix/util/collections/Test_ByteQueue.java @@ -0,0 +1,248 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.collections; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class Test_ByteQueue { + + /** + * Reproduces logic from {@link com.epam.deltix.util.vsocket.VSocketOutputStream#dumpInternal} + */ + @Test + void testGrowForVSocketOutputStream() { + ByteQueue queue = new ByteQueue(524288); + byte[] arr1 = new byte[524288]; + queue.offer(arr1, 0, arr1.length); + + byte[] arr2 = new byte[2]; + + int overflow = queue.size() + arr2.length - queue.capacity(); + if (overflow > 0) { + int increment = 1024 * 512 / 4; + int incrementsToAdd = divideRoundUp(overflow, increment); + queue.addCapacity(increment * incrementsToAdd); + } + + queue.offer(arr2, 0, arr2.length); + assertEquals(524290, queue.size()); + } + + + static int divideRoundUp(int val, int divisor) { + int result = val / divisor; + if (val > result * divisor) { + result += 1; + } + return result; + } + + // region Generated Tests + + @Test + void testOfferArrayContiguous() { + // Setup: Capacity 10 + ByteQueue queue = new ByteQueue(10); + + byte[] input = new byte[] { 1, 2, 3, 4, 5 }; + + // Act: Offer 5 bytes (fits comfortably at start) + // Branches: excess <= 0, tail < capacity + queue.offer(input, 0, 5); + + // Assert + assertEquals(5, queue.size()); + assertEquals(0, queue.getHead()); + assertEquals(5, queue.getTail()); + assertEquals(1, queue.get(0)); + assertEquals(5, queue.get(4)); + } + + @Test + void testOfferArrayWrapAround() { + ByteQueue queue = new ByteQueue(10); + + // Move tail to index 8 (fill 8 bytes) + queue.offer(new byte[8], 0, 8); + // Move head to index 4 (poll 4 bytes) -> Size is now 4, Tail is 8 + queue.poll(new byte[4], 0, 4); + + assertEquals(4, queue.size()); + assertEquals(8, queue.getTail()); + + // Act: Offer 4 bytes + // Logic: tail(8) + length(4) = 12. Capacity is 10. + // excess = 2. This triggers the "if (excess > 0)" split branch. + byte[] input = new byte[] { 10, 11, 12, 13 }; + queue.offer(input, 0, 4); + + // Assert + assertEquals(8, queue.size()); // 4 existing + 4 new + assertEquals(2, queue.getTail()); // Wrapped: (8 + 4) % 10 = 2 + + // Verify data integrity across wrap + // Index 0 in queue (logical) is at buffer[4] + // New data starts at logical index 4 + assertEquals(10, queue.get(4)); + assertEquals(11, queue.get(5)); + assertEquals(12, queue.get(6)); + assertEquals(13, queue.get(7)); + } + + @Test + void testOfferArrayExactFitAtEnd() { + ByteQueue queue = new ByteQueue(10); + + queue.offer(new byte[5], 0, 5); + + // Act: Offer exactly 5 bytes to fill the buffer to the max capacity. + // Logic: tail(5) + length(5) = 10. excess = 0. + byte[] input = new byte[] { 1, 2, 3, 4, 5 }; + queue.offer(input, 0, 5); + + assertEquals(10, queue.size()); + assertEquals(0, queue.getTail()); // Should wrap exactly to 0 + assertTrue(queue.isFull()); + + queue.skip(2); + assertEquals(8, queue.size()); + + queue.offer(new byte[] {6, 7}, 0, 2); + assertEquals(10, queue.size()); + } + + @Test + void testSkipSimple() { + ByteQueue queue = new ByteQueue(10); + queue.offer(new byte[] { 1, 2, 3, 4, 5 }, 0, 5); + + // Act: Skip 2 bytes + // Branches: head + length < capacity + queue.skip(2); + + // Assert + assertEquals(3, queue.size()); + assertEquals(2, queue.getHead()); // Head moved from 0 to 2 + assertEquals(3, queue.get(0)); // Logical 0 is now the old 2 + } + + @Test + void testSkipWrapAround() { + ByteQueue queue = new ByteQueue(10); + + // Arrange: Wrap the data. Tail at 2, Head at 8. Size 4. + // We do this by filling 10, then polling 8, then offering 2. + queue.offer(new byte[10], 0, 10); + queue.poll(new byte[8], 0, 8); // Head is now 8 + queue.offer(new byte[] { 99, 100 }, 0, 2); // Tail is now 2 + + assertEquals(8, queue.getHead()); + assertEquals(4, queue.size()); + + // Act: Skip 3 bytes. + // Logic: Head(8) + 3 = 11. Capacity 10. + // Branch: if (head >= capacity) head -= capacity + queue.skip(3); + + // Assert + assertEquals(1, queue.size()); // 4 - 3 = 1 left + assertEquals(1, queue.getHead()); // (8 + 3) - 10 = 1 + } + + @Test + void testSetCapacityIncreaseContiguous() { + ByteQueue queue = new ByteQueue(5); + queue.offer(new byte[] { 1, 2, 3 }, 0, 3); + + // Act: Increase capacity + // Branch: tail > head (3 > 0) + boolean changed = queue.setCapacity(10); + + // Assert + assertTrue(changed); + assertEquals(10, queue.capacity()); + assertEquals(3, queue.size()); + assertEquals(1, queue.get(0)); // Data preserved + + // Verify we can actually use the new space + queue.offer(new byte[6], 0, 6); // Should fit (3 + 6 = 9 <= 10) + assertEquals(9, queue.size()); + } + + @Test + void testSetCapacityIncreaseWrapped() { + ByteQueue queue = new ByteQueue(5); + // Wrap: Head 3, Tail 2, Size 4 + queue.offer(new byte[5], 0, 5); // Full + queue.poll(new byte[3], 0, 3); // Head 3, Size 2 + queue.offer(new byte[] { 10, 11 }, 0, 2); // Tail 2, Size 4 + + assertEquals(3, queue.getHead()); + assertEquals(2, queue.getTail()); + + // Act: Increase capacity to 10 + // Branch: tail <= head (2 <= 3). Must linearize buffer. + queue.setCapacity(10); + + // Assert + assertEquals(10, queue.capacity()); + assertEquals(4, queue.size()); + assertEquals(0, queue.getHead()); // Linearized -> Head 0 + assertEquals(4, queue.getTail()); // Linearized -> Tail 4 + + // Verify internal data consistency + // Old buffer logical order: [index 3, index 4, index 0, index 1] + // Logic values should be preserved + // We pushed 5 empty bytes, then pulled 3. Remaining: 2 empty bytes. + // Then pushed 10, 11. + // Queue content: [0, 0, 10, 11] + assertEquals(0, queue.get(0)); + assertEquals(0, queue.get(1)); + assertEquals(10, queue.get(2)); + assertEquals(11, queue.get(3)); + } + + @Test + void testSetCapacityResizesBufferCorrectly() { + ByteQueue queue = new ByteQueue(10); + queue.setCapacity(20); + + // Assertion 1: Variable is updated + assertEquals(20, queue.capacity()); + + // Assertion 2: Internal buffer is actually updated + // We can check this by filling it beyond the old capacity + byte[] largeData = new byte[15]; + Arrays.fill(largeData, (byte) 1); + + // If buffer was still size 10, this would throw ArrayIndexOutOfBoundsException + assertDoesNotThrow(() -> { + queue.offer(largeData, 0, 15); + }, "Buffer should be resized to accommodate new capacity"); + + assertEquals(15, queue.size()); + } + + // endregion +} diff --git a/collections/src/test/java/com/epam/deltix/util/collections/Test_LongToObjectHashMap.java b/collections/src/test/java/com/epam/deltix/util/collections/Test_LongToObjectHashMap.java new file mode 100644 index 00000000..60ceb114 --- /dev/null +++ b/collections/src/test/java/com/epam/deltix/util/collections/Test_LongToObjectHashMap.java @@ -0,0 +1,85 @@ +///* +// * Copyright 2021 EPAM Systems, Inc +// * +// * See the NOTICE file distributed with this work for additional information +// * regarding copyright ownership. Licensed under the Apache License, +// * Version 2.0 (the "License"); you may not use this file except in compliance +// * with the License. You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// * License for the specific language governing permissions and limitations under +// * the License. +// */ +// +//package com.epam.deltix.util.collections; +// +//import com.epam.deltix.util.collections.generated.LongToObjectHashMap; +// +//import org.junit.Test; +//import static org.junit.Assert.*; +// +//public class Test_LongToObjectHashMap { +// private final static int INITIAL_CAPACITY = 4; +// private final LongToObjectHashMap map = new LongToObjectHashMap<>(INITIAL_CAPACITY); +// +// @Test +// public void testComputeIfAbsent1() { +// map.put(1, "ONE"); +// +// assertEquals("ONE", map.computeIfAbsent(1, +// (key)->{ +// fail("Not supposed to be called - key is present!"); +// return null; +// }) +// ); +// } +// +// +// @Test +// public void testComputeIfAbsent2() { +// map.put(1, "ONE"); +// +// assertEquals("TWO", map.computeIfAbsent(2, +// (key)->{ +// assertEquals(2, key); +// return "TWO"; +// }) +// ); +// } +// +// @Test +// public void testComputeIfAbsent3() { +// assertEquals("ONE", map.computeIfAbsent(1, +// (key)->{ +// assertEquals(1, key); +// return "ONE"; +// }) +// ); +// +// assertEquals("ONE", map.get(1, null)); +// +// assertEquals("ONE", map.computeIfAbsent(1, +// (key)->{ +// fail("Not supposed to be called - key is present!"); +// return null; +// }) +// ); +// } +// +// @Test +// public void testComputeReturnNull() { +// assertNull(map.computeIfAbsent(1, (key)->null)); +// assertNull(map.get(1, null)); +// } +// +// @Test +// public void testComputeIfAbsent4() { +// for (int i=0; i < INITIAL_CAPACITY*10; i++) { +// assertEquals("ValueFor#" + i, map.computeIfAbsent(i, (key) -> "ValueFor#" + key)); +// } +// } +//} diff --git a/collections/src/test/java/com/epam/deltix/util/collections/Test_ObjectToObjectHashMapRemove.java b/collections/src/test/java/com/epam/deltix/util/collections/Test_ObjectToObjectHashMapRemove.java new file mode 100644 index 00000000..7b96dfaf --- /dev/null +++ b/collections/src/test/java/com/epam/deltix/util/collections/Test_ObjectToObjectHashMapRemove.java @@ -0,0 +1,158 @@ +///* +// * Copyright 2021 EPAM Systems, Inc +// * +// * See the NOTICE file distributed with this work for additional information +// * regarding copyright ownership. Licensed under the Apache License, +// * Version 2.0 (the "License"); you may not use this file except in compliance +// * with the License. You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// * License for the specific language governing permissions and limitations under +// * the License. +// */ +// +//package com.epam.deltix.util.collections; +// +//import com.epam.deltix.util.collections.generated.ObjectToObjectHashMap; +//import org.junit.Assert; +//import org.junit.Test; +// +//import java.util.SortedSet; +//import java.util.TreeSet; +//import java.util.function.BiPredicate; +// +//import static org.junit.Assert.*; +// +//public class Test_ObjectToObjectHashMapRemove { +// private final ObjectToObjectHashMap map = new ObjectToObjectHashMap<>(); +// +// @Test +// public void testRemoveByKey () { +// map.put("1", "one"); +// map.put("2", "two"); +// map.put("3", "three"); +// +// assertContent("one, two, three"); +// +// map.remove("2"); +// +// assertContent("one, three"); +// +// map.remove("X"); +// +// assertContent("one, three"); +// } +// +// @Test +// public void testRemoveByValue () { +// map.put("1", "one"); +// map.put("2", "two"); +// map.put("3", "three"); +// +// assertNull(map.remove((value) -> value.equals("unknown"))); // no such value +// assertContent("one, two, three"); +// +// assertNotNull(map.remove((value) -> value.equals("two"))); +// assertContent("one, three"); +// +// assertNotNull(map.remove((value) -> value.equals("three"))); +// assertContent("one"); +// +// assertNull(map.remove((value) -> value.equals("three"))); // again? +// assertContent("one"); +// +// assertNotNull(map.remove((value) -> value.equals("one"))); +// assertContent(""); +// +// assertNull(map.remove((value) -> value.equals("one"))); // again? +// assertContent(""); +// } +// +// @Test +// public void testRemoveByValueWithCookie () { +// BiPredicate filter = String::equals; +// +// map.put("1", "one"); +// map.put("2", "two"); +// map.put("3", "three"); +// +// assertNull(map.remove(filter, "unknown")); // no such value +// assertContent("one, two, three"); +// +// assertNotNull(map.remove(filter, "two")); +// assertContent("one, three"); +// +// assertNotNull(map.remove(filter, "three")); +// assertContent("one"); +// +// assertNull(map.remove(filter, "three")); // again? +// assertContent("one"); +// +// assertNotNull(map.remove(filter, "one")); +// assertContent(""); +// +// assertNull(map.remove(filter, "one")); // again? +// assertContent(""); +// } +// +// @Test +// public void testRemoveAllByValue () { +// map.put("1", "one"); +// map.put("2", "two"); +// map.put("3", "three"); +// +// assertFalse(map.removeAll((value) -> value.equals("unknown"))); // no such value +// assertContent("one, two, three"); +// +// assertTrue(map.removeAll((value) -> value.equals("two"))); +// assertContent("one, three"); +// +// assertTrue(map.removeAll((value) -> value.equals("three"))); +// assertContent("one"); +// +// assertFalse(map.removeAll((value) -> value.equals("three"))); // again? +// assertContent("one"); +// +// assertTrue(map.removeAll((value) -> value.equals("one"))); +// assertContent(""); +// +// assertFalse(map.removeAll((value) -> value.equals("one"))); // again? +// assertContent(""); +// +// map.put("1", "one"); +// map.put("2", "two"); +// map.put("2a", "two"); +// map.put("2b", "two"); +// map.put("3", "three"); +// assertContent("one, two, two, two, three"); +// +// assertTrue(map.removeAll((value) -> value.equals("two"))); +// assertContent("one, three"); +// } +// +// private void assertContent(String expectedDump) { +// String actualDump = dump(); +// Assert.assertEquals(expectedDump, actualDump); +// } +// +// private String dump() { +// StringBuilder result = new StringBuilder(); +// +// ElementsEnumeration keys = map.keys(); +// SortedSet sortedKeys = new TreeSet<>(); +// while(keys.hasMoreElements()) +// sortedKeys.add(keys.nextElement()); +// +// for (String key : sortedKeys) { +// if (result.length() > 0) +// result.append(", "); +// +// result.append(map.get(key, null)); +// } +// return result.toString(); +// } +//} diff --git a/collections/src/test/java/com/epam/deltix/util/collections/Test_RingedList.java b/collections/src/test/java/com/epam/deltix/util/collections/Test_RingedList.java new file mode 100644 index 00000000..f8c1b5af --- /dev/null +++ b/collections/src/test/java/com/epam/deltix/util/collections/Test_RingedList.java @@ -0,0 +1,130 @@ +///* +// * Copyright 2021 EPAM Systems, Inc +// * +// * See the NOTICE file distributed with this work for additional information +// * regarding copyright ownership. Licensed under the Apache License, +// * Version 2.0 (the "License"); you may not use this file except in compliance +// * with the License. You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// * License for the specific language governing permissions and limitations under +// * the License. +// */ +// +//package com.epam.deltix.util.collections; +// +//import com.epam.deltix.util.collections.generated.LongRingedList; +//import org.junit.Assert; +//import org.junit.Test; +// +//import java.util.ArrayDeque; +//import java.util.Random; +// +//public class Test_RingedList { +// +// @Test +// public void testAddRemove() { +// LongRingedList queue = new LongRingedList(); +// for (int i = 0; i < 100_000; ++i) { +// queue.add(i); +// } +// for (int i = 0; i < 50_000; ++i) { +// queue.pop(); +// } +// for (int i = 0; i < 25_000; ++i) { +// queue.add(i); +// } +// Assert.assertEquals(75_000, queue.size()); +// +// for (int i = 50_000, j = 0; i < 100_000; ++i, ++j) { +// Assert.assertEquals(i, queue.getLong(j)); +// } +// for (int i = 0, j = 50_000; i < 25_000; ++i, ++j) { +// Assert.assertEquals(i, queue.getLong(j)); +// } +// } +// +// @Test +// public void testPrimitiveStress() { +// LongRingedList queue = new LongRingedList(); +// ArrayDeque etalon = new ArrayDeque<>(); +// +// Random rand = new Random(); +// int iterations = 5_000_000; +// for (int i = 0; i < iterations; ++i) { +// int type = Math.abs(rand.nextInt()) % 4; +// if (Math.abs(type) == 0 && queue.size() > 0) { +// // pop and compare +// Assert.assertEquals(etalon.removeFirst().longValue(), queue.pop()); +// } else { +// // add random value +// long newValue = rand.nextLong(); +// queue.add(newValue); +// etalon.addLast(newValue); +// } +// +// if (i % 100_000 == 0) { +// // compare queues +// int index = 0; +// Assert.assertEquals(etalon.size(), queue.size()); +// Long[] etalonArray = etalon.toArray(new Long[0]); +// for (Long element : queue) { +// Assert.assertEquals(etalonArray[index], element); +// Assert.assertEquals(element, queue.get(index)); +// Assert.assertEquals(element.longValue(), queue.getLong(index)); +// index++; +// } +// } +// +// if (i % 300_001 == 0) { +// // drain queues +// Assert.assertEquals(etalon.size(), queue.size()); +// int size = etalon.size(); +// for (int j = 0; j < size; ++j) { +// Assert.assertEquals(etalon.pop().longValue(), queue.pop()); +// } +// +// Assert.assertEquals(etalon.size(), 0); +// Assert.assertEquals(queue.size(), 0); +// } +// } +// } +// +// @Test +// public void testSetSize() { +// LongRingedList queue = new LongRingedList(); +// for (int i = 0; i < 100_000; ++i) { +// queue.add(i); +// } +// Assert.assertEquals(100_000, queue.size()); +// +// queue.setSize(1_000_000); +// Assert.assertEquals(1_000_000, queue.size()); +// for (int i = 0; i < 100_000; ++i) { +// Assert.assertEquals(i, queue.getLong(i)); +// } +// for (int i = 100_000; i < 1_000_000; ++i) { +// Assert.assertEquals(0, queue.getLong(i)); +// } +// } +// +// @Test +// public void testSort() { +// LongRingedList queue = new LongRingedList(); +// for (int i = 1; i <= 10_000; ++i) { +// queue.add(10_000 - i); +// } +// Assert.assertEquals(10_000, queue.size()); +// +// queue.sort(); +// Assert.assertEquals(queue.size(), 10_000); +// for (int i = 0; i < 10_000; ++i) { +// Assert.assertEquals(i, queue.getLong(i)); +// } +// } +// +//} diff --git a/lang/build.gradle b/lang/build.gradle index 14700453..9af2d96d 100644 --- a/lang/build.gradle +++ b/lang/build.gradle @@ -11,6 +11,8 @@ dependencies { implementation group: 'com.epam.deltix', name: 'gflog-api', version: gflogVersion implementation group: 'com.epam.deltix', name: 'gflog-core', version: gflogVersion + testImplementation 'org.junit.jupiter:junit-jupiter-params' + // For JMH-based tests testImplementation 'org.openjdk.jmh:jmh-core' testImplementation 'org.openjdk.jmh:jmh-generator-annprocess' diff --git a/lang/src/main/java/com/epam/deltix/util/LangUtil.java b/lang/src/main/java/com/epam/deltix/util/LangUtil.java index b6c0ef60..0196a5c7 100644 --- a/lang/src/main/java/com/epam/deltix/util/LangUtil.java +++ b/lang/src/main/java/com/epam/deltix/util/LangUtil.java @@ -14,17 +14,40 @@ * License for the specific language governing permissions and limitations under * the License. */ + package com.epam.deltix.util; public final class LangUtil { - + // This method is preserved for binary compatibility with older versions public static RuntimeException rethrowUnchecked(Exception e) { throw LangUtil.throwUnchecked(e); } + public static RuntimeException rethrowUnchecked(Throwable e) { + throw LangUtil.throwUnchecked(e); + } + @SuppressWarnings("unchecked") - private static T throwUnchecked(Exception e) throws T { + private static T throwUnchecked(Throwable e) throws T { throw (T) e; } -} \ No newline at end of file + /** + * If provided throwable is not an {@link Exception}, rethrows it using "sneaky throw". + *

+ * This way major errors like {@link OutOfMemoryError} or {@link StackOverflowError} are propagated out. + *

+ * This method mainly intended to be used in catch blocks that catch {@link Throwable}. + * Normally, only {@link Exception} instances should be caught. + * However, we have a lot of legacy code that catches {@link Throwable} and suppresses it without rethrowing (possibly with logging). + * With this method, we can fix such code to rethrow major errors without major refactoring. + *

+ * Additionally, this method is useful when we use global default exception handler ({@link Thread#setDefaultUncaughtExceptionHandler}) + * to detect OOM events. So OOM would be propagated to such handler even if caught by existing "catch Throwable" blocks. + */ + public static void propagateError(Throwable e) { + if (!(e instanceof Exception)) { + throwUnchecked(e); + } + } +} diff --git a/messages/src/main/java/com/epam/deltix/timebase/messages/SchemaStaticType.java b/lang/src/main/java/com/epam/deltix/util/annotations/Duration.java similarity index 67% rename from messages/src/main/java/com/epam/deltix/timebase/messages/SchemaStaticType.java rename to lang/src/main/java/com/epam/deltix/util/annotations/Duration.java index 5a6c3e89..efbbc098 100644 --- a/messages/src/main/java/com/epam/deltix/timebase/messages/SchemaStaticType.java +++ b/lang/src/main/java/com/epam/deltix/util/annotations/Duration.java @@ -14,19 +14,24 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.epam.deltix.timebase.messages; + +package com.epam.deltix.util.annotations; import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import java.util.concurrent.TimeUnit; +/** + * Marks field/parameter/local-variable/method-return-type of integer type which should be considered as duration. + */ +@Target({ElementType.FIELD, ElementType.PARAMETER, ElementType.LOCAL_VARIABLE, ElementType.METHOD}) @Retention(RetentionPolicy.RUNTIME) -@Target( { - ElementType.METHOD, ElementType.FIELD} -) -public @interface SchemaStaticType { - String value(); +@Inherited +public @interface Duration { + + TimeUnit timeUnit() default TimeUnit.MILLISECONDS; - SchemaDataType dataType() default SchemaDataType.DEFAULT; -} +} \ No newline at end of file diff --git a/lang/src/main/java/com/epam/deltix/util/lang/Util.java b/lang/src/main/java/com/epam/deltix/util/lang/Util.java index 51f2eb67..3277e74b 100644 --- a/lang/src/main/java/com/epam/deltix/util/lang/Util.java +++ b/lang/src/main/java/com/epam/deltix/util/lang/Util.java @@ -14,6 +14,7 @@ * License for the specific language governing permissions and limitations under * the License. */ + package com.epam.deltix.util.lang; import com.epam.deltix.gflog.api.Log; @@ -30,6 +31,7 @@ import java.util.prefs.Preferences; /** Set of useful methods */ +@SuppressWarnings("unused") public class Util { public static final boolean IS64BIT = "64".equals(System.getProperty("sun.arch.data.model")); public static final boolean IS32BIT = "32".equals(System.getProperty("sun.arch.data.model")); @@ -94,7 +96,7 @@ public static int doubleUntilAtLeast (int a, int limit) { // Double value a = a << 1; } - + return (a); } @@ -552,9 +554,7 @@ public static T newInstanceNoX ( return (newInstance (clazz, args)); } catch (RuntimeException x) { throw x; - } catch (Error x) { - throw x; - } catch (Throwable other) { + } catch (Exception other) { throw new RuntimeException (clazz.getName () + " instantiation failed", other); } } @@ -568,9 +568,7 @@ public static Object newInstanceNoX ( return (newInstance (className, args)); } catch (RuntimeException x) { throw x; - } catch (Error x) { - throw x; - } catch (Throwable other) { + } catch (Exception other) { throw new RuntimeException (className + " instantiation failed", other); } } @@ -1225,7 +1223,7 @@ public static T findCause(Throwable error, Class causeC } c = c.getSuperclass(); } - return result.toArray(new Class [result.size()]); + return result.toArray(new Class[0]); } /** @return true if given cls is instanceof interface specified by className */ @@ -1646,4 +1644,4 @@ public static String getSimpleName (String className) { return (dot < 0 ? className : className.substring (dot + 1)); } -} \ No newline at end of file +} diff --git a/lang/src/main/java/com/epam/deltix/util/memory/MemoryDataInput.java b/lang/src/main/java/com/epam/deltix/util/memory/MemoryDataInput.java index eb95ddd1..3efca42d 100644 --- a/lang/src/main/java/com/epam/deltix/util/memory/MemoryDataInput.java +++ b/lang/src/main/java/com/epam/deltix/util/memory/MemoryDataInput.java @@ -14,6 +14,7 @@ * License for the specific language governing permissions and limitations under * the License. */ + package com.epam.deltix.util.memory; import com.epam.deltix.dfp.Decimal64Utils; @@ -133,15 +134,52 @@ public int getCurrentOffset () { return (mPos); } + /** + * This method uses AssertionError as a way to signal insufficient data. + * This is bad practice, as AssertionError is generally intended for detecting programming errors, + * not for handling runtime conditions like insufficient data. + *

+ * Thus, this method is deprecated and should be avoided in favor of proper exception handling. + * Use {@link #hasAvailable(int)} to check availability before reading. + * If you need to enforce availability, consider using {@link #ensureAvailable(int)} which throws IllegalStateException. + */ + @Deprecated public boolean checkAvailable (int n) { - if (getAvail () < n) - throw new AssertionError ("Cannot read " + n + " bytes; available: " + getAvail ()); + return assertAvailable(n); + } + + /** + * Should be used only for assertions. + */ + private boolean assertAvailable(int n) { + if (getAvail() < n) { + throw new AssertionError("Cannot read " + n + " bytes; available: " + getAvail()); + } + return true; + } + + /** + * Ensures that at least n bytes are available for reading. + * Throws IllegalStateException if not enough bytes are available. + *

+ * Same as {@link #checkAvailable(int)} but uses a more appropriate exception type. + */ + public void ensureAvailable(int n) { + int avail = getAvail(); + if (avail < n) { + throw new IllegalStateException("Cannot read " + n + " bytes; available: " + avail); + } + } - return (true); + /** + * Checks if at least n bytes are available for reading. + */ + public boolean hasAvailable(int n) { + return getAvail() >= n; } public void readFully (byte[] b, int off, int len) { - assert checkAvailable (len); + assert assertAvailable (len); System.arraycopy (mBuffer, mPos, b, off, len); mPos += len; @@ -152,7 +190,7 @@ public void readFully (byte[] b) { } public void skipBytes (int n) { - assert checkAvailable (n); + assert assertAvailable (n); mPos += n; } @@ -181,7 +219,7 @@ public void seek (int n) { } public int readUnsignedShort () { - assert checkAvailable (2); + assert assertAvailable (2); int ret = DataExchangeUtils.readUnsignedShort (mBuffer, mPos); mPos += 2; @@ -189,7 +227,7 @@ public int readUnsignedShort () { } public long readUnsignedInt () { - assert checkAvailable (4); + assert assertAvailable (4); long ret = DataExchangeUtils.readUnsignedInt (mBuffer, mPos); mPos += 4; @@ -197,25 +235,25 @@ public long readUnsignedInt () { } public int readUnsignedByte () { - assert checkAvailable (1); + assert assertAvailable (1); return (mBuffer [mPos++] & 0xFF); } public boolean readBoolean () { - assert checkAvailable (1); + assert assertAvailable (1); return (mBuffer [mPos++] == 1); } public byte readByte () { - assert checkAvailable (1); + assert assertAvailable (1); return (mBuffer [mPos++]); } public char readChar () { - assert checkAvailable (2); + assert assertAvailable (2); char ret = DataExchangeUtils.readChar (mBuffer, mPos); mPos += 2; @@ -223,7 +261,7 @@ public char readChar () { } public double readDouble () { - assert checkAvailable (8); + assert assertAvailable (8); double ret = DataExchangeUtils.readDouble (mBuffer, mPos); mPos += 8; @@ -231,7 +269,7 @@ public double readDouble () { } public float readFloat () { - assert checkAvailable (4); + assert assertAvailable (4); float ret = DataExchangeUtils.readFloat (mBuffer, mPos); mPos += 4; @@ -239,7 +277,7 @@ public float readFloat () { } public int readInt () { - assert checkAvailable (4); + assert assertAvailable (4); int ret = DataExchangeUtils.readInt (mBuffer, mPos); mPos += 4; @@ -247,7 +285,7 @@ public int readInt () { } public long readLong () { - assert checkAvailable (8); + assert assertAvailable (8); long ret = DataExchangeUtils.readLong (mBuffer, mPos); mPos += 8; @@ -255,7 +293,7 @@ public long readLong () { } public long readLong48 () { - assert checkAvailable (6); + assert assertAvailable (6); long ret = DataExchangeUtils.readLong48 (mBuffer, mPos); mPos += 6; @@ -263,7 +301,7 @@ public long readLong48 () { } public long readLongUnsignedByte () { - assert checkAvailable (1); + assert assertAvailable (1); return (((long) mBuffer [mPos++]) & 0xFFL); } @@ -359,7 +397,7 @@ public int readPackedUnsignedInt () { } public short readShort () { - assert checkAvailable (2); + assert assertAvailable (2); short ret = DataExchangeUtils.readShort (mBuffer, mPos); mPos += 2; @@ -538,4 +576,4 @@ public double readScaledDouble () { return (lv / scale); } -} \ No newline at end of file +} diff --git a/messages/src/main/java/com/epam/deltix/timebase/messages/IdentityKey.java b/messages/src/main/java/com/epam/deltix/timebase/messages/IdentityKey.java index e6b773a8..a9da9467 100644 --- a/messages/src/main/java/com/epam/deltix/timebase/messages/IdentityKey.java +++ b/messages/src/main/java/com/epam/deltix/timebase/messages/IdentityKey.java @@ -25,5 +25,5 @@ public interface IdentityKey { * Identity name. * @return String representation of identity name. */ - public CharSequence getSymbol (); + CharSequence getSymbol (); } \ No newline at end of file diff --git a/messages/src/main/java/com/epam/deltix/timebase/messages/IdentityKeyInterface.java b/messages/src/main/java/com/epam/deltix/timebase/messages/IdentityKeyInterface.java index a354b527..1688eb30 100644 --- a/messages/src/main/java/com/epam/deltix/timebase/messages/IdentityKeyInterface.java +++ b/messages/src/main/java/com/epam/deltix/timebase/messages/IdentityKeyInterface.java @@ -19,26 +19,26 @@ /** */ public interface IdentityKeyInterface extends IdentityKey { - /** - * Instrument name. - * @return Symbol - */ - CharSequence getSymbol(); + /** + * Instrument name. + * @return Symbol + */ + CharSequence getSymbol(); - /** - * Instrument name. - * @return true if Symbol is not null - */ - boolean hasSymbol(); + /** + * Instrument name. + * @return true if Symbol is not null + */ + boolean hasSymbol(); - /** - * Instrument name. - * @param value - Symbol - */ - void setSymbol(CharSequence value); + /** + * Instrument name. + * @param value - Symbol + */ + void setSymbol(CharSequence value); - /** - * Instrument name. - */ - void nullifySymbol(); + /** + * Instrument name. + */ + void nullifySymbol(); } diff --git a/messages/src/main/java/com/epam/deltix/timebase/messages/SchemaGuid.java b/messages/src/main/java/com/epam/deltix/timebase/messages/SchemaGuid.java index 85762907..bd046565 100644 --- a/messages/src/main/java/com/epam/deltix/timebase/messages/SchemaGuid.java +++ b/messages/src/main/java/com/epam/deltix/timebase/messages/SchemaGuid.java @@ -23,8 +23,8 @@ @Retention(RetentionPolicy.RUNTIME) @Target( { - ElementType.TYPE} + ElementType.TYPE} ) public @interface SchemaGuid { - String value(); + String value(); } diff --git a/util/build.gradle b/util/build.gradle index fc2b47c1..561e3437 100644 --- a/util/build.gradle +++ b/util/build.gradle @@ -1,16 +1,22 @@ description = 'Timebase general purpose utilities' +configurations { + antlrGenerator +} + dependencies { api project(':timebase-lang') api project(':timebase-collections') + compileOnly 'org.jetbrains:annotations' + implementation group: 'com.epam.deltix', name: 'gflog-api', version: gflogVersion implementation group: 'com.epam.deltix', name: 'gflog-core', version: gflogVersion implementation group: 'com.epam.deltix', name: 'dfp', version: dfpVersion implementation group: 'com.epam.deltix', name: 'hd-date-time', version: hdTimeVersion - implementation 'com.epam.deltix:thread-affinity:1.0.4' + implementation 'com.epam.deltix:thread-affinity' implementation 'org.apache.commons:commons-lang3:3.7' implementation 'org.apache.commons:commons-compress:1.26.0' @@ -21,6 +27,12 @@ dependencies { implementation 'com.github.sarxos:windows-registry-util:0.3' + implementation ("io.github.green4j:green-jelly:0.1.6") + + implementation 'com.nimbusds:nimbus-jose-jwt:9.40' + implementation 'org.bouncycastle:bcpkix-jdk18on:1.78.1' + implementation 'com.github.scribejava:scribejava-apis:8.3.3' + // compile('com.sun.xml.bind:jaxb-xjc:2.2.11') // 2.2.11 has only bug fixes compared 2.2.8-b130911.1802 (which used by Java 8) //compile 'org.glassfish.jaxb:jaxb-runtime:2.2.11' // Glassfish replaces JAXB RI //compile('javax.xml:jsr173:1.0'){ transitive = false } @@ -28,4 +40,54 @@ dependencies { implementation('net.java.dev.jna:jna:4.2.1') implementation('net.java.dev.jna:jna-platform:4.2.1') implementation('org.apache.bcel:bcel:6.6.0') + + testImplementation 'com.epam.deltix:gflog-jul' + testRuntimeOnly 'com.epam.deltix:gflog-slf4j' + + implementation group: 'org.antlr', name: 'antlr4-runtime', version: '4.6' + antlrGenerator group: 'org.antlr', name: 'antlr4', version: '4.6' + + testImplementation "org.mockito:mockito-core" + testImplementation 'org.junit.jupiter:junit-jupiter-params' + + // For simulation of connection loss scenarios + testImplementation 'com.github.netcrusherorg:netcrusher-core:0.10' +} + +// Synthetic Rule parser generator +def generatedAntlrDir = file('build/generated-src/antlr') +def grammarPath = "src/main/resources/antlr/SyntheticRule.g4" +def packageName = "com.epam.deltix.qsrv.hf.framework.mdp.impl.parser.generated" +def generatedPath = "$generatedAntlrDir/com/epam/deltix/qsrv/hf/framework/mdp/impl/parser/generated/" + +tasks.register('generateSyntheticRuleGrammar', JavaExec) { + group = "antlr" + description = "Generates ANTLR files (Lexer, Parser, Visitor, etc.)" + + classpath = configurations.antlrGenerator + mainClass = "org.antlr.v4.Tool" + + args = [ + grammarPath, + "-package", packageName, + "-o", generatedPath, + "-no-listener", + "-visitor" + ] + + inputs.file(file(grammarPath)) + .withPropertyName("grammarFile") + .withPathSensitivity(PathSensitivity.RELATIVE) + + outputs.dir(file(generatedPath)) + .withPropertyName("outputDir") +} + +sourceSets.main.java.srcDirs generatedAntlrDir + +tasks.compileJava { + dependsOn generateSyntheticRuleGrammar +} +tasks.processResources { + dependsOn generateSyntheticRuleGrammar } \ No newline at end of file diff --git a/util/src/main/java/com/epam/deltix/qsrv/hf/spi/conn/ReconnectableImpl.java b/util/src/main/java/com/epam/deltix/qsrv/hf/spi/conn/ReconnectableImpl.java index f75d19be..2d131f9a 100644 --- a/util/src/main/java/com/epam/deltix/qsrv/hf/spi/conn/ReconnectableImpl.java +++ b/util/src/main/java/com/epam/deltix/qsrv/hf/spi/conn/ReconnectableImpl.java @@ -14,8 +14,10 @@ * License for the specific language governing permissions and limitations under * the License. */ + package com.epam.deltix.qsrv.hf.spi.conn; +import com.epam.deltix.util.LangUtil; import com.epam.deltix.util.time.GlobalTimer; import com.epam.deltix.util.time.TimerRunner; import net.jcip.annotations.GuardedBy; @@ -26,7 +28,7 @@ import java.util.logging.Logger; /** - * Helps implement the {@link deltix.qsrv.hf.spi.conn.Disconnectable} interface, including reconnect + * Helps implement the {@link com.epam.deltix.qsrv.hf.spi.conn.Disconnectable} interface, including reconnect * capability. */ public class ReconnectableImpl extends DisconnectableEventHandler { @@ -287,7 +289,7 @@ private void tryReconnect() { System.currentTimeMillis() - timeDisconnected, this ); - } catch (Throwable x) { + } catch (Exception x) { String check = x.toString(); if (check.equals(lastExceptionAsString)) { //logger.log (logLevel, "[%s] Reconnect failed due to: %s").with(logprefix).with(lastExceptionAsString); @@ -333,6 +335,7 @@ public void runInternal () { } catch (Throwable x) { //logger.error("[%s] Unexpected: %s").with(logprefix ).with(x); logger.log (Level.SEVERE, "[" + logprefix + "] Unexpected", x); + LangUtil.propagateError(x); } } }; @@ -380,4 +383,4 @@ private static String getDefaultPrefix(Class current) { } return current.getSimpleName(); } -} \ No newline at end of file +} diff --git a/util/src/main/java/com/epam/deltix/qsrv/util/log/RollingFileHandler.java b/util/src/main/java/com/epam/deltix/qsrv/util/log/RollingFileHandler.java new file mode 100644 index 00000000..11f7a16e --- /dev/null +++ b/util/src/main/java/com/epam/deltix/qsrv/util/log/RollingFileHandler.java @@ -0,0 +1,139 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.qsrv.util.log; + +import com.epam.deltix.util.io.UncheckedIOException; + +import java.io.IOException; +import java.util.logging.FileHandler; +import java.util.logging.Level; +import java.util.logging.LogManager; +import java.util.logging.LogRecord; + +/** + * Description: deltix.util.log.RollingFileHandler + * Date: Jul 27, 2009 + * + * @author Nickolay Dul + */ +public class RollingFileHandler extends FileHandler { + private static final Level DEFAULT_PUSH_LEVEL = Level.SEVERE; + private static final long DEFAULT_PUSH_PERIOD = 1000; // 1s + + private Level pushLevel; + private long pushPeriod; + + private long lastPushTime; + + public RollingFileHandler(String pattern, int limit, int count, boolean append, + Level pushLevel, long pushPeriod) + throws IOException, SecurityException { + super(pattern, limit, count, append); + initialize(pushLevel, pushPeriod); + } + + public RollingFileHandler() throws IOException, SecurityException { + super(); + LogManager manager = LogManager.getLogManager(); + initialize(getPushLevel(manager, DEFAULT_PUSH_LEVEL), + getPushPeriod(manager, DEFAULT_PUSH_PERIOD)); + } + + private void initialize(Level pushLevel, long pushPeriod) { + this.pushLevel = pushLevel; + this.pushPeriod = pushPeriod; + + lastPushTime = System.currentTimeMillis(); + } + + @Override + public void publish(LogRecord record) { + super.publish(record); + + // flush buffered output if needed + if ((record.getLevel().intValue() >= pushLevel.intValue()) || + (System.currentTimeMillis() - lastPushTime) >= pushPeriod) { + push(); + } + } + + @Override + public void flush() { + // do nothing + } + + /** + * Flush any buffered output to the file stream. + */ + public void push() { + // perform flushing + super.flush(); + // remember last push time + lastPushTime = System.currentTimeMillis(); + } + + public Level getPushLevel() { + return pushLevel; + } + + public void setPushLevel(Level pushLevel) { + this.pushLevel = pushLevel; + } + + public long getPushPeriod() { + return pushPeriod; + } + + public void setPushPeriod(long pushPeriod) { + this.pushPeriod = pushPeriod; + } + + public RollingFileHandler copy(String pattern) { + try { + LogManager logManager = LogManager.getLogManager(); + String handlerClassName = RollingFileHandler.class.getName(); + // read predefined FileHandler properties if any + int limit = parseInt(logManager.getProperty(handlerClassName + ".limit"), 10000000); + int count = parseInt(logManager.getProperty(handlerClassName + ".count"), 30); + + RollingFileHandler handler = new RollingFileHandler(pattern, limit, count, false, pushLevel, pushPeriod); + handler.setLevel(getLevel()); + handler.setFormatter(getFormatter()); + handler.setFilter(getFilter()); + return handler; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + ////////////////////////// FACTORY METHOD /////////////////////// + + private static int parseInt(String value, int defaultValue) { + value = value != null ? value.trim() : null; + return value != null ? Integer.parseInt(value) : defaultValue; + } + + private static Level getPushLevel(LogManager manager, Level defaultLevel) { + String pushLevelValue = manager.getProperty(RollingFileHandler.class.getName() + ".push"); + return pushLevelValue != null ? Level.parse(pushLevelValue) : defaultLevel; + } + + private static long getPushPeriod(LogManager manager, long defaultPushPeriod) { + String pushPeriodValue = manager.getProperty(RollingFileHandler.class.getName() + ".period"); + return pushPeriodValue != null ? Long.parseLong(pushPeriodValue) : defaultPushPeriod; + } +} diff --git a/util/src/main/java/com/epam/deltix/qsrv/util/log/SMTPHandler.java b/util/src/main/java/com/epam/deltix/qsrv/util/log/SMTPHandler.java new file mode 100644 index 00000000..e69de29b diff --git a/util/src/main/java/com/epam/deltix/qsrv/util/log/TickFormatter.java b/util/src/main/java/com/epam/deltix/qsrv/util/log/TickFormatter.java new file mode 100644 index 00000000..84227d6f --- /dev/null +++ b/util/src/main/java/com/epam/deltix/qsrv/util/log/TickFormatter.java @@ -0,0 +1,81 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.qsrv.util.log; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.io.Writer; +import java.text.MessageFormat; +import java.util.Date; +import java.util.logging.Formatter; +import java.util.logging.Level; +import java.util.logging.LogRecord; + +public class TickFormatter extends Formatter{ + + private Date date = new Date (); + private static final String format = "{0,date} {0,time}"; + private java.text.MessageFormat formatter; + private String lineSeparator = "\r\n"; + private Object args[] = new Object[1]; + private StringBuffer buffer = new StringBuffer (); + private StringBuffer text = new StringBuffer (); + + @SuppressWarnings({"ThrowableResultOfMethodCallIgnored"}) + @Override + public String format(LogRecord record) { + buffer.setLength (0); + text.setLength (0); + // Minimize memory allocations here. + date.setTime(record.getMillis()); + args[0] = date; + if (formatter == null) { + formatter = new MessageFormat(format); + } + formatter.format(args, text, null); + buffer.append (text); + buffer.append (" "); + + String message = formatMessage(record); + buffer.append (record.getLevel ().getLocalizedName ()); + buffer.append (": "); + if (message != null) + buffer.append (message); + + buffer.append (lineSeparator); + + final Throwable throwable = record.getThrown (); + if (throwable != null) { + if (Level.SEVERE.equals(record.getLevel())){ + buffer.append (getStackTrace(throwable)); + }else{ + buffer.append(throwable.toString()); + } + buffer.append (lineSeparator); + } + + return buffer.toString (); + } + + public static String getStackTrace(Throwable aThrowable) { + final Writer result = new StringWriter(); + final PrintWriter printWriter = new PrintWriter(result); + aThrowable.printStackTrace(printWriter); + return result.toString(); + } + +} diff --git a/util/src/main/java/com/epam/deltix/util/cmdline/AbstractShell.java b/util/src/main/java/com/epam/deltix/util/cmdline/AbstractShell.java index b6b4f354..3f3db792 100644 --- a/util/src/main/java/com/epam/deltix/util/cmdline/AbstractShell.java +++ b/util/src/main/java/com/epam/deltix/util/cmdline/AbstractShell.java @@ -14,6 +14,7 @@ * License for the specific language governing permissions and limitations under * the License. */ + package com.epam.deltix.util.cmdline; import com.epam.deltix.util.io.IOUtil; @@ -154,7 +155,7 @@ protected final void set (String option, String value) { m.invoke (this, arg); else if (!doSet (option, value)) System.err.println ("set " + option + ": unrecognized option. (Type ? for usage)"); - } catch (Throwable x) { + } catch (Exception x) { printException (x, true); } } @@ -235,7 +236,7 @@ public final void runCommand (String key, String args, String fileId, LineNum default: throw new RuntimeException (); } - } catch (Throwable x) { + } catch (Exception x) { printException (x, true); error (2); } @@ -407,4 +408,4 @@ public boolean getConfirm () { public void setConfirm (boolean verbose) { this.confirm = verbose; } -} \ No newline at end of file +} diff --git a/util/src/main/java/com/epam/deltix/util/concurrent/ContextContainer.java b/util/src/main/java/com/epam/deltix/util/concurrent/ContextContainer.java new file mode 100644 index 00000000..1fa7a9ac --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/concurrent/ContextContainer.java @@ -0,0 +1,108 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.concurrent; + +import com.epam.deltix.thread.affinity.AffinityConfig; +import org.jetbrains.annotations.VisibleForTesting; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.annotation.ParametersAreNonnullByDefault; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author Alexei Osipov + */ +@ParametersAreNonnullByDefault +// TODO: rename to ThreadExecutor and move to util + +public class ContextContainer { + private static final AtomicInteger namelessExecutorIndex = new AtomicInteger(); + private static final AtomicInteger testExecutorIndex = new AtomicInteger(); + + private volatile AffinityConfig affinityConfig = getDefaultAffinityConfig(); + + private volatile String quickExecutorName = null; + private volatile QuickExecutor quickExecutor = null; + + @Nullable + public AffinityConfig getAffinityConfig() { + return affinityConfig; + } + + public void setAffinityConfig(@Nullable AffinityConfig affinityConfig) { + this.affinityConfig = affinityConfig; + } + + @Nonnull + public QuickExecutor getQuickExecutor() { + // Lazy singleton + if (this.quickExecutor == null) { + synchronized (QuickExecutor.class) { + if (this.quickExecutor == null) { + this.quickExecutor = createQuickExecutor(); + } + } + } + return this.quickExecutor; + } + + /** + * Marks QuickExecutor for shutdown. Tries to avoid executor creation. + */ + public void shutdownQuickExecutor() { + // Note: current implementation DOES breaks instance creation check. + if (this.quickExecutor == null) { + return; + } + getQuickExecutor().shutdownInstance(); + } + + @Nonnull + private QuickExecutor createQuickExecutor() { + String name = this.quickExecutorName; + if (name == null) { + name = "ContextContainer Executor #" + namelessExecutorIndex.incrementAndGet(); + } + return QuickExecutor.createNewInstance(name, this.affinityConfig); + } + + public void setQuickExecutorName(String quickExecutorName) { + this.quickExecutorName = quickExecutorName; + } + + + @VisibleForTesting + public static ContextContainer getContextContainerForClientTests() { + ContextContainer contextContainer = new ContextContainer(); + contextContainer.setQuickExecutorName("Client Test Executor #" + testExecutorIndex.incrementAndGet()); + return contextContainer; + } + + @VisibleForTesting + public static ContextContainer getContextContainerForServerTests() { + ContextContainer contextContainer = new ContextContainer(); + contextContainer.setQuickExecutorName("Server Test Executor #" + testExecutorIndex.incrementAndGet()); + return contextContainer; + } + + private static AffinityConfig getDefaultAffinityConfig() { + // Null value mean no any explicit affinity + return null; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/concurrent/DisposableDataSourceMultiplexer.java b/util/src/main/java/com/epam/deltix/util/concurrent/DisposableDataSourceMultiplexer.java index 1686c994..7b247f10 100644 --- a/util/src/main/java/com/epam/deltix/util/concurrent/DisposableDataSourceMultiplexer.java +++ b/util/src/main/java/com/epam/deltix/util/concurrent/DisposableDataSourceMultiplexer.java @@ -14,6 +14,7 @@ * License for the specific language governing permissions and limitations under * the License. */ + package com.epam.deltix.util.concurrent; import com.epam.deltix.util.lang.Util; @@ -34,8 +35,8 @@ public void close () { for (T ds : dataSources ()) try { ds.close (); - } catch (Throwable x) { + } catch (Exception x) { Util.logException ("close () threw exception", x); } } -} \ No newline at end of file +} diff --git a/util/src/main/java/com/epam/deltix/util/concurrent/FrequencyLimiter.java b/util/src/main/java/com/epam/deltix/util/concurrent/FrequencyLimiter.java index 3f1b372e..d80c197d 100644 --- a/util/src/main/java/com/epam/deltix/util/concurrent/FrequencyLimiter.java +++ b/util/src/main/java/com/epam/deltix/util/concurrent/FrequencyLimiter.java @@ -14,8 +14,10 @@ * License for the specific language governing permissions and limitations under * the License. */ + package com.epam.deltix.util.concurrent; +import com.epam.deltix.util.LangUtil; import com.epam.deltix.util.lang.Util; import java.util.Timer; @@ -40,6 +42,7 @@ public FrequencyLimiter (Timer timer) { */ protected void onError (Throwable x) { Util.logException(this + " failed", x); + LangUtil.propagateError(x); } /** @@ -98,4 +101,4 @@ public synchronized boolean disarm () { task = null; return (true); } -} \ No newline at end of file +} diff --git a/util/src/main/java/com/epam/deltix/util/concurrent/QuickExecutor.java b/util/src/main/java/com/epam/deltix/util/concurrent/QuickExecutor.java index 8091b6c2..df196bd1 100644 --- a/util/src/main/java/com/epam/deltix/util/concurrent/QuickExecutor.java +++ b/util/src/main/java/com/epam/deltix/util/concurrent/QuickExecutor.java @@ -14,6 +14,7 @@ * License for the specific language governing permissions and limitations under * the License. */ + package com.epam.deltix.util.concurrent; import com.epam.deltix.gflog.api.Log; @@ -21,6 +22,7 @@ import com.epam.deltix.gflog.api.LogLevel; import com.epam.deltix.thread.affinity.AffinityConfig; import com.epam.deltix.thread.affinity.AffinityThreadFactoryBuilder; +import com.epam.deltix.util.LangUtil; import com.epam.deltix.util.collections.QuickList; import com.epam.deltix.util.collections.generated.ObjectHashSet; @@ -269,6 +271,7 @@ public void run () { LOGGER.log(LogLevel.DEBUG).append(task).append(" interrupted.").append(x).commit(); } catch (Throwable x) { LOGGER.log(LogLevel.ERROR).append(task).append(" failed.").append(x).commit(); + LangUtil.propagateError(x); } finally { if (!task.setDone ()) { task = null; @@ -488,4 +491,4 @@ private void shutdown(boolean waitForCompleteShut shutdownInProgress = false; } -} \ No newline at end of file +} diff --git a/util/src/main/java/com/epam/deltix/util/concurrent/QuickExecutorWithQueue.java b/util/src/main/java/com/epam/deltix/util/concurrent/QuickExecutorWithQueue.java index 608c1133..49367ab4 100644 --- a/util/src/main/java/com/epam/deltix/util/concurrent/QuickExecutorWithQueue.java +++ b/util/src/main/java/com/epam/deltix/util/concurrent/QuickExecutorWithQueue.java @@ -14,16 +14,21 @@ * License for the specific language governing permissions and limitations under * the License. */ + package com.epam.deltix.util.concurrent; import com.epam.deltix.gflog.api.Log; import com.epam.deltix.gflog.api.LogFactory; import com.epam.deltix.gflog.api.LogLevel; import com.epam.deltix.util.collections.QuickList; -import java.util.*; -import java.util.concurrent.locks.LockSupport; +import com.epam.deltix.util.LangUtil; import net.jcip.annotations.GuardedBy; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.locks.LockSupport; + /** * Similar to standard Java executors, but does not allocate memory on task * reschedule. @@ -195,6 +200,7 @@ public void run () { LOGGER.log(LogLevel.DEBUG).append(task).append(" interrupted.").append(x).commit(); } catch (Throwable x) { LOGGER.log(LogLevel.ERROR).append(task).append(" failed.").append(x).commit(); + LangUtil.propagateError(x); } finally { if (task.finished ()) task = null; @@ -202,6 +208,7 @@ public void run () { } } catch (Throwable x) { x.printStackTrace (); + LangUtil.propagateError(x); } } } @@ -415,4 +422,4 @@ private void shutdown ( if (DEBUG_TASKS) System.out.println (this + ": shutdown completed."); } -} \ No newline at end of file +} diff --git a/util/src/main/java/com/epam/deltix/util/concurrent/ThrottlingExecutor.java b/util/src/main/java/com/epam/deltix/util/concurrent/ThrottlingExecutor.java index 729f3c69..6eb4e3d1 100644 --- a/util/src/main/java/com/epam/deltix/util/concurrent/ThrottlingExecutor.java +++ b/util/src/main/java/com/epam/deltix/util/concurrent/ThrottlingExecutor.java @@ -14,10 +14,12 @@ * License for the specific language governing permissions and limitations under * the License. */ + package com.epam.deltix.util.concurrent; import com.epam.deltix.gflog.api.Log; import com.epam.deltix.gflog.api.LogFactory; +import com.epam.deltix.util.LangUtil; import com.epam.deltix.util.collections.QuickList; import com.epam.deltix.util.lang.ExceptionHandler; import com.epam.deltix.util.time.TimeKeeper; @@ -171,6 +173,8 @@ private long performMeasurableWork () LOG.error("Exception in %s: %s").with(next).with(x); else handler.handle (x); + + LangUtil.propagateError(x); } t1 = TimeKeeper.currentTime; @@ -217,4 +221,4 @@ public void run () { LOG.trace("%s is terminating.").with(this); } -} \ No newline at end of file +} diff --git a/util/src/main/java/com/epam/deltix/util/csvx/CSVFilter-usage.txt b/util/src/main/java/com/epam/deltix/util/csvx/CSVFilter-usage.txt new file mode 100644 index 00000000..b9c7e442 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/csvx/CSVFilter-usage.txt @@ -0,0 +1,23 @@ +Usage: csv

+ * Try using {@link YieldingIdleStrategy} before trying {@link BusySpinIdleStrategy}. + * + * @author Alexei Osipov + */ +public class BusySpinIdleStrategy implements IdleStrategy { + @Override + public void idle(int workCount) { + if (workCount == 0) { + idle(); + } + } + + @Override + public void idle() { + Thread.onSpinWait(); + } + + @Override + public void reset() { + } +} diff --git a/util/src/main/java/com/epam/deltix/util/io/idlestrat/IdleStrategy.java b/util/src/main/java/com/epam/deltix/util/io/idlestrat/IdleStrategy.java new file mode 100644 index 00000000..a3ebf2eb --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/io/idlestrat/IdleStrategy.java @@ -0,0 +1,37 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.io.idlestrat; + +/** + * Different implementation + */ +public interface IdleStrategy { + /** + * @param workCount amount of work done in the last cycle. Value "0" means that no work as done and some data form an external source expected. + */ + void idle(int workCount); + + /** + * Idle action (sleep, wait, etc). + */ + void idle(); + + /** + * Reset the internal state (after doing some work). + */ + void reset(); +} diff --git a/util/src/main/java/com/epam/deltix/util/io/offheap/OffHeap.java b/util/src/main/java/com/epam/deltix/util/io/offheap/OffHeap.java new file mode 100644 index 00000000..3c78bb95 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/io/offheap/OffHeap.java @@ -0,0 +1,96 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.io.offheap; + +import com.epam.deltix.util.collections.OffHeapByteQueue; +import com.epam.deltix.util.io.idlestrat.BusySpinIdleStrategy; +import com.epam.deltix.util.io.waitstrat.ParkWaitStrategy; +import com.epam.deltix.util.io.IOUtil; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * + */ +public final class OffHeap { + public static final Logger LOGGER = Logger.getLogger("deltix.vsoffheap"); + + public static final int QUEUE_SIZE = 1 << 19; + public static String OFFHEAP_DIR; + + private OffHeap() { + throw new RuntimeException("Not for you!"); + } + + public synchronized static void start(String offheapDir, boolean isServer) { + OFFHEAP_DIR = offheapDir; + if (isServer) + prepareDirectory(offheapDir); + } + + public static InputStream createInputStream(String name) throws IOException { + return createInputStream(createRandomAccessFile(name)); + } + + public static InputStream createInputStream(RandomAccessFile raf) throws IOException { + OffHeapPollableInputStream in = + new OffHeapPollableInputStream(createByteQueue(raf), new ParkWaitStrategy()); + OffHeapPollableInputStream.POLLER.add(in); + return in; + //return new OffHeapInputStream(createByteQueue(raf), new NoOpIdleStrategy()); + } + + public static OutputStream createOutputStream(String name) throws IOException { + return createOutputStream(createRandomAccessFile(name)); + } + + public static OffHeapOutputStream createOutputStream(RandomAccessFile raf) throws IOException { + return new OffHeapOutputStream(createByteQueue(raf), new BusySpinIdleStrategy()); + } + + public synchronized static RandomAccessFile createRandomAccessFile(String name) throws FileNotFoundException { + File file = new File(OFFHEAP_DIR + "/" + name); + file.deleteOnExit(); + return new RandomAccessFile(file, "rw"); + } + + public static OffHeapByteQueue createByteQueue(RandomAccessFile raf) throws IOException { + return OffHeapByteQueue.newInstance(raf, QUEUE_SIZE); + } + + public synchronized static String getOffHeapDir() { + return OFFHEAP_DIR; + } + + private static void prepareDirectory(String offheapDir) { + try { + IOUtil.delete(new File(offheapDir)); + } catch (IOException e) { + LOGGER.log(Level.WARNING, "Delete '" + offheapDir + "' failed.", e); + } + + if (!new File(offheapDir).mkdirs()) + LOGGER.log(Level.WARNING, "Create '" + offheapDir + "' failed."); + } +} diff --git a/util/src/main/java/com/epam/deltix/util/io/offheap/OffHeapInputStream.java b/util/src/main/java/com/epam/deltix/util/io/offheap/OffHeapInputStream.java new file mode 100644 index 00000000..335d761f --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/io/offheap/OffHeapInputStream.java @@ -0,0 +1,77 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.io.offheap; + +import com.epam.deltix.util.collections.OffHeapByteQueue; +import com.epam.deltix.util.io.idlestrat.BusySpinIdleStrategy; +import com.epam.deltix.util.io.idlestrat.IdleStrategy; + +import java.io.IOException; +import java.io.InputStream; + +/** + * + */ +public class OffHeapInputStream extends InputStream { + protected OffHeapByteQueue queue; + private IdleStrategy idleStrategy; + + OffHeapInputStream(OffHeapByteQueue queue) { + this(queue, new BusySpinIdleStrategy()); + } + + OffHeapInputStream(OffHeapByteQueue queue, IdleStrategy idleStrategy) { + this.queue = queue; + this.idleStrategy = idleStrategy; + } + + @Override + public int read() throws IOException { + int workCount = 0; + int res; + while ((res = queue.poll()) < 0) { + idleStrategy.idle(workCount++); + } + + return res; + } + + @Override + public int read(byte b[], int off, int len) + throws IOException { + + if (b == null) { + throw new NullPointerException(); + } else if (off < 0 || len < 0 || len > b.length - off) { + throw new IndexOutOfBoundsException(); + } else if (len == 0) { + return 0; + } + + int count; + int workCount = 0; + while ((count = queue.poll(b, off, len)) <= 0) { + idleStrategy.idle(workCount++); + } + + return count; + } + + @Override + public void close() { + } +} diff --git a/util/src/main/java/com/epam/deltix/util/io/offheap/OffHeapOutputStream.java b/util/src/main/java/com/epam/deltix/util/io/offheap/OffHeapOutputStream.java new file mode 100644 index 00000000..08ecb6fd --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/io/offheap/OffHeapOutputStream.java @@ -0,0 +1,76 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.io.offheap; + +import com.epam.deltix.util.collections.OffHeapByteQueue; +import com.epam.deltix.util.io.idlestrat.BusySpinIdleStrategy; +import com.epam.deltix.util.io.idlestrat.IdleStrategy; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * + */ +public class OffHeapOutputStream extends OutputStream { + private OffHeapByteQueue queue; + private IdleStrategy idleStrategy; + + OffHeapOutputStream(OffHeapByteQueue queue) { + this(queue, new BusySpinIdleStrategy()); + } + + OffHeapOutputStream(OffHeapByteQueue queue, IdleStrategy idleStrategy) { + this.queue = queue; + this.idleStrategy = idleStrategy; + } + + @Override + public void write(int n) throws IOException { + byte b = (byte) n; + int workCount = 0; + while (!queue.offer(b)) + idleStrategy.idle(workCount++); + } + + @Override + public void write(byte b[], int off, int len) + throws IOException { + if (b == null) { + throw new NullPointerException(); + } else if ((off < 0) || (off > b.length) || (len < 0) || + ((off + len) > b.length) || ((off + len) < 0) || + len > queue.capacity()) { + throw new IndexOutOfBoundsException(); + } else if (len == 0) { + return; + } + + int workCount = 0; + while (len > 0) { + int written = queue.offer(b, off, len); + len -= written; + off += written; + + idleStrategy.idle(workCount++); + } + } + + @Override + public void close() throws IOException { + } +} diff --git a/util/src/main/java/com/epam/deltix/util/io/offheap/OffHeapPollableInputStream.java b/util/src/main/java/com/epam/deltix/util/io/offheap/OffHeapPollableInputStream.java new file mode 100644 index 00000000..a6d3d848 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/io/offheap/OffHeapPollableInputStream.java @@ -0,0 +1,102 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.io.offheap; + +import com.epam.deltix.util.collections.OffHeapByteQueue; +import com.epam.deltix.util.io.idlestrat.BusySpinIdleStrategy; +import com.epam.deltix.util.io.waitstrat.ParkWaitStrategy; +import com.epam.deltix.util.lang.Pollable; +import com.epam.deltix.util.io.PollingThread; +import com.epam.deltix.util.io.waitstrat.WaitStrategy; + +import java.io.IOException; +import java.io.InputStream; + +/** + * + */ +public class OffHeapPollableInputStream extends InputStream implements Pollable { + + static final PollingThread POLLER; + + static { + synchronized (OffHeapPollableInputStream.class) { + (POLLER = new PollingThread(new BusySpinIdleStrategy())).start(); + } + } + + private OffHeapByteQueue queue; + private WaitStrategy waitStrategy; + + OffHeapPollableInputStream(OffHeapByteQueue queue) throws IOException { + this(queue, new ParkWaitStrategy()); + } + + OffHeapPollableInputStream(OffHeapByteQueue queue, WaitStrategy waitStrategy) { + this.queue = queue; + this.waitStrategy = waitStrategy; + } + + @Override + public void poll() { + if (queue.getTailSequence().changed()) + waitStrategy.signal(); + } + + @Override + public int read() throws IOException { + int res; + while ((res = queue.poll()) < 0) { + waitUnchecked(); + } + + return res; + } + + @Override + public int read(byte b[], int off, int len) + throws IOException { + + if (b == null) { + throw new NullPointerException(); + } else if (off < 0 || len < 0 || len > b.length - off) { + throw new IndexOutOfBoundsException(); + } else if (len == 0) { + return 0; + } + + int count; + while ((count = queue.poll(b, off, len)) <= 0) { + waitUnchecked(); + } + + return count; + } + + private void waitUnchecked() throws IOException { + try { + waitStrategy.waitSignal(); + } catch (InterruptedException e) { + //ignore + } + } + + @Override + public void close() { + POLLER.remove(this); + } +} diff --git a/util/src/main/java/com/epam/deltix/util/io/waitstrat/InterprocessLock.java b/util/src/main/java/com/epam/deltix/util/io/waitstrat/InterprocessLock.java new file mode 100644 index 00000000..8518de0b --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/io/waitstrat/InterprocessLock.java @@ -0,0 +1,64 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.io.waitstrat; + +import com.epam.deltix.util.lang.Disposable; + +/** + * + */ +public class InterprocessLock implements Disposable { + +// static { +// System.load(Home.getFile("bin").getAbsolutePath() + +// "/interproc" + +// System.getProperty("os.arch") + ".dll"); +// } + + private long handle; + + /** + * Named system event. + * Create or open system event. + * @param name - name of event. + */ + public InterprocessLock(String name) { + handle = init0(name, true); + } + + /** + * Wait for 'set' state. + * Than event auto-resets. + */ + public void waitFor() { + wait0(handle); + } + + public void signal() { + set0(handle); + } + + public void close() { + close0(handle); + } + + private native long init0(String name, boolean autoreset); + private native boolean wait0(long handle); + private native boolean set0(long handle); + private native boolean reset0(long handle); + private native void close0(long handle); +} diff --git a/lang/src/test/java/com/epam/deltix/util/memory/MemoryDataOutputTest.java b/util/src/main/java/com/epam/deltix/util/io/waitstrat/NativeEventsWaitStrategy.java similarity index 57% rename from lang/src/test/java/com/epam/deltix/util/memory/MemoryDataOutputTest.java rename to util/src/main/java/com/epam/deltix/util/io/waitstrat/NativeEventsWaitStrategy.java index 77055655..90985929 100644 --- a/lang/src/test/java/com/epam/deltix/util/memory/MemoryDataOutputTest.java +++ b/util/src/main/java/com/epam/deltix/util/io/waitstrat/NativeEventsWaitStrategy.java @@ -1,39 +1,40 @@ -/* - * Copyright 2021 EPAM Systems, Inc - * - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. Licensed under the Apache License, - * Version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. */ -package com.epam.deltix.util.memory; - -import org.junit.Test; - -import static org.junit.Assert.*; - -/** - * @author Alexei Osipov - */ -public class MemoryDataOutputTest { - @Test - public void writeLongBytes() throws Exception { - MemoryDataOutput mdo = new MemoryDataOutput(); - int result = mdo.writeLongBytes(0xFF_00_FFL); - assertEquals(3, result); - assertEquals(3, mdo.getSize()); - - int result2 = mdo.writeLongBytes(0xCAFE_BABEL); - assertEquals(4, result2); - assertEquals(7, mdo.getSize()); - } - -} \ No newline at end of file + package com.epam.deltix.util.io.waitstrat; + +/** + * + */ +public class NativeEventsWaitStrategy implements WaitStrategy { + private InterprocessLock lock; + + public NativeEventsWaitStrategy(String name) { + lock = new InterprocessLock(name); + } + + public void waitSignal() { + lock.waitFor(); + } + + public void signal() { + lock.signal(); + } + + public void close() { + lock.close(); + } +} diff --git a/util/src/main/java/com/epam/deltix/util/io/waitstrat/NotifyWaitStrategy.java b/util/src/main/java/com/epam/deltix/util/io/waitstrat/NotifyWaitStrategy.java new file mode 100644 index 00000000..b929dcf9 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/io/waitstrat/NotifyWaitStrategy.java @@ -0,0 +1,43 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.io.waitstrat; + +/** + * + */ +public class NotifyWaitStrategy implements WaitStrategy { + private boolean wasSignalled = false; + + @Override + public synchronized void waitSignal() throws InterruptedException { + while (!wasSignalled){ + wait(); + } + wasSignalled = false; + } + + @Override + public synchronized void signal() { + wasSignalled = true; + notify(); + } + + @Override + public void close() { + signal(); + } +} diff --git a/util/src/main/java/com/epam/deltix/util/io/waitstrat/ParkWaitStrategy.java b/util/src/main/java/com/epam/deltix/util/io/waitstrat/ParkWaitStrategy.java new file mode 100644 index 00000000..48dc3b7a --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/io/waitstrat/ParkWaitStrategy.java @@ -0,0 +1,68 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.io.waitstrat; + +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import java.util.concurrent.locks.LockSupport; + +/** + * + */ +public class ParkWaitStrategy implements WaitStrategy { + @SuppressWarnings("unused") + private volatile Thread owner; + + private static final AtomicReferenceFieldUpdater ownerAccess = + AtomicReferenceFieldUpdater.newUpdater(ParkWaitStrategy.class, Thread.class, "owner"); + + public ParkWaitStrategy() { + owner = null; + } + + @Override + public void waitSignal() throws InterruptedException { + Thread t = Thread.currentThread(); + if (!ownerAccess.compareAndSet(this, null, t)) { + throw new IllegalStateException("A second thread tried to acquire a signal barrier that is already owned."); + } + + LockSupport.park(this); + + // If a thread has called #signal() the owner should already be null. + // However the documentation for LockSupport.unpark makes it clear that + // threads can wake up for absolutely no reason. Do a compare and set + // to make sure we don't wipe out a new owner, keeping in mind that only + // thread should be awaiting at any given moment! + ownerAccess.compareAndSet(this, t, null); + + // Check to see if we've been unparked because of a thread interrupt. + if (t.isInterrupted()) + throw new InterruptedException(); + } + + @Override + public void signal() { + Thread t = ownerAccess.getAndSet(this, null); + if (t != null) + LockSupport.unpark(t); + } + + @Override + public void close() { + signal(); + } +} \ No newline at end of file diff --git a/util/src/main/java/com/epam/deltix/util/io/waitstrat/SleepingWaitForChangeStrategy.java b/util/src/main/java/com/epam/deltix/util/io/waitstrat/SleepingWaitForChangeStrategy.java new file mode 100644 index 00000000..a999d4bb --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/io/waitstrat/SleepingWaitForChangeStrategy.java @@ -0,0 +1,49 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.io.waitstrat; + +import com.epam.deltix.util.lang.Changeable; + +import java.util.concurrent.locks.LockSupport; + +/** + * + */ +public class SleepingWaitForChangeStrategy implements WaitForChangeStrategy { + private int count = 1000; + + public SleepingWaitForChangeStrategy() { + } + + public void waitFor(Changeable value) { + while (!value.changed()) { + if (count > 500) { + --count; + } else if (count > 0) { + --count; + Thread.yield(); + } else { + LockSupport.parkNanos(1); + } + } + + count = 1000; + } + + public void close() { + } +} diff --git a/messages/src/main/java/com/epam/deltix/streaming/MessageChannel.java b/util/src/main/java/com/epam/deltix/util/io/waitstrat/WaitForChangeStrategy.java similarity index 64% rename from messages/src/main/java/com/epam/deltix/streaming/MessageChannel.java rename to util/src/main/java/com/epam/deltix/util/io/waitstrat/WaitForChangeStrategy.java index 7af93f96..23f816a1 100644 --- a/messages/src/main/java/com/epam/deltix/streaming/MessageChannel.java +++ b/util/src/main/java/com/epam/deltix/util/io/waitstrat/WaitForChangeStrategy.java @@ -1,33 +1,27 @@ -/* - * Copyright 2021 EPAM Systems, Inc - * - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. Licensed under the Apache License, - * Version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. */ -package com.epam.deltix.streaming; - -import com.epam.deltix.util.lang.Disposable; - -/** - * Generic message consumer interface. - */ -public interface MessageChannel extends Disposable { - /** - * This method is invoked to send a message to the object. - * - * @param msg A temporary buffer with the message. - * By convention, the message is only valid for the duration - * of this call. - */ - public void send (T msg); -} \ No newline at end of file + package com.epam.deltix.util.io.waitstrat; + +import com.epam.deltix.util.lang.Changeable; +import com.epam.deltix.util.lang.Disposable; + +/** + * + */ +public interface WaitForChangeStrategy extends Disposable { + public void waitFor(Changeable value) throws InterruptedException; +} diff --git a/messages/src/main/java/com/epam/deltix/timebase/messages/Bitmask.java b/util/src/main/java/com/epam/deltix/util/io/waitstrat/WaitStrategy.java similarity index 70% rename from messages/src/main/java/com/epam/deltix/timebase/messages/Bitmask.java rename to util/src/main/java/com/epam/deltix/util/io/waitstrat/WaitStrategy.java index db7894e1..bc42eaf5 100644 --- a/messages/src/main/java/com/epam/deltix/timebase/messages/Bitmask.java +++ b/util/src/main/java/com/epam/deltix/util/io/waitstrat/WaitStrategy.java @@ -1,29 +1,27 @@ -/* - * Copyright 2021 EPAM Systems, Inc - * - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. Licensed under the Apache License, - * Version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. */ -package com.epam.deltix.timebase.messages; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Retention(RetentionPolicy.RUNTIME) -@Target( { - ElementType.TYPE} -) -public @interface Bitmask { -} + package com.epam.deltix.util.io.waitstrat; + +import com.epam.deltix.util.lang.Disposable; + +/** + * + */ +public interface WaitStrategy extends Disposable { + public void waitSignal() throws InterruptedException; + public void signal(); +} diff --git a/util/src/main/java/com/epam/deltix/util/io/waitstrat/YieldingWaitForChangeStrategy.java b/util/src/main/java/com/epam/deltix/util/io/waitstrat/YieldingWaitForChangeStrategy.java new file mode 100644 index 00000000..5eec5756 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/io/waitstrat/YieldingWaitForChangeStrategy.java @@ -0,0 +1,36 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.io.waitstrat; + +import com.epam.deltix.util.lang.Changeable; + +/** + * + */ +public class YieldingWaitForChangeStrategy implements WaitForChangeStrategy { + public YieldingWaitForChangeStrategy() { + } + + public void waitFor(Changeable value) { + while (!value.changed()) { + Thread.yield(); + } + } + + public void close() { + } +} diff --git a/util/src/main/java/com/epam/deltix/util/log/CompoundHandler.java b/util/src/main/java/com/epam/deltix/util/log/CompoundHandler.java new file mode 100644 index 00000000..837e22b6 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/log/CompoundHandler.java @@ -0,0 +1,70 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.log; + +import java.util.logging.Formatter; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; + +/** +* Description: deltix.util.log.CompoundHandler +* Date: 8/5/11 +* +* @author Nickolay Dul +*/ +public class CompoundHandler extends Handler { + private final Handler[] handlers; + + public CompoundHandler(Handler[] handlers) { + this.handlers = handlers; + } + + public Handler[] getHandlers() { + return handlers.clone(); + } + + @Override + public void setFormatter(Formatter newFormatter) throws SecurityException { + for (int i = 0; i < handlers.length; i++) + handlers[i].setFormatter(newFormatter); + } + + @Override + public void setLevel(Level newLevel) throws SecurityException { + for (int i = 0; i < handlers.length; i++) + handlers[i].setLevel(newLevel); + } + + @Override + public void publish(LogRecord record) { + for (int i = 0; i < handlers.length; i++) + handlers[i].publish(record); + } + + @Override + public void flush() { + for (int i = 0; i < handlers.length; i++) + handlers[i].flush(); + } + + @Override + public void close() throws SecurityException { + for (int i = 0; i < handlers.length; i++) + handlers[i].close(); + } +} diff --git a/util/src/main/java/com/epam/deltix/util/log/CurrentMonthDate.java b/util/src/main/java/com/epam/deltix/util/log/CurrentMonthDate.java new file mode 100644 index 00000000..8e8ac8cc --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/log/CurrentMonthDate.java @@ -0,0 +1,113 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.log; + +import com.epam.deltix.util.time.GlobalTimer; +import com.epam.deltix.util.LangUtil; + +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.TimerTask; +import java.util.concurrent.TimeUnit; + +/** + * Used by TerseFormatter to log date. Usage: + * 

+ * CurrentMonthDate cmd = CurrentMonthDate.getInstance();
+ * String date = cmd.getDayMonth(timestamp);
+ *
+ * @author Andy +*/ +public class CurrentMonthDate extends TimerTask { + private final static long MILLISECONDS_IN_DAY = TimeUnit.DAYS.toMillis(1); + + private final String [] MONTH_CODES = new SimpleDateFormat().getDateFormatSymbols().getShortMonths(); + private final Calendar c = Calendar.getInstance(); + + private volatile String currentMonthDay; + private volatile long goodUntil; + + private static final CurrentMonthDate INSTANCE = create(); + + private CurrentMonthDate() { + roll (System.currentTimeMillis()); + } + + private static CurrentMonthDate create () { + final long now = System.currentTimeMillis(); + CurrentMonthDate result = new CurrentMonthDate(); + GlobalTimer.INSTANCE.scheduleAtFixedRate(result, result.goodUntil - now, MILLISECONDS_IN_DAY); + return result; + } + + + public static CurrentMonthDate getInstance () { + return INSTANCE; + } + + /** @return Date and Month of given timestamp. + * Method is slow when it is called at exactly midnight, or if timestamp represent previous day or some moment in the future. */ + public String getDayMonth (long timestamp) { + + final long goodUntilCopy = goodUntil; // volatile + if (timestamp < goodUntilCopy && timestamp > goodUntilCopy - MILLISECONDS_IN_DAY) + return currentMonthDay; + else + return slowFormat (timestamp); + } + + private synchronized String slowFormat (long timestamp) { + c.setTimeInMillis(timestamp); + c.set(Calendar.HOUR_OF_DAY, 0); + c.set(Calendar.MINUTE, 0); + c.set(Calendar.SECOND, 0); + c.set(Calendar.MILLISECOND, 0); + + return String.valueOf(c.get(Calendar.DAY_OF_MONTH)) + ' ' + MONTH_CODES[c.get(Calendar.MONTH)]; + } + + private synchronized void roll (long timestamp) { + assert timestamp >= goodUntil; + + c.setTimeInMillis(timestamp); + c.set(Calendar.HOUR_OF_DAY, 0); + c.set(Calendar.MINUTE, 0); + c.set(Calendar.SECOND, 0); + c.set(Calendar.MILLISECOND, 0); + + final long nextMidnight = c.getTimeInMillis() + MILLISECONDS_IN_DAY; + + currentMonthDay = String.valueOf(c.get(Calendar.DAY_OF_MONTH)) + ' ' + MONTH_CODES[c.get(Calendar.MONTH)]; + goodUntil = nextMidnight; + } + + + + @Override + public final void run() { + try { + // runs at midnight - optional taks that rolls currentMonthDay forward to avoid doing it during logging (if possible) + roll (System.currentTimeMillis()); + } catch (Throwable e) { + // this should not happened in the current roll() stack + // !!! DO NOT USE Logger here !!! TerseFormatter creates CurrentMonthDate statically and hangs during initialization (#13116) + System.out.println("Error while rolling CurrentMonthDate: " + e.getMessage()); + LangUtil.propagateError(e); + } + } +} diff --git a/util/src/main/java/com/epam/deltix/util/log/DetailFormatter.java b/util/src/main/java/com/epam/deltix/util/log/DetailFormatter.java new file mode 100644 index 00000000..8143094c --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/log/DetailFormatter.java @@ -0,0 +1,85 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.log; + +import com.epam.deltix.util.lang.Util; +import com.epam.deltix.util.text.SimpleMessageFormat; + +import java.util.logging.Formatter; +import java.util.logging.LogRecord; + +/** + * Description: deltix.util.log.DetailFormatter + * Date: Jul 15, 2010 + * + * @author Nickolay Dul + */ +public class DetailFormatter extends Formatter { + public static final String PRINT_CONTEXT_PROPERTY = "QuantServer.logging.detailFormatter.printContext"; + + private final boolean printContext; + + public DetailFormatter() { + this(Boolean.getBoolean(PRINT_CONTEXT_PROPERTY)); + } + + public DetailFormatter(boolean printContext) { + this.printContext = printContext; + } + + @Override + public String format(LogRecord record) { + StringBuilder sbuf = new StringBuilder(256); + // time + sbuf.append(String.format("%1$tF %1$tT.%1$tL", record.getMillis())).append(' '); + + // context + if (printContext) { + if (record.getSourceClassName() != null) { + sbuf.append(record.getSourceClassName()); + } else { + sbuf.append(record.getLoggerName()); + } + if (record.getSourceMethodName() != null) { + sbuf.append(' ').append(record.getSourceMethodName()); + } + sbuf.append(Util.NATIVE_LINE_BREAK); + } + + // level + sbuf.append(record.getLevel()).append(' '); + // thread id + sbuf.append('[').append(record.getThreadID()).append("] "); + + // message + String message = record.getMessage (); + Object[] params = record.getParameters(); + if (params != null && params.length > 0) { + SimpleMessageFormat.format(sbuf, message, params); + } else { + sbuf.append(message); + } + + if (record.getThrown() != null) { + sbuf.append(Util.NATIVE_LINE_BREAK); + sbuf.append(Util.printStackTrace(record.getThrown())); + } + + sbuf.append(Util.NATIVE_LINE_BREAK); + return sbuf.toString(); + } +} diff --git a/util/src/main/java/com/epam/deltix/util/log/SafeHandler.java b/util/src/main/java/com/epam/deltix/util/log/SafeHandler.java new file mode 100644 index 00000000..44250654 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/log/SafeHandler.java @@ -0,0 +1,280 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.log; + +import com.epam.deltix.util.lang.StringUtils; +import com.epam.deltix.util.lang.Util; +import com.epam.deltix.util.text.SimpleMessageFormat; +import com.epam.deltix.util.time.GlobalTimer; +import com.epam.deltix.util.time.TimerRunner; + +import java.util.ArrayDeque; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.*; + +/** + * Prevents large amount of similar messages from swamping log files. + * Date: 8/4/11 + * + * @author Nickolay Dul + */ +public class SafeHandler extends Handler { + private static final String HANDLERS_CLASSES_PROP = "handlers"; + private static final String MAX_RECORDS_PER_SEC_PROP = "maxRecordCountPerSecond"; + private static final String PUSH_INTERVAL_PROP = "pushIntervalMillis"; + private static final String SIMILAR_BUFFER_SIZE_PROP = "similarRecordBufferSize"; + + private Handler target; + private int maxRecordCountPerSecond; + + private RecordBuffer buffer; + private TimerRunner timerTask; + + private final AtomicLong recordCounter = new AtomicLong(0); + private volatile boolean isSafeMode; + + private long lastTimerTimestamp; + + public SafeHandler() { + LogManager manager = LogManager.getLogManager(); + String prefix = getClass().getName() + "."; + + String handlersValue = StringUtils.trim(manager.getProperty(prefix + HANDLERS_CLASSES_PROP)); + if (handlersValue == null) + throw new IllegalArgumentException("Target handlers should be specified!"); + + String[] handlerClasses = handlersValue.split(","); + Handler[] handlers = new Handler[handlerClasses.length]; + for (int i = 0; i < handlerClasses.length; i++) + handlers[i] = (Handler) Util.newInstanceNoX(handlerClasses[i].trim()); + Handler target = handlers.length == 1 ? handlers[0] : new CompoundHandler(handlers); + + initialize(target); + } + + public SafeHandler(Handler target) { + initialize(target); + } + + public SafeHandler(Handler target, int maxRecordCountPerSecond, long pushIntervalMillis, int similarRecordBufferSize) { + initialize(target, maxRecordCountPerSecond, pushIntervalMillis, similarRecordBufferSize); + } + + private void initialize(Handler target) { + LogManager manager = LogManager.getLogManager(); + String prefix = getClass().getName() + "."; + + String formatter = StringUtils.trim(manager.getProperty(prefix + "formatter")); + if (formatter != null) + target.setFormatter((Formatter) Util.newInstanceNoX(formatter)); + + int maxRecordCountPerSecond = parseInt(manager.getProperty(prefix + MAX_RECORDS_PER_SEC_PROP), 1000); + long pushIntervalMillis = parseLong(manager.getProperty(prefix + PUSH_INTERVAL_PROP), 3000); + int similarRecordBufferSize = parseInt(manager.getProperty(prefix + SIMILAR_BUFFER_SIZE_PROP), 20); + + setLevel(parseLevel(manager.getProperty(prefix + "level"), Level.ALL)); + + initialize(target, maxRecordCountPerSecond, pushIntervalMillis, similarRecordBufferSize); + } + + private void initialize(Handler target, int maxRecordCountPerSecond, long pushIntervalMillis, int similarRecordBufferSize) { + this.target = target; + this.maxRecordCountPerSecond = maxRecordCountPerSecond; + + buffer = new RecordBuffer(similarRecordBufferSize); + lastTimerTimestamp = System.currentTimeMillis(); + + timerTask = new TimerRunner() { + @Override + protected void runInternal() throws Exception { + onTimer(); + } + }; + GlobalTimer.INSTANCE.scheduleAtFixedRate(timerTask, pushIntervalMillis, pushIntervalMillis); + } + + private void onTimer() { + // retrieve record count and reset counter + long recordCount = recordCounter.getAndSet(0); + + // calculate record frequency per second + long now = System.currentTimeMillis(); + double actualIntervalSec = (now - lastTimerTimestamp) / 1000d; + double frequencyPerSec = recordCount / actualIntervalSec; + + // update safe mode flag + isSafeMode = frequencyPerSec >= maxRecordCountPerSecond; + + // publish record buffer + buffer.publish(target); + + // update last timer timestamp + lastTimerTimestamp = now; + } + + public Handler getTarget() { + return target; + } + + @Override + public void publish(LogRecord record) { + // increment record counter + recordCounter.incrementAndGet(); + // publish log record + if (isSafeMode) + buffer.add(record); + else + target.publish(record); + } + + @Override + public void flush() { + buffer.publish(target); + target.flush(); + } + + @Override + public void close() throws SecurityException { + buffer.publish(target); + timerTask.cancel(); + target.close(); + } + + private static int parseInt(String value, int defaultValue) { + value = value != null ? value.trim() : null; + return value != null ? Integer.parseInt(value) : defaultValue; + } + + private static long parseLong(String value, long defaultValue) { + value = value != null ? value.trim() : null; + return value != null ? Long.parseLong(value) : defaultValue; + } + + private static Level parseLevel(String value, Level defaultLevel) { + value = value != null ? value.trim() : null; + return value != null ? Level.parse(value) : defaultLevel; + } + + ///////////////////////// HELPER CLASSES //////////////////// + + private static final class RecordBuffer { + private static final int KEY_LENGTH = 40; + + private final int similarRecordBufferSize; + + private Map buffers; + + public RecordBuffer(int similarRecordBufferSize) { + this.similarRecordBufferSize = similarRecordBufferSize; + buffers = new HashMap(1000); + } + + public synchronized void add(LogRecord record) { + String key = getRecordKey(record); + SimilarRecordBuffer buffer = buffers.get(key); + if (buffer == null) { + buffer = new SimilarRecordBuffer(similarRecordBufferSize); + buffers.put(key, buffer); + } + buffer.add(record); + } + + public synchronized void publish(Handler target) { + if (buffers.isEmpty()) + return; + + target.publish(new LogRecord(Level.WARNING, ">>> Start publishing buffered log records")); + for (SimilarRecordBuffer buffer : buffers.values()) + target.publish(buffer.toRecord()); + target.publish(new LogRecord(Level.WARNING, "<<< Finish publishing buffered log records")); + buffers.clear(); + } + + private String getRecordKey(LogRecord record) { + // if exception details exists - use first stack trace line as key + Throwable thrown = record.getThrown(); + if (thrown != null && thrown.getStackTrace() != null && thrown.getStackTrace().length > 0) + return thrown.getStackTrace()[0].toString(); + // if record contains only message - use first KEY_LENGTH chars as key + String message = record.getMessage().trim(); + return message.substring(0, Math.min(message.length(), KEY_LENGTH)); + } + } + + private static final class SimilarRecordBuffer { + private final ArrayDeque buffer; + private final int bufferSize; + private long totalCount; + + private SimilarRecordBuffer(int bufferSize) { + this.bufferSize = bufferSize; + this.buffer = new ArrayDeque(bufferSize); + } + + public void add(LogRecord record) { + // remove first element from buffer + if (buffer.size() >= bufferSize) + buffer.poll(); + // add current record to the end of buffer + buffer.offer(record); + + // increment total record counter + totalCount++; + } + + public LogRecord toRecord() { + if (buffer.size() == 1) + return buffer.getFirst(); + + Level summaryRecordLevel = Level.WARNING; + + StringBuilder sbuf = new StringBuilder(512); + sbuf.append("There were published ").append(totalCount).append(" records with similar messages."); + sbuf.append(Util.NATIVE_LINE_BREAK); + sbuf.append("Here are the last ").append(bufferSize).append(" messages:"); + + Throwable thrown = null; + for (LogRecord record : buffer) { + // if at least one record has SEVERE level - set this level for summary record + if (record.getLevel() == Level.SEVERE) + summaryRecordLevel = Level.SEVERE; + + sbuf.append(Util.NATIVE_LINE_BREAK); + sbuf.append('\t'); + // time + sbuf.append(String.format("%1$tF %1$tT.%1$tL", record.getMillis())).append(' '); + // message + String message = record.getMessage (); + Object[] params = record.getParameters(); + if (params != null) + SimpleMessageFormat.format(sbuf, message, params); + else + sbuf.append(message); + + if (thrown == null) + thrown = record.getThrown(); + } + if (thrown != null) { + sbuf.append(Util.NATIVE_LINE_BREAK); + sbuf.append(Util.printStackTrace(thrown)); + } + return new LogRecord(summaryRecordLevel, sbuf.toString()); + } + } +} diff --git a/util/src/main/java/com/epam/deltix/util/log/TerseFormatter.java b/util/src/main/java/com/epam/deltix/util/log/TerseFormatter.java new file mode 100644 index 00000000..f0588c76 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/log/TerseFormatter.java @@ -0,0 +1,72 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.log; + +import com.epam.deltix.util.lang.Util; +import com.epam.deltix.util.text.SimpleMessageFormat; +import com.epam.deltix.util.time.TimeFormatter; + +import java.util.logging.Formatter; +import java.util.logging.LogRecord; + +/** + * Simple formatter that displays only a log message + * + * THIS CLASS IS INSTALLED AS JAVA EXTENSION - PLEASE AVOID ADDING ANY DEPENDENCIES. + * + */ +public class TerseFormatter extends Formatter { + private static CurrentMonthDate currentMonthDate = CurrentMonthDate.getInstance(); + + /** + * Format the given log record and return the formatted string. + */ + public String format (LogRecord record) { + StringBuilder sbuf = new StringBuilder(256); + + // time + long time = record.getMillis(); + if (time != 0) { + sbuf.append (currentMonthDate.getDayMonth(time)).append (' '); + sbuf.append (formatTimestamp(time)).append(' '); + } + + // level + sbuf.append(record.getLevel()).append(' '); + + // message + String message = record.getMessage (); + Object[] params = record.getParameters(); + if (params != null && params.length > 0) { + SimpleMessageFormat.format(sbuf, message, params); + } else { + sbuf.append(message); + } + + if (record.getThrown() != null) { + sbuf.append(Util.NATIVE_LINE_BREAK); + sbuf.append(Util.printStackTrace(record.getThrown())); + } + + sbuf.append(Util.NATIVE_LINE_BREAK); + return sbuf.toString(); + } + + protected String formatTimestamp(long time) { + return TimeFormatter.formatTimeOfDayGMT(time); + } +} diff --git a/util/src/main/java/com/epam/deltix/util/memory/DisposableResourceTracker.java b/util/src/main/java/com/epam/deltix/util/memory/DisposableResourceTracker.java new file mode 100644 index 00000000..cc19c4ec --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/memory/DisposableResourceTracker.java @@ -0,0 +1,138 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.memory; + +import com.epam.deltix.gflog.api.Log; +import com.epam.deltix.gflog.api.LogFactory; +import com.epam.deltix.util.lang.Disposable; +import com.epam.deltix.util.lang.Util; + +import java.io.PrintStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.HashSet; + + +/** + * + */ +public class DisposableResourceTracker implements Disposable { + private static final Log LOG = LogFactory.getLog(DisposableResourceTracker.class); + private static final HashSet mOpenResources = + new HashSet (); + + private Disposable mResource; + private Throwable mCreationStackTrace; + + public DisposableResourceTracker () { + this (null); + } + + public DisposableResourceTracker (Disposable id) { + mResource = id; + + if (id == null) + id = this; + + mCreationStackTrace = new Throwable (id + " allocated below"); + + synchronized (mOpenResources) { + mOpenResources.add (this); + } + } + + public Disposable getResource () { + return mResource; + } + + public static DisposableResourceTracker [] getOpenResources () { + synchronized (mOpenResources) { + return (mOpenResources.toArray (new DisposableResourceTracker [mOpenResources.size ()])); + } + } + + public static void dumpOpenResources () { + dumpOpenResources (System.out); + } + + public static void dumpOpenResources (PrintStream ps) { + synchronized (mOpenResources) { + for (DisposableResourceTracker dtt : mOpenResources) + dtt.dump (ps); + } + } + + public void dump (PrintStream ps) { + mCreationStackTrace.printStackTrace (ps); + } + + public String dumpToString () { + StringWriter swr = new StringWriter (); + PrintWriter pwr = new PrintWriter (swr); + + dump (pwr); + + pwr.close (); + return (swr.toString ()); + } + + public static String dumpOpenResourcesToString () { + StringWriter swr = new StringWriter (); + PrintWriter pwr = new PrintWriter (swr); + + dumpOpenResources (pwr); + + pwr.close (); + return (swr.toString ()); + } + + public static void dumpOpenResources (PrintWriter ps) { + synchronized (mOpenResources) { + for (DisposableResourceTracker dtt : mOpenResources) + dtt.dump (ps); + } + } + + public void dump (PrintWriter ps) { + mCreationStackTrace.printStackTrace (ps); + } + + public void close () { + synchronized (mOpenResources) { + mOpenResources.remove (this); + } + + mCreationStackTrace = null; + mResource = null; + } + + @Override + protected void finalize () + throws Throwable + { + if (mCreationStackTrace != null) { + LOG.error ("%s was never closed: %s").with(mCreationStackTrace.getMessage ()).with(mCreationStackTrace); + close (); + } + + super.finalize (); + + Util.close(mResource); + } + + +} diff --git a/util/src/main/java/com/epam/deltix/util/memory/HeapDumper.java b/util/src/main/java/com/epam/deltix/util/memory/HeapDumper.java new file mode 100644 index 00000000..db936094 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/memory/HeapDumper.java @@ -0,0 +1,98 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.memory; + +import com.sun.management.HotSpotDiagnosticMXBean; + +import javax.management.MBeanServer; +import java.lang.management.ManagementFactory; + +/** + * + */ +public final class HeapDumper { + // This is the name of the HotSpot Diagnostic MBean + private static final String HOTSPOT_BEAN_NAME = "com.sun.management:type=HotSpotDiagnostic"; + + // field to store the hotspot diagnostic MBean + private static volatile HotSpotDiagnosticMXBean hotspotMBean; + + /** + * Call this method from your application whenever you + * want to dump the heap snapshot into a file. + * + * @param fileName name of the heap dump file + * @param live flag that tells whether to dump + * only the live objects + */ + public static void dumpHeap(String fileName, boolean live) { + // initialize hotspot diagnostic MBean + initHotspotMBean(); + try { + hotspotMBean.dumpHeap(fileName, live); + } catch (RuntimeException re) { + throw re; + } catch (Exception exp) { + throw new RuntimeException(exp); + } + } + + // initialize the hotspot diagnostic MBean field + private static void initHotspotMBean() { + if (hotspotMBean == null) { + synchronized (HeapDumper.class) { + if (hotspotMBean == null) { + hotspotMBean = getHotspotMBean(); + } + } + } + } + + // get the hotspot diagnostic MBean from the + // platform MBean server + private static HotSpotDiagnosticMXBean getHotspotMBean() { + try { + MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + HotSpotDiagnosticMXBean bean = + ManagementFactory.newPlatformMXBeanProxy(server, + HOTSPOT_BEAN_NAME, HotSpotDiagnosticMXBean.class); + return bean; + } catch (RuntimeException re) { + throw re; + } catch (Exception exp) { + throw new RuntimeException(exp); + } + } + + public static void main(String[] args) { + // default heap dump file name + String fileName = "heap.bin"; + // by default dump only the live objects + boolean live = true; + + // simple command line options + switch (args.length) { + case 2: + live = args[1].equals("true"); + case 1: + fileName = args[0]; + } + + // dump the heap + dumpHeap(fileName, live); + } +} diff --git a/util/src/main/java/com/epam/deltix/util/net/NetUtils.java b/util/src/main/java/com/epam/deltix/util/net/NetUtils.java index 6fe6ca2a..a2da85e5 100644 --- a/util/src/main/java/com/epam/deltix/util/net/NetUtils.java +++ b/util/src/main/java/com/epam/deltix/util/net/NetUtils.java @@ -14,6 +14,7 @@ * License for the specific language governing permissions and limitations under * the License. */ + package com.epam.deltix.util.net; import com.epam.deltix.util.io.ByteArrayOutputStreamEx; @@ -40,7 +41,7 @@ public boolean isUrlAccessible (String s, int connectTimeout, int readT try { checkUrl (s, connectTimeout, readTimeout); return (true); - } catch (Throwable x) { + } catch (Exception x) { return (false); } } @@ -206,4 +207,4 @@ public static void main (String [] args) throws Exception { for (byte b : bytes) System.out.print ((char) b); } -} \ No newline at end of file +} diff --git a/util/src/main/java/com/epam/deltix/util/net/timer/TimerServerThread.java b/util/src/main/java/com/epam/deltix/util/net/timer/TimerServerThread.java index 15aa5a3a..ed2efe58 100644 --- a/util/src/main/java/com/epam/deltix/util/net/timer/TimerServerThread.java +++ b/util/src/main/java/com/epam/deltix/util/net/timer/TimerServerThread.java @@ -14,9 +14,12 @@ * License for the specific language governing permissions and limitations under * the License. */ + package com.epam.deltix.util.net.timer; -import com.epam.deltix.util.lang.*; +import com.epam.deltix.util.LangUtil; +import com.epam.deltix.util.lang.Util; + import java.io.*; import java.net.*; @@ -46,8 +49,9 @@ public void run () { } } catch (Throwable x) { Util.handleException (x); + LangUtil.propagateError(x); } finally { Util.close (ss); } } -} \ No newline at end of file +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/AuthResult.java b/util/src/main/java/com/epam/deltix/util/oauth/AuthResult.java new file mode 100644 index 00000000..d9b6feba --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/AuthResult.java @@ -0,0 +1,55 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth; + +public class AuthResult { + + private final String userName; + + private final String accessToken; + + private final String refreshToken; + + private final long expiresInSec; + + public AuthResult(String userName, String accessToken, long expiresInSec) { + this(userName, accessToken, null, expiresInSec); + } + + public AuthResult(String userName, String accessToken, String refreshToken, long expiresInSec) { + this.userName = userName; + this.accessToken = accessToken; + this.expiresInSec = expiresInSec; + this.refreshToken = refreshToken; + } + + public String userName() { + return userName; + } + + public String accessToken() { + return accessToken; + } + + public String refreshToken() { + return refreshToken; + } + + public long expiresInSec() { + return expiresInSec; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/KeystoreConfig.java b/util/src/main/java/com/epam/deltix/util/oauth/KeystoreConfig.java new file mode 100644 index 00000000..75151c90 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/KeystoreConfig.java @@ -0,0 +1,152 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth; + +/** + * Configuration for Keystore which supports different types of keystores. + * If the Keystore type is PKCS12 or JKS, the {@code location}, {@code alias}, and {@code password} are required. + * If the Keystore type is PEM, you can either specify the locations of the private key and certificate files + * or provide the key and certificate directly as strings. + */ +public class KeystoreConfig { + + private KeystoreType type = KeystoreType.PKCS12; + + private String location; + + private String alias; + + private String password; + + private String pemKeyLocation; + + private String pemCertLocation; + + private String pemKey; + + private String pemCert; + + private KeystoreConfig() { + } + + public KeystoreType getType() { + return type; + } + + public String getLocation() { + return location; + } + + public String getAlias() { + return alias; + } + + public String getPassword() { + return password; + } + + public String getPemKeyLocation() { + return pemKeyLocation; + } + + public String getPemCertLocation() { + return pemCertLocation; + } + + public String getPemKey() { + return pemKey; + } + + public String getPemCert() { + return pemCert; + } + + public static KeystoreConfig.Builder builder() { + return new KeystoreConfig().new Builder(); + } + + public class Builder { + + private Builder() { + } + + /** + * Configures the Keystore as PKCS12 type. + * + * @param location the location of the keystore file + * @param alias the alias used in the keystore + * @param password the password for the keystore + * @return the builder + */ + public KeystoreConfig.Builder withPkcs12(String location, String alias, String password) { + KeystoreConfig.this.type = KeystoreType.PKCS12; + KeystoreConfig.this.location = location; + KeystoreConfig.this.alias = alias; + KeystoreConfig.this.password = password; + return this; + } + + /** + * Configures the Keystore as JKS type. + * + * @param location the location of the keystore file + * @param alias the alias used in the keystore + * @param password the password for the keystore + * @return the builder + */ + public KeystoreConfig.Builder withJks(String location, String alias, String password) { + KeystoreConfig.this.type = KeystoreType.JKS; + KeystoreConfig.this.location = location; + KeystoreConfig.this.alias = alias; + KeystoreConfig.this.password = password; + return this; + } + + /** + * Configures the PEM Keystore with file locations. + * + * @param pemKeyLocation the location of the PEM private key file + * @param pemCertLocation the location of the PEM certificate file + * @return the builder + */ + public KeystoreConfig.Builder withPemFiles(String pemKeyLocation, String pemCertLocation) { + KeystoreConfig.this.type = KeystoreType.PEM; + KeystoreConfig.this.pemKeyLocation = pemKeyLocation; + KeystoreConfig.this.pemCertLocation = pemCertLocation; + return this; + } + + /** + * Configures the PEM Keystore with PEM strings. + * + * @param pemKey the PEM private key as a string + * @param pemCert the PEM certificate as a string + * @return the builder + */ + public KeystoreConfig.Builder withPem(String pemKey, String pemCert) { + KeystoreConfig.this.type = KeystoreType.PEM; + KeystoreConfig.this.pemKey = pemKey; + KeystoreConfig.this.pemCert = pemCert; + return this; + } + + public KeystoreConfig build() { + return KeystoreConfig.this; + } + + } +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/KeystoreType.java b/util/src/main/java/com/epam/deltix/util/oauth/KeystoreType.java new file mode 100644 index 00000000..f5d45cba --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/KeystoreType.java @@ -0,0 +1,23 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth; + +public enum KeystoreType { + PKCS12, + JKS, + PEM +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/Oauth2Client.java b/util/src/main/java/com/epam/deltix/util/oauth/Oauth2Client.java new file mode 100644 index 00000000..c3ceb473 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/Oauth2Client.java @@ -0,0 +1,38 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.epam.deltix.util.oauth; + +import com.epam.deltix.util.lang.Disposable; +import com.epam.deltix.util.oauth.service.Oauth2ClientImpl; + +public interface Oauth2Client extends Disposable { + + static Oauth2Client create(Oauth2ClientConfig config) { + return Oauth2ClientImpl.of(config); + } + + AuthResult login(); + + default String clientId() { + return login().userName(); + } + + default String token() { + return login().accessToken(); + } + +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/Oauth2ClientConfig.java b/util/src/main/java/com/epam/deltix/util/oauth/Oauth2ClientConfig.java new file mode 100644 index 00000000..5498f9a3 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/Oauth2ClientConfig.java @@ -0,0 +1,155 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth; + +import com.epam.deltix.util.oauth.service.Oauth2ClientImpl; + +import java.util.HashMap; +import java.util.Map; +import java.util.Timer; + +public class Oauth2ClientConfig { + + private String url; + + private final Map parameters = new HashMap<>(); + + private KeystoreConfig keystoreConfig; + + private Timer timer; + + private RefreshTokenListener listener; + + private long timeoutMs = 5000; + private int connectTimeoutMs = 5000; + private int readTimeoutMs = 5000; + + private double expirationMultiplier = 0.7; + + private Oauth2ClientConfig() { + } + + public String getUrl() { + return url; + } + + public Map getParameters() { + return parameters; + } + + public KeystoreConfig getKeystoreConfig() { + return keystoreConfig; + } + + public Timer getTimer() { + return timer; + } + + public RefreshTokenListener getListener() { + return listener; + } + + public long getTimeoutMs() { + return timeoutMs; + } + + public int getConnectTimeoutMs() { + return connectTimeoutMs; + } + + public int getReadTimeoutMs() { + return readTimeoutMs; + } + + public double getExpirationMultiplier() { + return expirationMultiplier; + } + + public static Oauth2ClientConfig.Builder builder() { + return new Oauth2ClientConfig().new Builder(); + } + + public class Builder { + + private Builder() { + } + + public Oauth2ClientConfig.Builder withUrl(String url) { + Oauth2ClientConfig.this.url = url; + return this; + } + + public Oauth2ClientConfig.Builder withClientCredentials(String clientId, String clientSecret) { + parameters.put(Oauth2ClientImpl.GRANT_TYPE_PARAM, Oauth2ClientImpl.CLIENT_CREDENTIALS_GRANT_TYPE); + parameters.put(Oauth2ClientImpl.CLIENT_ID_PARAM, clientId); + parameters.put(Oauth2ClientImpl.CLIENT_SECRET_PARAM, clientSecret); + return this; + } + + public Oauth2ClientConfig.Builder withClientCredentials(String clientId, KeystoreConfig keystoreConfig) { + parameters.put(Oauth2ClientImpl.GRANT_TYPE_PARAM, Oauth2ClientImpl.CLIENT_CREDENTIALS_GRANT_TYPE); + parameters.put(Oauth2ClientImpl.CLIENT_ID_PARAM, clientId); + Oauth2ClientConfig.this.keystoreConfig = keystoreConfig; + return this; + } + + public Oauth2ClientConfig.Builder withKeystoreConfig(KeystoreConfig keystoreConfig) { + Oauth2ClientConfig.this.keystoreConfig = keystoreConfig; + return this; + } + + public Oauth2ClientConfig.Builder withParameter(String name, String value) { + Oauth2ClientConfig.this.parameters.put(name, value); + return this; + } + + public Oauth2ClientConfig.Builder withTimer(Timer timer) { + Oauth2ClientConfig.this.timer = timer; + return this; + } + + public Oauth2ClientConfig.Builder withListener(RefreshTokenListener listener) { + Oauth2ClientConfig.this.listener = listener; + return this; + } + + public Oauth2ClientConfig.Builder withTimeout(long timeoutMs) { + Oauth2ClientConfig.this.timeoutMs = timeoutMs; + return this; + } + + public Oauth2ClientConfig.Builder withConnectTimeout(int connectTimeoutMs) { + Oauth2ClientConfig.this.connectTimeoutMs = connectTimeoutMs; + return this; + } + + public Oauth2ClientConfig.Builder withReadTimeout(int readTimeoutMs) { + Oauth2ClientConfig.this.readTimeoutMs = readTimeoutMs; + return this; + } + + public Oauth2ClientConfig.Builder withExpirationMultiplier(double multiplier) { + Oauth2ClientConfig.this.expirationMultiplier = multiplier; + return this; + } + + public Oauth2ClientConfig build() { + return Oauth2ClientConfig.this; + } + + } +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/Oauth2CodeClient.java b/util/src/main/java/com/epam/deltix/util/oauth/Oauth2CodeClient.java new file mode 100644 index 00000000..7acf614d --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/Oauth2CodeClient.java @@ -0,0 +1,29 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth; + +import com.epam.deltix.util.oauth.authcode.Oauth2CodeClientImpl; + +public interface Oauth2CodeClient extends Oauth2Client { + static Oauth2CodeClient create(Oauth2CodeClientConfig config) { + return Oauth2CodeClientImpl.of(config); + } + + static Oauth2CodeClient create(Oauth2CodeClientConfig config, RefreshTokenListener listener) { + return Oauth2CodeClientImpl.of(config, listener); + } +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/Oauth2CodeClientConfig.java b/util/src/main/java/com/epam/deltix/util/oauth/Oauth2CodeClientConfig.java new file mode 100644 index 00000000..a96a7de1 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/Oauth2CodeClientConfig.java @@ -0,0 +1,207 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth; + +import java.util.HashMap; +import java.util.Map; +import java.util.Timer; +import java.util.concurrent.ScheduledExecutorService; + +public class Oauth2CodeClientConfig { + + private String issuer; + + private String clientId; + + private int redirectPort = 4278; + + private String scope; + + private String usernameClaim = "preferred_username"; + + private String authorizationEndpoint; + + private String tokenEndpoint; + + private boolean withPkce = false; + + private boolean validateState = true; + + private final Map additionalParams = new HashMap<>(); + + private int connectTimeoutMs = 10000; + + private int readTimeoutMs = 10000; + + private double expirationMultiplier = 0.8; + + private double refreshRetriesCount = 5; + + private Timer timer; + + private ScheduledExecutorService executor; + + public static Builder builder() { + return new Oauth2CodeClientConfig().new Builder(); + } + + public String getIssuer() { + return issuer; + } + + public String getClientId() { + return clientId; + } + + public String getScope() { + return scope; + } + + public int getRedirectPort() { + return redirectPort; + } + + public String getUsernameClaim() { + return usernameClaim; + } + + public String getAuthorizationEndpoint() { + return authorizationEndpoint; + } + + public String getTokenEndpoint() { + return tokenEndpoint; + } + + public boolean isWithPkce() { + return withPkce; + } + + public boolean isValidateState() { + return validateState; + } + + public Map getAdditionalParams() { + return additionalParams; + } + + public double getExpirationMultiplier() { + return expirationMultiplier; + } + + public int getConnectTimeoutMs() { + return connectTimeoutMs; + } + + public int getReadTimeoutMs() { + return readTimeoutMs; + } + + public double getRefreshRetriesCount() { + return refreshRetriesCount; + } + + public Timer getTimer() { + return timer; + } + + public ScheduledExecutorService getExecutor() { + return executor; + } + + + public class Builder { + private Builder() { + } + + public Builder withIssuer(String issuer) { + Oauth2CodeClientConfig.this.issuer = issuer; + return this; + } + + public Builder withClientId(String clientId) { + Oauth2CodeClientConfig.this.clientId = clientId; + return this; + } + + public Builder withScope(String scope) { + Oauth2CodeClientConfig.this.scope = scope; + return this; + } + + public Builder withRedirectPort(int redirectPort) { + Oauth2CodeClientConfig.this.redirectPort = redirectPort; + return this; + } + + public Builder withUsernameClaim(String usernameClaim) { + Oauth2CodeClientConfig.this.usernameClaim = usernameClaim; + return this; + } + + public Builder withAuthorizationEndpoint(String authorizationEndpoint) { + Oauth2CodeClientConfig.this.authorizationEndpoint = authorizationEndpoint; + return this; + } + + public Builder withTokenEndpoint(String tokenEndpoint) { + Oauth2CodeClientConfig.this.tokenEndpoint = tokenEndpoint; + return this; + } + + public Builder withPkce(boolean withPkce) { + Oauth2CodeClientConfig.this.withPkce = withPkce; + return this; + } + + public Builder withValidateState(boolean validateState) { + Oauth2CodeClientConfig.this.validateState = validateState; + return this; + } + + public Builder withAdditionalParam(String key, String value) { + Oauth2CodeClientConfig.this.additionalParams.put(key, value); + return this; + } + + public Builder withExpirationMultiplier(double multiplier) { + Oauth2CodeClientConfig.this.expirationMultiplier = multiplier; + return this; + } + + public Builder withRefreshRetriesCount(int count) { + Oauth2CodeClientConfig.this.refreshRetriesCount = count; + return this; + } + + public Builder withTimer(Timer timer) { + Oauth2CodeClientConfig.this.timer = timer; + return this; + } + + public Builder withExecutor(ScheduledExecutorService executor) { + Oauth2CodeClientConfig.this.executor = executor; + return this; + } + + public Oauth2CodeClientConfig build() { + return Oauth2CodeClientConfig.this; + } + + } + +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/RefreshTokenListener.java b/util/src/main/java/com/epam/deltix/util/oauth/RefreshTokenListener.java new file mode 100644 index 00000000..9431b5d1 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/RefreshTokenListener.java @@ -0,0 +1,28 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.oauth; + +import com.epam.deltix.util.LangUtil; + +public interface RefreshTokenListener { + + void refreshed(AuthResult authResult); + + default void refreshFailed(Throwable t) { LangUtil.propagateError(t); } + +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/authcode/AuthCodeClient.java b/util/src/main/java/com/epam/deltix/util/oauth/authcode/AuthCodeClient.java new file mode 100644 index 00000000..2a6c262d --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/authcode/AuthCodeClient.java @@ -0,0 +1,164 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.oauth.authcode; + +import com.github.scribejava.core.builder.ServiceBuilder; +import com.github.scribejava.core.builder.api.DefaultApi20; +import com.github.scribejava.core.httpclient.jdk.JDKHttpClientConfig; +import com.github.scribejava.core.model.OAuth2AccessToken; +import com.github.scribejava.core.oauth.AccessTokenRequestParams; +import com.github.scribejava.core.oauth.AuthorizationUrlBuilder; +import com.github.scribejava.core.oauth.OAuth20Service; +import com.github.scribejava.core.oauth2.clientauthentication.ClientAuthentication; +import com.github.scribejava.core.oauth2.clientauthentication.RequestBodyAuthenticationScheme; +import com.epam.deltix.util.oauth.AuthResult; +import com.epam.deltix.util.oauth.Oauth2CodeClientConfig; +import com.epam.deltix.util.oauth.utils.TokenUtils; + +import java.io.IOException; +import java.util.concurrent.ExecutionException; + +class AuthCodeClient { + + private static final int AUTHORIZATION_TIMEOUT = Integer.getInteger("TimeBase.oauth2.browserAuthTimeoutSec", 60); + + private static class OpenIdConfigurationApi extends DefaultApi20 { + + private final String tokenEndpoint; + + private final String authorizationEndpoint; + + public OpenIdConfigurationApi(String tokenEndpoint, String authorizationEndpoint) { + this.tokenEndpoint = tokenEndpoint; + this.authorizationEndpoint = authorizationEndpoint; + } + + @Override + public String getAccessTokenEndpoint() { + return tokenEndpoint; + } + + @Override + protected String getAuthorizationBaseUrl() { + return authorizationEndpoint; + } + + @Override + public ClientAuthentication getClientAuthentication() { + return RequestBodyAuthenticationScheme.instance(); + } + + } + + private final Oauth2CodeClientConfig config; + private final OpenIdConfigurationApi openIdConfig; + + AuthCodeClient(Oauth2CodeClientConfig config) { + this.config = config; + this.openIdConfig = new OpenIdConfigurationApi(config.getTokenEndpoint(), config.getAuthorizationEndpoint()); + } + + AuthResult requestToken() { + try (AuthCodeProvider codeProvider = + new SystemBrowserAuthCodeProvider(config.getRedirectPort(), AUTHORIZATION_TIMEOUT); + OAuth20Service service = openClient(codeProvider)) { + + String state = generateState(); + AuthorizationUrlBuilder authorizationUrlBuilder = service.createAuthorizationUrlBuilder(); + if (config.isValidateState()) { + authorizationUrlBuilder = authorizationUrlBuilder.state(state); + } + if (config.isWithPkce()) { + authorizationUrlBuilder = authorizationUrlBuilder.initPKCE(); + } + if (config.getAdditionalParams() != null && !config.getAdditionalParams().isEmpty()) { + authorizationUrlBuilder = authorizationUrlBuilder.additionalParams(config.getAdditionalParams()); + } + + String authorizationUrl = authorizationUrlBuilder.build(); + AuthCodeResult code = codeProvider.requestCode(authorizationUrl); + if (config.isValidateState()) { + if (!state.equals(code.state())) { + throw new RuntimeException("Invalid state received: " + code.state() + "; required: " + state); + } + } + + OAuth2AccessToken token; + if (config.isWithPkce()) { + token = service.getAccessToken( + AccessTokenRequestParams.create(code.code()) + .pkceCodeVerifier(authorizationUrlBuilder.getPkce().getCodeVerifier()) + ); + } else { + token = service.getAccessToken(code.code()); + } + + return new AuthResult( + TokenUtils.extractUserName(token.getAccessToken(), config.getUsernameClaim()), + token.getAccessToken(), + token.getRefreshToken(), + token.getExpiresIn() + ); + } catch (Exception t) { + throw new RuntimeException(t); + } + } + + private String generateState() { + return "state" + (long) (Math.random() * 1_000_000L); + } + + AuthResult refreshToken(String refreshToken) { + try (OAuth20Service service = openClient()) { + + OAuth2AccessToken token = service.refreshAccessToken(refreshToken); + return new AuthResult( + TokenUtils.extractUserName(token.getAccessToken(), config.getUsernameClaim()), + token.getAccessToken(), + token.getRefreshToken(), + token.getExpiresIn() + ); + } catch (IOException | InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + + private OAuth20Service openClient() { + return new ServiceBuilder(config.getClientId()) + .defaultScope(config.getScope()) + .httpClientConfig( + JDKHttpClientConfig.defaultConfig() + .withConnectTimeout(config.getConnectTimeoutMs()) + .withReadTimeout(config.getReadTimeoutMs()) + ) + .build(openIdConfig); + } + + private OAuth20Service openClient(AuthCodeProvider codeProvider) { + return new ServiceBuilder(config.getClientId()) + .defaultScope(config.getScope()) + .callback("http://localhost:" + codeProvider.redirectPort()) + .httpClientConfig( + JDKHttpClientConfig.defaultConfig() + .withConnectTimeout(config.getConnectTimeoutMs()) + .withReadTimeout(config.getReadTimeoutMs()) + ) + .build(openIdConfig); + } + +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/authcode/AuthCodeProvider.java b/util/src/main/java/com/epam/deltix/util/oauth/authcode/AuthCodeProvider.java new file mode 100644 index 00000000..18e8dfa5 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/authcode/AuthCodeProvider.java @@ -0,0 +1,25 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth.authcode; + +interface AuthCodeProvider extends AutoCloseable { + + AuthCodeResult requestCode(String authorizationUrl); + + int redirectPort(); + +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/authcode/AuthCodeResponseHandler.java b/util/src/main/java/com/epam/deltix/util/oauth/authcode/AuthCodeResponseHandler.java new file mode 100644 index 00000000..72258cba --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/authcode/AuthCodeResponseHandler.java @@ -0,0 +1,168 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.oauth.authcode; + +import com.epam.deltix.util.oauth.utils.Utils; +import com.sun.net.httpserver.Headers; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.BlockingQueue; + +class AuthCodeResponseHandler implements HttpHandler { + + private static final String DEFAULT_SUCCESS_MESSAGE = "Authentication CompleteAuthentication complete. You can close the browser and return to the application."; + private static final String DEFAULT_SUCCESS_RESOURCE = "/deltix/util/oauth/success-login.html"; + + private static final String DEFAULT_FAILURE_MESSAGE = "Authentication FailedAuthentication failed. You can return to the application. Feel free to close this browser tab.



Error details: error {0} error_description: {1}"; + private static final String DEFAULT_FAILURE_RESOURCE = "/deltix/util/oauth/failure-login.html"; + + private final BlockingQueue authCodeResultQueue; + + private final AuthorizationBrowserOptions browserOptions; + + AuthCodeResponseHandler(BlockingQueue authCodeResultQueue, + AuthorizationBrowserOptions browserOptions) { + + this.authCodeResultQueue = authCodeResultQueue; + this.browserOptions = browserOptions; + } + + @Override + public void handle(HttpExchange httpExchange) throws IOException { + try { + if (!httpExchange.getRequestURI().getPath().equalsIgnoreCase("/")) { + httpExchange.sendResponseHeaders(200, 0); + return; + } + + AuthCodeResult result = AuthCodeResult.fromResponseBody( + extractResponseBody(httpExchange.getRequestURI()) + ); + sendResponse(httpExchange, result); + authCodeResultQueue.put(result); + } catch (Exception t) { + throw new RuntimeException(t); + } finally { + httpExchange.close(); + } + } + + private String extractResponseBody(URI uri) { + String requestUri = uri.toString(); + int startParams = requestUri.indexOf("?"); + if (startParams >= 0) { + return requestUri.substring(startParams + 1); + } + + return requestUri; + } + + private void sendResponse(HttpExchange httpExchange, AuthCodeResult result) throws IOException { + switch (result.status()) { + case Success: + sendSuccessResponse(httpExchange, getSuccessfulResponseMessage()); + break; + case ProtocolError: + case UnknownError: + sendErrorResponse(httpExchange, getErrorResponseMessage(result)); + break; + } + } + + private void sendSuccessResponse(HttpExchange httpExchange, String response) throws IOException { + if (browserOptions == null || browserOptions.getSuccessRedirectUri() == null) { + send200Response(httpExchange, response); + } else { + send302Response(httpExchange, browserOptions.getSuccessRedirectUri().toString()); + } + } + + private void sendErrorResponse(HttpExchange httpExchange, String response) throws IOException { + if (browserOptions == null || browserOptions.getErrorRedirectUri() == null) { + send200Response(httpExchange, response); + } else { + send302Response(httpExchange, browserOptions.getErrorRedirectUri().toString()); + } + } + + private void send302Response(HttpExchange httpExchange, String redirectUri) throws IOException { + Headers responseHeaders = httpExchange.getResponseHeaders(); + responseHeaders.set("Location", redirectUri); + httpExchange.sendResponseHeaders(302, 0); + } + + private void send200Response(HttpExchange httpExchange, String response) throws IOException { + byte[] bytes = response.getBytes(StandardCharsets.UTF_8); + httpExchange.sendResponseHeaders(200, bytes.length); + try (OutputStream os = httpExchange.getResponseBody()) { + os.write(bytes); + } + } + + private String getSuccessfulResponseMessage() { + if (browserOptions == null || browserOptions.getSuccessMessage() == null) { + return getDefaultSuccessMessage(); + } + + return browserOptions.getSuccessMessage(); + } + + private String getDefaultSuccessMessage() { + try { + return Utils.getResourceFileAsString(DEFAULT_SUCCESS_RESOURCE); + } catch (Exception e) { + return DEFAULT_SUCCESS_MESSAGE; + } + } + + private String getErrorResponseMessage(AuthCodeResult result) { + if (browserOptions == null || browserOptions.getErrorMessage() == null) { + return formatErrorMessage(getDefaultFailureMessage(), result); + } + + return formatErrorMessage(browserOptions.getErrorMessage(), result); + } + + private String getDefaultFailureMessage() { + try { + return Utils.getResourceFileAsString(DEFAULT_FAILURE_RESOURCE); + } catch (Exception e) { + return DEFAULT_FAILURE_MESSAGE; + } + } + + private String formatErrorMessage(String message, AuthCodeResult result) { + try { + return message + .replace("{{result-error}}", result.error()) + .replace("{{result-error-description}}", result.errorDescription()); + } catch (Exception t) { + return browserOptions.getErrorMessage(); + } + } + + public BlockingQueue authorizationResultQueue() { + return this.authCodeResultQueue; + } + +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/authcode/AuthCodeResult.java b/util/src/main/java/com/epam/deltix/util/oauth/authcode/AuthCodeResult.java new file mode 100644 index 00000000..6b09f53a --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/authcode/AuthCodeResult.java @@ -0,0 +1,136 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth.authcode; + +import java.net.URLDecoder; +import java.util.LinkedHashMap; +import java.util.Map; + +class AuthCodeResult { + private String code; + private String state; + private AuthorizationStatus status; + private String error; + private String errorDescription; + + enum AuthorizationStatus { + Success, ProtocolError, UnknownError; + } + + static AuthCodeResult fromResponseBody(String responseBody) { + if (isBlank(responseBody)) { + return new AuthCodeResult(AuthorizationStatus.UnknownError, "Error", + "The authorization server returned an invalid response: response is null or empty" + ); + } + + Map queryParameters = parseParameters(responseBody); + if (queryParameters.containsKey("error")) { + return new AuthCodeResult(AuthorizationStatus.ProtocolError, queryParameters.get("error"), + !isBlank(queryParameters.get("error_description")) ? queryParameters.get("error_description") : null + ); + } + if (!queryParameters.containsKey("code")) { + return new AuthCodeResult(AuthorizationStatus.UnknownError, "Error", + "Authorization result response does not contain authorization code" + ); + } + + AuthCodeResult result = new AuthCodeResult(); + result.code = queryParameters.get("code"); + result.status = AuthorizationStatus.Success; + if (queryParameters.containsKey("state")) { + result.state = queryParameters.get("state"); + } + + return result; + } + + private AuthCodeResult() { + } + + private AuthCodeResult(AuthorizationStatus status, String error, String errorDescription) { + this.status = status; + this.error = error; + this.errorDescription = errorDescription; + } + + private static Map parseParameters(String serverResponse) { + Map query_pairs = new LinkedHashMap<>(); + try { + String[] pairs = serverResponse.split("&"); + for (String pair : pairs) { + int idx = pair.indexOf("="); + String key = URLDecoder.decode(pair.substring(0, idx), "UTF-8"); + String value = URLDecoder.decode(pair.substring(idx + 1), "UTF-8"); + query_pairs.put(key, value); + } + } catch (Exception ex) { + throw new RuntimeException(String.format("Error parsing authorization result: %s", ex.getMessage())); + } + + return query_pairs; + } + + String code() { + return this.code; + } + + String state() { + return this.state; + } + + AuthorizationStatus status() { + return this.status; + } + + String error() { + return this.error; + } + + String errorDescription() { + return errorDescription; + } + + AuthCodeResult code(final String code) { + this.code = code; + return this; + } + + AuthCodeResult state(final String state) { + this.state = state; + return this; + } + + AuthCodeResult status(final AuthorizationStatus status) { + this.status = status; + return this; + } + + AuthCodeResult error(final String error) { + this.error = error; + return this; + } + + AuthCodeResult errorDescription(final String errorDescription) { + this.errorDescription = errorDescription; + return this; + } + private static boolean isBlank(String value) { + return value == null || value.isEmpty(); + } +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/authcode/AuthorizationBrowserOptions.java b/util/src/main/java/com/epam/deltix/util/oauth/authcode/AuthorizationBrowserOptions.java new file mode 100644 index 00000000..64a18a82 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/authcode/AuthorizationBrowserOptions.java @@ -0,0 +1,62 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth.authcode; + +import java.net.URI; + +public class AuthorizationBrowserOptions { + + private String successMessage; + + private String errorMessage; + + private URI successRedirectUri; + + private URI errorRedirectUri; + + public String getSuccessMessage() { + return successMessage; + } + + public void setSuccessMessage(String successMessage) { + this.successMessage = successMessage; + } + + public String getErrorMessage() { + return errorMessage; + } + + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } + + public URI getSuccessRedirectUri() { + return successRedirectUri; + } + + public void setSuccessRedirectUri(URI successRedirectUri) { + this.successRedirectUri = successRedirectUri; + } + + public URI getErrorRedirectUri() { + return errorRedirectUri; + } + + public void setErrorRedirectUri(URI errorRedirectUri) { + this.errorRedirectUri = errorRedirectUri; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/authcode/HttpListener.java b/util/src/main/java/com/epam/deltix/util/oauth/authcode/HttpListener.java new file mode 100644 index 00000000..7a629abe --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/authcode/HttpListener.java @@ -0,0 +1,55 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth.authcode; + +import com.epam.deltix.gflog.api.Log; +import com.epam.deltix.gflog.api.LogFactory; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; + +import java.net.InetSocketAddress; + +class HttpListener { + private static final Log LOG = LogFactory.getLog(HttpListener.class); + + private HttpServer server; + private int port; + + void startListener(int port, HttpHandler httpHandler) { + try { + server = HttpServer.create(new InetSocketAddress(port), 0); + server.createContext("/", httpHandler); + this.port = server.getAddress().getPort(); + server.start(); + LOG.debug("Http listener started. Listening on port: " + port); + } catch (Exception e) { + LOG.error().append("Http listener (").append(port).append(".").append(e).commit(); + throw new RuntimeException(e.getMessage()); + } + } + + void stopListener() { + if (server != null) { + server.stop(0); + LOG.debug("Http listener stopped"); + } + } + + int port() { + return port; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/authcode/Oauth2CodeClientImpl.java b/util/src/main/java/com/epam/deltix/util/oauth/authcode/Oauth2CodeClientImpl.java new file mode 100644 index 00000000..4202e49f --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/authcode/Oauth2CodeClientImpl.java @@ -0,0 +1,164 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.oauth.authcode; + +import com.epam.deltix.gflog.api.Log; +import com.epam.deltix.gflog.api.LogFactory; +import com.epam.deltix.util.oauth.AuthResult; +import com.epam.deltix.util.oauth.Oauth2CodeClient; +import com.epam.deltix.util.oauth.Oauth2CodeClientConfig; +import com.epam.deltix.util.oauth.RefreshTokenListener; +import com.epam.deltix.util.oauth.utils.ExecutorRefreshScheduler; +import com.epam.deltix.util.oauth.utils.IncreasingDelayRetryStrategy; +import com.epam.deltix.util.oauth.utils.RefreshTokenScheduler; +import com.epam.deltix.util.oauth.utils.RetryStrategy; +import com.epam.deltix.util.oauth.utils.TimerTokenScheduler; +import com.epam.deltix.util.time.TimeKeeper; + +import java.time.Instant; + +public class Oauth2CodeClientImpl implements Oauth2CodeClient { + + public static final Log LOGGER = LogFactory.getLog(AuthCodeClient.class.getName()); + + private final Oauth2CodeClientConfig config; + private final AuthCodeClient client; + + private final RefreshTokenScheduler refreshScheduler; + + private final RefreshTokenListener listener; + + private final RetryStrategy retryStrategy = new IncreasingDelayRetryStrategy(); + + private volatile AuthResult authResult; + private volatile long expirationTimestampMs; + + private volatile boolean closed; + + public static Oauth2CodeClientImpl of(Oauth2CodeClientConfig config) { + return new Oauth2CodeClientImpl(config, null); + } + + public static Oauth2CodeClientImpl of(Oauth2CodeClientConfig config, RefreshTokenListener listener) { + return new Oauth2CodeClientImpl(config, listener); + } + + private Oauth2CodeClientImpl(Oauth2CodeClientConfig config, RefreshTokenListener listener) { + this.config = config; + this.client = new AuthCodeClient(config); + + if (config.getTimer() != null) { + this.refreshScheduler = new TimerTokenScheduler(config.getTimer()); + } else if (config.getExecutor() != null) { + this.refreshScheduler = new ExecutorRefreshScheduler(config.getExecutor()); + } else { + this.refreshScheduler = null; + } + + this.listener = listener; + } + + @Override + public synchronized AuthResult login() { + if (authResult != null) { + return authResult; + } + + long requestTime = currentTime(); + authResult = client.requestToken(); + long delayMs = updateExpirationTimeAndGetDelay(requestTime); + LOGGER.info().append("Token requested (grant type: authorization_code); expiration timestamp: ") + .append(Instant.ofEpochMilli(expirationTimestampMs)).commit(); + + scheduleRefresh(delayMs); + return authResult; + } + + private long currentTime() { + return TimeKeeper.currentTime; + } + + private void scheduleRefresh(long delayMs) { + if (refreshScheduler != null && !closed) { + if (authResult.refreshToken() != null) { + refreshScheduler.schedule(delayMs, this::refreshTokenTask); + LOGGER.info().append("Refresh token task scheduled in ").append(delayMs / 1000).append(" seconds.").commit(); + } else { + LOGGER.warn().append("Refresh token is empty. Refresh task can't be scheduled.").commit(); + } + } + } + + private void refreshTokenTask() { + try { + if (closed) { + return; + } + + long requestTime = currentTime(); + authResult = refreshToken(); + long delayMs = updateExpirationTimeAndGetDelay(requestTime); + + LOGGER.info().append("Token refreshed (grant type: authorization_code); expiration timestamp: ") + .append(Instant.ofEpochMilli(expirationTimestampMs)).commit(); + + notifyRefreshed(); + retryStrategy.refreshRetryDelay(); + + scheduleRefresh(delayMs); + } catch (Exception t) { + LOGGER.warn().append("Failed to refresh token.").append(t).commit(); + + if (retryStrategy.retriesMade() > config.getRefreshRetriesCount()) { + notifyRefreshFailed(t); + } else { + scheduleRefresh(retryStrategy.nextRetryDelay()); + } + } + } + + private long updateExpirationTimeAndGetDelay(long requestTime) { + long expirationDelayMs = authResult.expiresInSec() * (long) (config.getExpirationMultiplier() * 1000); + expirationTimestampMs = requestTime + expirationDelayMs; + return expirationDelayMs; + } + + private AuthResult refreshToken() { + return client.refreshToken(authResult.refreshToken()); + } + + private void notifyRefreshed() { + if (listener != null) { + listener.refreshed(authResult); + } + } + + private void notifyRefreshFailed(Exception t) { + if (listener != null) { + listener.refreshFailed(t); + } + } + + @Override + public void close() { + closed = true; + if (refreshScheduler != null) { + refreshScheduler.close(); + } + } +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/authcode/SystemBrowserAuthCodeProvider.java b/util/src/main/java/com/epam/deltix/util/oauth/authcode/SystemBrowserAuthCodeProvider.java new file mode 100644 index 00000000..b90b9c07 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/authcode/SystemBrowserAuthCodeProvider.java @@ -0,0 +1,98 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth.authcode; + +import java.awt.*; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +class SystemBrowserAuthCodeProvider implements AuthCodeProvider { + + private final int redirectPort; + private final int authorizationTimeoutSec; + + private final HttpListener httpListener = new HttpListener(); + private final LinkedBlockingQueue resultQueue; + + SystemBrowserAuthCodeProvider(int redirectPort) { + this(redirectPort, 60); + } + + SystemBrowserAuthCodeProvider(int redirectPort, int authorizationTimeoutSec) { + this.redirectPort = redirectPort; + this.authorizationTimeoutSec = authorizationTimeoutSec; + this.resultQueue = startHttpListener(httpListener); + } + + @Override + public AuthCodeResult requestCode(String authorizationUrl) { + openDefaultSystemBrowser(authorizationUrl); + return getAuthorizationResultFromHttpListener(resultQueue); + } + + @Override + public int redirectPort() { + return httpListener.port(); + } + + @Override + public void close() { + httpListener.stopListener(); + } + + private LinkedBlockingQueue startHttpListener(HttpListener httpListener) { + LinkedBlockingQueue authCodeResultQueue = new LinkedBlockingQueue<>(); + AuthCodeResponseHandler authCodeResponseHandler = + new AuthCodeResponseHandler(authCodeResultQueue, null); + + int port = redirectPort == -1 ? 0 : redirectPort; + httpListener.startListener(port, authCodeResponseHandler); + return authCodeResultQueue; + } + + private void openDefaultSystemBrowser(String authorizationUrl) { + try { + if (Desktop.isDesktopSupported() && Desktop.getDesktop().isSupported(Desktop.Action.BROWSE)) { + Desktop.getDesktop().browse(new URL(authorizationUrl).toURI()); + } else { + throw new RuntimeException("Unable to open default system browser."); + } + } catch (URISyntaxException | IOException ex) { + throw new RuntimeException(ex); + } + } + + private AuthCodeResult getAuthorizationResultFromHttpListener(LinkedBlockingQueue authCodeResultQueue) { + AuthCodeResult result = null; + try { + long expirationTime = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()) + authorizationTimeoutSec; + while (result == null && TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()) < expirationTime) { + result = authCodeResultQueue.poll(100, TimeUnit.MILLISECONDS); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + if (result == null || result.code() == null || result.code().isEmpty()) { + throw new RuntimeException("No Authorization code was returned from the server"); + } + return result; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/service/CertificateTokenQuery.java b/util/src/main/java/com/epam/deltix/util/oauth/service/CertificateTokenQuery.java new file mode 100644 index 00000000..b0275a23 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/service/CertificateTokenQuery.java @@ -0,0 +1,259 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.oauth.service; + +import com.nimbusds.jose.JOSEObjectType; +import com.nimbusds.jose.JWSAlgorithm; +import com.nimbusds.jose.JWSHeader; +import com.nimbusds.jose.JWSSigner; +import com.nimbusds.jose.crypto.ECDSASigner; +import com.nimbusds.jose.crypto.RSASSASigner; +import com.nimbusds.jose.jwk.ECKey; +import com.nimbusds.jose.jwk.JWK; +import com.nimbusds.jose.jwk.RSAKey; +import com.nimbusds.jose.util.IOUtils; +import com.nimbusds.jose.util.X509CertUtils; +import com.nimbusds.jwt.JWTClaimsSet; +import com.nimbusds.jwt.SignedJWT; +import com.epam.deltix.util.oauth.KeystoreConfig; +import com.epam.deltix.util.oauth.KeystoreType; + +import java.io.File; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.*; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; +import java.security.interfaces.ECPrivateKey; +import java.security.interfaces.RSAPrivateKey; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +class CertificateTokenQuery implements TokenQuery { + + private static final long EXPIRATION_TIME_MS = 10 * 60 * 1000; + + private final String url; + private final String clientId; + private final Map parameters = new HashMap<>(); + + private final KeystoreConfig keystoreConfig; + + private final JWK certJwk; + private final JWSSigner jwsSigner; + + CertificateTokenQuery(String url, String clientId, KeystoreConfig keystoreConfig, Map parameters) { + this.url = url; + this.clientId = clientId; + this.keystoreConfig = keystoreConfig; + this.parameters.putAll(parameters); + this.parameters.putIfAbsent("client_assertion_type", "urn:ietf:params:oauth:client-assertion-type:jwt-bearer"); + + KeyProvider keyProvider = createKeyProvider(); + try { + certJwk = JWK.parse(keyProvider.certificate()); + jwsSigner = createSigner(keyProvider.privateKey()); + } catch (Exception t) { + throw new RuntimeException("Failed to initialize key store.", t); + } + } + + @Override + public Map getParameters() { + this.parameters.put("client_assertion", buildAssertion()); + return parameters; + } + + private String buildAssertion() { + JWSHeader.Builder headerBuilder = new JWSHeader.Builder(JWSAlgorithm.PS256) + .type(new JOSEObjectType("JWT")) + .x509CertSHA256Thumbprint(certJwk.getX509CertSHA256Thumbprint()); + + Date curDate = new Date(); + Date expirationDate = new Date(curDate.getTime() + EXPIRATION_TIME_MS); + JWTClaimsSet.Builder claimsBuilder = new JWTClaimsSet.Builder() + .audience(url) + .expirationTime(expirationDate) + .issuer(clientId) + .jwtID(UUID.randomUUID().toString()) + .notBeforeTime(curDate) + .subject(clientId) + .issueTime(curDate); + + SignedJWT signedJWT = new SignedJWT( + headerBuilder.build(), + claimsBuilder.build() + ); + + try { + signedJWT.sign(jwsSigner); + return signedJWT.serialize(); + } catch (Exception t) { + throw new RuntimeException("Failed to create assertion.", t); + } + } + + private KeyProvider createKeyProvider() { + if (keystoreConfig.getType() == KeystoreType.PKCS12) { + return new KeystoreKeyProvider( + "PKCS12", keystoreConfig.getLocation(), + keystoreConfig.getAlias(), keystoreConfig.getPassword() + ); + } else if (keystoreConfig.getType() == KeystoreType.JKS) { + return new KeystoreKeyProvider( + "JKS", keystoreConfig.getLocation(), + keystoreConfig.getAlias(), keystoreConfig.getPassword() + ); + } else if (keystoreConfig.getType() == KeystoreType.PEM) { + return new PemKeyProvider( + keystoreConfig.getPemCertLocation(), keystoreConfig.getPemCert(), + keystoreConfig.getPemKeyLocation(), keystoreConfig.getPemKey() + ); + } else { + throw new RuntimeException("Keystore type is empty"); + } + } + + private JWSSigner createSigner(PrivateKey privateKey) { + try { + if (privateKey instanceof RSAPrivateKey) { + return new RSASSASigner(privateKey); + } else if (privateKey instanceof ECPrivateKey) { + return new ECDSASigner((ECPrivateKey) privateKey); + } + } catch (Exception t) { + throw new RuntimeException("Failed to create JWTSigner.", t); + } + + throw new RuntimeException("Failed to create JWTSigner. Unknown private key format. RSA or EC required."); + } + + private interface KeyProvider { + + PrivateKey privateKey(); + + X509Certificate certificate(); + + } + + private static class KeystoreKeyProvider implements KeyProvider { + + private final String alias; + private final KeyStore keyStore; + private final char[] keystorePassword; + + public KeystoreKeyProvider(String keystoreType, String keystorePath, String alias, String password) { + this.alias = alias; + this.keystorePassword = password.toCharArray(); + + try { + keyStore = KeyStore.getInstance(keystoreType); + try (InputStream is = Files.newInputStream(Paths.get(keystorePath))) { + keyStore.load(is, keystorePassword); + } + } catch (Exception t) { + throw new RuntimeException("Failed to load KeyStore: " + keystorePath, t); + } + } + + @Override + public PrivateKey privateKey() { + try { + return (PrivateKey) keyStore.getKey(alias, keystorePassword); + } catch (Exception e) { + throw new RuntimeException("Failed to get private key from keystore.", e); + } + } + + @Override + public X509Certificate certificate() { + Certificate certificate; + try { + certificate = keyStore.getCertificate(alias); + } catch (Exception e) { + throw new RuntimeException("Failed to get certificate from keystore.", e); + } + + if (certificate instanceof X509Certificate) { + return (X509Certificate) certificate; + } + throw new RuntimeException("Unknown certificate format, X.509 is required."); + } + } + + private static class PemKeyProvider implements KeyProvider { + + private final String pemCertLocation; + private final String pemCert; + private final String pemKeyLocation; + private final String pemKey; + + public PemKeyProvider(String pemCertLocation, String pemCert, String pemKeyLocation, String pemKey) { + this.pemCertLocation = pemCertLocation; + this.pemCert = pemCert; + this.pemKeyLocation = pemKeyLocation; + this.pemKey = pemKey; + } + + @Override + public PrivateKey privateKey() { + try { + String pemEncodedKey = pemKey; + if (pemEncodedKey == null) { + if (pemKeyLocation == null) { + throw new RuntimeException("pemKeyLocation is not specified."); + } + pemEncodedKey = IOUtils.readFileToString(new File(pemKeyLocation), StandardCharsets.UTF_8); + } + + JWK jwk = JWK.parseFromPEMEncodedObjects(pemEncodedKey); + if (jwk instanceof RSAKey) { + return jwk.toRSAKey().toKeyPair().getPrivate(); + } else if (jwk instanceof ECKey) { + return jwk.toECKey().toKeyPair().getPrivate(); + } else { + throw new RuntimeException("Invalid Key type. RSA or EC required."); + } + } catch (Exception e) { + throw new RuntimeException("Failed to get private key from keystore.", e); + } + } + + @Override + public X509Certificate certificate() { + try { + String pemEncodedCert = pemCert; + if (pemEncodedCert == null) { + if (pemCertLocation == null) { + throw new RuntimeException("pemCertLocation is not specified."); + } + pemEncodedCert = IOUtils.readFileToString(new File(pemCertLocation), StandardCharsets.UTF_8); + } + + return X509CertUtils.parse(pemEncodedCert); + } catch (Exception e) { + throw new RuntimeException("Failed to get certificate from keystore.", e); + } + } + } + +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/service/HttpConnectionRestClient.java b/util/src/main/java/com/epam/deltix/util/oauth/service/HttpConnectionRestClient.java new file mode 100644 index 00000000..4850e406 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/service/HttpConnectionRestClient.java @@ -0,0 +1,92 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth.service; + +import java.io.BufferedReader; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Map; + +class HttpConnectionRestClient implements RestClient { + + private final URL url; + private final int connectTimeoutMs; + private final int readTimeoutMs; + + public static RestClient create(String url, int connectTimeoutMs, int readTimeoutMs) { + try { + return new HttpConnectionRestClient(new URL(url), connectTimeoutMs, readTimeoutMs); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + private HttpConnectionRestClient(URL url, int connectTimeoutMs, int readTimeoutMs) { + this.url = url; + this.connectTimeoutMs = connectTimeoutMs; + this.readTimeoutMs = readTimeoutMs; + } + + @Override + public String postForm(TokenQuery query) throws IOException { + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setConnectTimeout(connectTimeoutMs); + connection.setReadTimeout(readTimeoutMs); + + connection.setRequestMethod("POST"); + connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); + connection.setRequestProperty("charset", "UTF-8"); + connection.setUseCaches(false); + connection.setDoOutput(true); + + try (DataOutputStream outputStream = new DataOutputStream(connection.getOutputStream())) { + boolean first = true; + Map parameters = query.getParameters(); + for (Map.Entry entry : parameters.entrySet()) { + if (first) { + first = false; + } else { + outputStream.writeBytes("&"); + } + + outputStream.writeBytes(entry.getKey()); + outputStream.writeBytes("="); + outputStream.writeBytes(entry.getValue()); + } + } + + if (connection.getResponseCode() != 200) { + throw new RuntimeException("Http request failed with code: " + connection.getResponseCode() + + "; message: " + connection.getResponseMessage()); + } + + StringBuilder response = new StringBuilder(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))) { + String line; + while ((line = reader.readLine()) != null) { + response.append(line); + } + } + + return response.toString(); + } + +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/service/Oauth2ClientImpl.java b/util/src/main/java/com/epam/deltix/util/oauth/service/Oauth2ClientImpl.java new file mode 100644 index 00000000..d5d7d876 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/service/Oauth2ClientImpl.java @@ -0,0 +1,259 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.oauth.service; + +import com.epam.deltix.gflog.api.Log; +import com.epam.deltix.gflog.api.LogFactory; +import com.epam.deltix.util.oauth.Oauth2Client; +import com.epam.deltix.util.oauth.Oauth2ClientConfig; +import com.epam.deltix.util.oauth.utils.IncreasingDelayRetryStrategy; +import com.epam.deltix.util.oauth.utils.RetryStrategy; +import com.epam.deltix.util.oauth.utils.TimerTokenScheduler; +import com.epam.deltix.util.time.TimeKeeper; +import com.epam.deltix.util.oauth.AuthResult; +import com.epam.deltix.util.oauth.RefreshTokenListener; +import com.epam.deltix.util.oauth.utils.RefreshTokenScheduler; + +import java.io.IOException; +import java.time.Instant; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +public class Oauth2ClientImpl implements Oauth2Client { + + public static final Log LOGGER = LogFactory.getLog(Oauth2ClientImpl.class.getName()); + + public static final String GRANT_TYPE_PARAM = "grant_type"; + public static final String CLIENT_ID_PARAM = "client_id"; + public static final String CLIENT_SECRET_PARAM = "client_secret"; + + public static final String CLIENT_CREDENTIALS_GRANT_TYPE = "client_credentials"; + + private final Oauth2ClientConfig config; + private final RestClient restClient; + private final TokenQuery tokenQuery; + + private final TokenResponseParser parser; + private final String clientId; + private final Map parameters = new HashMap<>(); + + private final RefreshTokenListener listener; + private final RefreshTokenScheduler refreshScheduler; + + private final RetryStrategy retryStrategy = new IncreasingDelayRetryStrategy(); + + private volatile AuthResult authResult; + private volatile long expirationTimestampMs; + + private volatile boolean closed; + + private final ReentrantLock lock = new ReentrantLock(); + + public static Oauth2ClientImpl of(Oauth2ClientConfig config) { + RestClient restClient = HttpConnectionRestClient.create( + config.getUrl(), config.getConnectTimeoutMs(), config.getReadTimeoutMs() + ); + return new Oauth2ClientImpl(config, restClient); + } + + private Oauth2ClientImpl(Oauth2ClientConfig config, RestClient restClient) { + this.config = config; + this.restClient = restClient; + this.tokenQuery = createTokenQuery(config); + this.parser = new TokenResponseParser(); + this.parameters.putAll(config.getParameters()); + this.clientId = parameters.get(CLIENT_ID_PARAM); + this.listener = config.getListener(); + this.refreshScheduler = config.getTimer() != null + ? new TimerTokenScheduler(config.getTimer()) : null; + + // initial token request + try { + requestToken(); + } catch (Exception t) { + LOGGER.error().append("Failed to request token").append(t).commit(); + } + } + + private static TokenQuery createTokenQuery(Oauth2ClientConfig config) { + if (CLIENT_CREDENTIALS_GRANT_TYPE.equals(config.getParameters().get(GRANT_TYPE_PARAM))) { + String clientId = config.getParameters().get(CLIENT_ID_PARAM); + if (clientId == null) { + throw new RuntimeException(CLIENT_ID_PARAM + " is not specified"); + } + + if (config.getParameters().get(CLIENT_SECRET_PARAM) == null) { + if (config.getKeystoreConfig() != null) { + return new CertificateTokenQuery(config.getUrl(), clientId, + config.getKeystoreConfig(), config.getParameters() + ); + } + + throw new RuntimeException("Invalid credentials: specify `" + CLIENT_SECRET_PARAM + "` parameter or keystore config."); + } + } + + return new ParametersTokenQuery(config.getParameters()); + } + + @Override + public AuthResult login() { + return getOrRequestToken(); + } + + public long expirationTimestampMs() { + return expirationTimestampMs; + } + + private AuthResult getOrRequestToken() { + if (authResult == null) { + requestTokenIfNeed(); + } else if (refreshScheduler == null) { + // configured without refresh task + // perform refresh token in current thread if needed + if (refreshRequired()) { + refreshTokenIfNeed(); + } + } + + return authResult; + } + + private void requestTokenIfNeed() { + boolean requestTimeout = !tryUnderLock(() -> { + // double check under lock + if (authResult == null) { + requestToken(); + } + }); + if (requestTimeout) { + throw new RuntimeException("Request token timeout"); + } + } + + private void refreshTokenIfNeed() { + tryUnderLock(() -> { + // double check under lock + if (refreshRequired()) { + requestToken(); + } + }); + } + + //returns false in case of timeout + private boolean tryUnderLock(Runnable logic) { + boolean locked; + try { + locked = lock.tryLock(config.getTimeoutMs(), TimeUnit.MILLISECONDS); + if (locked) { + try { + logic.run(); + } finally { + lock.unlock(); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + + return locked; + } + + private boolean refreshRequired() { + return currentTime() >= expirationTimestampMs; + } + + private long currentTime() { + return TimeKeeper.currentTime; + } + + private void requestToken() { + if (closed) { + return; + } + + long requestTime = currentTime(); + authResult = parser.parse(clientId, sendTokenRequest()); + if (authResult.expiresInSec() == 0) { + LOGGER.warn().append("Token updated (grant type: ").append(parameters.get(GRANT_TYPE_PARAM)) + .append("); expiration timestamp unknown, refresh task is not scheduled.").commit(); + } else { + // update expiration time + long expirationDelayMs = authResult.expiresInSec() * (long) (config.getExpirationMultiplier() * 1000.0d); + expirationTimestampMs = requestTime + expirationDelayMs; + + LOGGER.info().append("Token updated (grant type: ").append(parameters.get(GRANT_TYPE_PARAM)) + .append("); expiration timestamp: ") + .append(Instant.ofEpochMilli(expirationTimestampMs)).commit(); + + scheduleRefresh(expirationDelayMs); + } + + notifyRefreshed(); + } + + private String sendTokenRequest() { + try { + return restClient.postForm(tokenQuery); + } catch (IOException e) { + throw new RuntimeException("Failed to perform REST query", e); + } catch (Exception t) { + LOGGER.warn().append("Failed to request token").append(t).commit(); + throw t; + } + } + + private void notifyRefreshed() { + if (listener != null) { + listener.refreshed(authResult); + } + } + + private void scheduleRefresh(long delayMs) { + if (refreshScheduler != null && !closed) { + refreshScheduler.schedule(delayMs, this::refreshTokenTask); + LOGGER.info().append("Refresh token task scheduled in ").append(delayMs / 1000).append(" seconds.").commit(); + } + } + + private void refreshTokenTask() { + try { + lock.lock(); + try { + requestToken(); + retryStrategy.refreshRetryDelay(); + } finally { + lock.unlock(); + } + } catch (Exception t) { + LOGGER.warn().append("Failed to execute task").append(t).commit(); + scheduleRefresh(retryStrategy.nextRetryDelay()); + } + } + + @Override + public void close() { + closed = true; + if (refreshScheduler != null) { + refreshScheduler.close(); + } + } + +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/service/ParametersTokenQuery.java b/util/src/main/java/com/epam/deltix/util/oauth/service/ParametersTokenQuery.java new file mode 100644 index 00000000..bcb14beb --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/service/ParametersTokenQuery.java @@ -0,0 +1,34 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth.service; + +import java.util.HashMap; +import java.util.Map; + +class ParametersTokenQuery implements TokenQuery { + + private final Map parameters = new HashMap<>(); + + ParametersTokenQuery(Map parameters) { + this.parameters.putAll(parameters); + } + + @Override + public Map getParameters() { + return parameters; + } +} diff --git a/messages/src/main/java/com/epam/deltix/streaming/InstrumentMessageChannel.java b/util/src/main/java/com/epam/deltix/util/oauth/service/RestClient.java similarity index 79% rename from messages/src/main/java/com/epam/deltix/streaming/InstrumentMessageChannel.java rename to util/src/main/java/com/epam/deltix/util/oauth/service/RestClient.java index 2c7a2a35..73258566 100644 --- a/messages/src/main/java/com/epam/deltix/streaming/InstrumentMessageChannel.java +++ b/util/src/main/java/com/epam/deltix/util/oauth/service/RestClient.java @@ -1,22 +1,25 @@ -/* - * Copyright 2021 EPAM Systems, Inc - * - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. Licensed under the Apache License, - * Version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. */ -package com.epam.deltix.streaming; - -import com.epam.deltix.timebase.messages.InstrumentMessage; - -public interface InstrumentMessageChannel extends MessageChannel { -} \ No newline at end of file + package com.epam.deltix.util.oauth.service; + +import java.io.IOException; + +interface RestClient { + + String postForm(TokenQuery query) throws IOException; + +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/service/TokenQuery.java b/util/src/main/java/com/epam/deltix/util/oauth/service/TokenQuery.java new file mode 100644 index 00000000..b7e27001 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/service/TokenQuery.java @@ -0,0 +1,25 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth.service; + +import java.util.Map; + +interface TokenQuery { + + Map getParameters(); + +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/service/TokenResponseParser.java b/util/src/main/java/com/epam/deltix/util/oauth/service/TokenResponseParser.java new file mode 100644 index 00000000..40e0f37f --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/service/TokenResponseParser.java @@ -0,0 +1,159 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.oauth.service; + +import com.epam.deltix.util.lang.StringUtils; +import com.epam.deltix.util.oauth.AuthResult; +import io.github.green4j.jelly.JsonNumber; +import io.github.green4j.jelly.JsonParser; +import io.github.green4j.jelly.JsonParserListener; + +class TokenResponseParser { + + private final JsonParser parser = new JsonParser(); + private final Listener listener = new Listener(); + + public TokenResponseParser() { + parser.setListener(listener); + } + + synchronized AuthResult parse(String clientId, String response) { + parser.parseAndEoj(response); + if (listener.error != null) { + throw new RuntimeException("Failed to parse response: " + listener.error); + } + + return new AuthResult(clientId, listener.token(), listener.expiresInSec); + } + + private static class Listener implements JsonParserListener { + private final int STATE_UNKNOWN = -1; + private final int STATE_TOKEN = 0; + private final int STATE_TOKEN_EXPIRATION = 1; + + private int state = STATE_UNKNOWN; + + private String error; + + private String token; + + private long expiresInSec; + + public String token() { + return token; + } + + public long getExpiresInSec() { + return expiresInSec; + } + + @Override + public void onJsonStarted() { + this.state = STATE_UNKNOWN; + this.error = null; + this.token = null; + this.expiresInSec = 0; + } + + @Override + public void onError(String error, int position) { + this.error = error; + this.state = STATE_UNKNOWN; + } + + @Override + public void onJsonEnded() { + this.state = STATE_UNKNOWN; + } + + @Override + public boolean onObjectStarted() { + this.state = STATE_UNKNOWN; + return true; + } + + @Override + public boolean onObjectMember(CharSequence name) { + if (StringUtils.equals("access_token", name)) { + this.state = STATE_TOKEN; + } else if (StringUtils.equals("expires_in", name)) { + this.state = STATE_TOKEN_EXPIRATION; + } else { + this.state = STATE_UNKNOWN; + } + + return true; + } + + @Override + public boolean onObjectEnded() { + this.state = STATE_UNKNOWN; + return true; + } + + @Override + public boolean onArrayStarted() { + this.state = STATE_UNKNOWN; + return true; + } + + @Override + public boolean onArrayEnded() { + this.state = STATE_UNKNOWN; + return true; + } + + @Override + public boolean onStringValue(CharSequence data) { + if (state == STATE_TOKEN) { + token = data.toString(); + } + + this.state = STATE_UNKNOWN; + return true; + } + + @Override + public boolean onNumberValue(JsonNumber number) { + if (state == STATE_TOKEN_EXPIRATION) { + expiresInSec = number.mantissa(); + } + + this.state = STATE_UNKNOWN; + return true; + } + + @Override + public boolean onTrueValue() { + this.state = STATE_UNKNOWN; + return true; + } + + @Override + public boolean onFalseValue() { + this.state = STATE_UNKNOWN; + return true; + } + + @Override + public boolean onNullValue() { + this.state = STATE_UNKNOWN; + return true; + } + } +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/utils/ExecutorRefreshScheduler.java b/util/src/main/java/com/epam/deltix/util/oauth/utils/ExecutorRefreshScheduler.java new file mode 100644 index 00000000..a60651e6 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/utils/ExecutorRefreshScheduler.java @@ -0,0 +1,44 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth.utils; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +public class ExecutorRefreshScheduler implements RefreshTokenScheduler { + private final ScheduledExecutorService executor; + + private ScheduledFuture currentTask; + + public ExecutorRefreshScheduler(ScheduledExecutorService executor) { + this.executor = executor; + } + + @Override + public synchronized void schedule(long timestampMs, Runnable task) { + currentTask = executor.schedule(task, timestampMs, TimeUnit.MILLISECONDS); + } + + @Override + public synchronized void close() { + if (currentTask != null) { + currentTask.cancel(true); + currentTask = null; + } + } +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/utils/IncreasingDelayRetryStrategy.java b/util/src/main/java/com/epam/deltix/util/oauth/utils/IncreasingDelayRetryStrategy.java new file mode 100644 index 00000000..a888d553 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/utils/IncreasingDelayRetryStrategy.java @@ -0,0 +1,52 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth.utils; + +public class IncreasingDelayRetryStrategy implements RetryStrategy { + + private static final long DEFAULT_RETRY_DELAY_MS = 5 * 1000; + private static final long MAX_RETRY_DELAY_MS = 5 * 60 * 1000; // 5 min + + private long retryDelay = DEFAULT_RETRY_DELAY_MS; + + private int retriesMade; + + @Override + public synchronized void refreshRetryDelay() { + retryDelay = DEFAULT_RETRY_DELAY_MS; + retriesMade = 0; + } + + @Override + public synchronized long nextRetryDelay() { + ++retriesMade; + + long currentRetryDelay = retryDelay; + retryDelay *= 2; + if (retryDelay > MAX_RETRY_DELAY_MS) { + retryDelay = MAX_RETRY_DELAY_MS; + } + + return currentRetryDelay; + } + + @Override + public int retriesMade() { + return retriesMade; + } + +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/utils/RefreshTokenScheduler.java b/util/src/main/java/com/epam/deltix/util/oauth/utils/RefreshTokenScheduler.java new file mode 100644 index 00000000..a7d1ae66 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/utils/RefreshTokenScheduler.java @@ -0,0 +1,25 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth.utils; + +import com.epam.deltix.util.lang.Disposable; + +public interface RefreshTokenScheduler extends Disposable { + + void schedule(long delayMs, Runnable task); + +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/utils/RetryStrategy.java b/util/src/main/java/com/epam/deltix/util/oauth/utils/RetryStrategy.java new file mode 100644 index 00000000..dc21b31e --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/utils/RetryStrategy.java @@ -0,0 +1,27 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth.utils; + +public interface RetryStrategy { + + void refreshRetryDelay(); + + long nextRetryDelay(); + + int retriesMade(); + +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/utils/TimerTokenScheduler.java b/util/src/main/java/com/epam/deltix/util/oauth/utils/TimerTokenScheduler.java new file mode 100644 index 00000000..1e777776 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/utils/TimerTokenScheduler.java @@ -0,0 +1,43 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth.utils; + +import java.util.Timer; +import java.util.TimerTask; + +public class TimerTokenScheduler implements RefreshTokenScheduler { + + private final Timer timer; + + public TimerTokenScheduler(Timer timer) { + this.timer = timer; + } + + @Override + public void schedule(long timestampMs, Runnable task) { + timer.schedule(new TimerTask() { + @Override + public void run() { + task.run(); + } + }, timestampMs); + } + + @Override + public void close() { + } +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/utils/TokenUtils.java b/util/src/main/java/com/epam/deltix/util/oauth/utils/TokenUtils.java new file mode 100644 index 00000000..547b1d3f --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/utils/TokenUtils.java @@ -0,0 +1,41 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth.utils; + +import com.nimbusds.jwt.JWTParser; + +import java.text.ParseException; + +public class TokenUtils { + + public static String extractUserName(String token, String usernameClaim) { + Object userNameObj = extractClaim(token, usernameClaim); + if (userNameObj instanceof CharSequence) { + return ((CharSequence) userNameObj).toString(); + } + + throw new RuntimeException("Can not extract username from token."); + } + + public static Object extractClaim(String token, String claim) { + try { + return JWTParser.parse(token).getJWTClaimsSet().getClaim(claim); + } catch (ParseException e) { + throw new RuntimeException("Failed to decode token", e); + } + } +} diff --git a/util/src/main/java/com/epam/deltix/util/oauth/utils/Utils.java b/util/src/main/java/com/epam/deltix/util/oauth/utils/Utils.java new file mode 100644 index 00000000..72f638b0 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/oauth/utils/Utils.java @@ -0,0 +1,39 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.oauth.utils; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.stream.Collectors; + +public class Utils { + + public static String getResourceFileAsString(String fileName) throws IOException { + try (InputStream is = Utils.class.getResourceAsStream(fileName)) { + if (is == null) { + throw new RuntimeException("Can't find resource " + fileName); + } + try (InputStreamReader isr = new InputStreamReader(is); + BufferedReader reader = new BufferedReader(isr)) { + return reader.lines().collect(Collectors.joining(System.lineSeparator())); + } + } + } + +} diff --git a/util/src/main/java/com/epam/deltix/util/os/MemoryUtils.java b/util/src/main/java/com/epam/deltix/util/os/MemoryUtils.java index 07eec890..3c2a7564 100644 --- a/util/src/main/java/com/epam/deltix/util/os/MemoryUtils.java +++ b/util/src/main/java/com/epam/deltix/util/os/MemoryUtils.java @@ -14,6 +14,7 @@ * License for the specific language governing permissions and limitations under * the License. */ + package com.epam.deltix.util.os; import com.epam.deltix.gflog.api.Log; @@ -21,7 +22,9 @@ import com.epam.deltix.util.io.IOUtil; import com.epam.deltix.util.lang.Util; -import javax.management.*; +import javax.management.JMException; +import javax.management.MBeanServer; +import javax.management.ObjectName; import java.io.IOException; import java.lang.management.ManagementFactory; import java.util.regex.Matcher; @@ -140,7 +143,7 @@ private static void closeProcess(Process process) { Util.close(process.getInputStream()); Util.close(process.getOutputStream()); Util.close(process.getErrorStream()); - } catch (Throwable e) { + } catch (Exception e) { LOG.error("Close process exception: %s").with(e.getMessage()); } } @@ -153,4 +156,4 @@ public static void main(String[] args) throws Throwable { -} \ No newline at end of file +} diff --git a/util/src/main/java/com/epam/deltix/util/os/WindowsRegistry.java b/util/src/main/java/com/epam/deltix/util/os/WindowsRegistry.java new file mode 100644 index 00000000..f87d1696 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/os/WindowsRegistry.java @@ -0,0 +1,495 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.os; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.prefs.Preferences; + +@SuppressWarnings("unused") +public class WindowsRegistry { + // inspired by + // http://javabyexample.wisdomplug.com/java-concepts/34-core-java/62-java-registry-wrapper.html + // http://www.snipcode.org/java/1-java/23-java-class-for-accessing-reading-and-writing-from-windows-registry.html + // http://snipplr.com/view/6620/accessing-windows-registry-in-java/ + public static final int HKEY_CURRENT_USER = 0x80000001; + public static final int HKEY_LOCAL_MACHINE = 0x80000002; + public static final int REG_SUCCESS = 0; + public static final int REG_NOTFOUND = 2; + public static final int REG_ACCESSDENIED = 5; + + private static final int KEY_ALL_ACCESS = 0xf003f; + private static final int KEY_READ = 0x20019; + private static final Preferences userRoot = Preferences.userRoot ( ); + private static final Preferences systemRoot = Preferences.systemRoot ( ); + private static final Class userClass = userRoot.getClass ( ); + private static Method regOpenKey = null; + private static Method regCloseKey = null; + private static Method regQueryValueEx = null; + private static Method regEnumValue = null; + private static Method regQueryInfoKey = null; + private static Method regEnumKeyEx = null; + private static Method regCreateKeyEx = null; + private static Method regSetValueEx = null; + private static Method regDeleteKey = null; + private static Method regDeleteValue = null; + + static { + try { + regOpenKey = userClass.getDeclaredMethod ( "WindowsRegOpenKey", + int.class, + byte[].class, + int.class); + regOpenKey.setAccessible ( true ); + regCloseKey = userClass.getDeclaredMethod ( "WindowsRegCloseKey", + int.class); + regCloseKey.setAccessible ( true ); + regQueryValueEx = userClass.getDeclaredMethod ( "WindowsRegQueryValueEx", + int.class, + byte[].class); + regQueryValueEx.setAccessible ( true ); + regEnumValue = userClass.getDeclaredMethod ( "WindowsRegEnumValue", + int.class, + int.class, + int.class); + regEnumValue.setAccessible ( true ); + regQueryInfoKey = userClass.getDeclaredMethod ( "WindowsRegQueryInfoKey1", + int.class); + regQueryInfoKey.setAccessible ( true ); + regEnumKeyEx = userClass.getDeclaredMethod ( + "WindowsRegEnumKeyEx", + int.class, + int.class, + int.class); + regEnumKeyEx.setAccessible ( true ); + regCreateKeyEx = userClass.getDeclaredMethod ( + "WindowsRegCreateKeyEx", + int.class, + byte[].class); + regCreateKeyEx.setAccessible ( true ); + regSetValueEx = userClass.getDeclaredMethod ( + "WindowsRegSetValueEx", + int.class, + byte[].class, + byte[].class); + regSetValueEx.setAccessible ( true ); + regDeleteValue = userClass.getDeclaredMethod ( + "WindowsRegDeleteValue", + int.class, + byte[].class); + regDeleteValue.setAccessible ( true ); + regDeleteKey = userClass.getDeclaredMethod ( + "WindowsRegDeleteKey", + int.class, + byte[].class); + regDeleteKey.setAccessible ( true ); + } catch (final Exception e) { + e.printStackTrace ( ); + } + } + + private WindowsRegistry ( ) { + } + + /** + * Read a value from key and value name + */ + public static String readString ( final int hkey, + final String key, + final String valueName ) + throws IllegalArgumentException, + IllegalAccessException, + InvocationTargetException { + if (hkey == HKEY_LOCAL_MACHINE) { + return readString ( systemRoot, + hkey, + key, + valueName ); + } else if (hkey == HKEY_CURRENT_USER) { + return readString ( userRoot, + hkey, + key, + valueName ); + } else { + throw new IllegalArgumentException ( "hkey=" + hkey ); + } + } + + /** + * Read value(s) and value name(s) form given key + */ + public static Map readStringValues ( final int hkey, + final String key ) + throws IllegalArgumentException, + IllegalAccessException, + InvocationTargetException { + if (hkey == HKEY_LOCAL_MACHINE) { + return readStringValues ( systemRoot, + hkey, + key ); + } else if (hkey == HKEY_CURRENT_USER) { + return readStringValues ( userRoot, + hkey, + key ); + } else { + throw new IllegalArgumentException ( "hkey=" + hkey ); + } + } + + /** + * Read the value name(s) from a given key + */ + public static List readStringSubKeys ( final int hkey, + final String key ) + throws IllegalArgumentException, + IllegalAccessException, + InvocationTargetException { + if (hkey == HKEY_LOCAL_MACHINE) { + return readStringSubKeys ( systemRoot, + hkey, + key ); + } else if (hkey == HKEY_CURRENT_USER) { + return readStringSubKeys ( userRoot, + hkey, + key ); + } else { + throw new IllegalArgumentException ( "hkey=" + hkey ); + } + } + + /** + * Create a key + */ + public static void createKey ( final int hkey, + final String key ) + throws IllegalArgumentException, + IllegalAccessException, + InvocationTargetException { + int[] ret; + if (hkey == HKEY_LOCAL_MACHINE) { + ret = createKey ( systemRoot, + hkey, + key ); + regCloseKey.invoke ( systemRoot, ret[0]); + } else if (hkey == HKEY_CURRENT_USER) { + ret = createKey ( userRoot, + hkey, + key ); + regCloseKey.invoke ( userRoot, ret[0]); + } else { + throw new IllegalArgumentException ( "hkey=" + hkey ); + } + if (ret[1] != REG_SUCCESS) { + throw new IllegalArgumentException ( "rc=" + ret[1] + " key=" + key ); + } + } + + /** + * Write a value in a given key/value name + */ + public static void writeStringValue ( final int hkey, + final String key, + final String valueName, + final String value ) + throws IllegalArgumentException, + IllegalAccessException, + InvocationTargetException { + if (hkey == HKEY_LOCAL_MACHINE) { + writeStringValue ( systemRoot, + hkey, + key, + valueName, + value ); + } else if (hkey == HKEY_CURRENT_USER) { + writeStringValue ( userRoot, + hkey, + key, + valueName, + value ); + } else { + throw new IllegalArgumentException ( "hkey=" + hkey ); + } + } + + /** + * Delete a given key + */ + public static void deleteKey ( final int hkey, + final String key ) + throws IllegalArgumentException, + IllegalAccessException, + InvocationTargetException { + int rc = -1; + if (hkey == HKEY_LOCAL_MACHINE) { + rc = deleteKey ( systemRoot, + hkey, + key ); + } else if (hkey == HKEY_CURRENT_USER) { + rc = deleteKey ( userRoot, + hkey, + key ); + } + if (rc != REG_SUCCESS) { + throw new IllegalArgumentException ( "rc=" + rc + " key=" + key ); + } + } + + /** + * delete a value from a given key/value name + */ + public static void deleteValue ( final int hkey, + final String key, + final String value ) + throws IllegalArgumentException, + IllegalAccessException, + InvocationTargetException { + int rc = -1; + if (hkey == HKEY_LOCAL_MACHINE) { + rc = deleteValue ( systemRoot, + hkey, + key, + value ); + } else if (hkey == HKEY_CURRENT_USER) { + rc = deleteValue ( userRoot, + hkey, + key, + value ); + } + if (rc != REG_SUCCESS) { + throw new IllegalArgumentException ( "rc=" + rc + " key=" + key + " value=" + value ); + } + } + + // ===================== + + private static int deleteValue ( final Preferences root, + final int hkey, + final String key, + final String value ) + throws IllegalArgumentException, + IllegalAccessException, + InvocationTargetException { + final int[] handles = (int[]) regOpenKey.invoke ( root, + new Object[] + { + hkey, + toCstr ( key ), + KEY_ALL_ACCESS + } ); + if (handles[1] != REG_SUCCESS) { + return handles[1]; // can be REG_NOTFOUND, REG_ACCESSDENIED + } + final int rc = (Integer) regDeleteValue.invoke(root, + new Object[] + { + handles[0], + toCstr(value) + }); + regCloseKey.invoke ( root, + handles[0]); + return rc; + } + + private static int deleteKey ( final Preferences root, + final int hkey, + final String key ) + throws IllegalArgumentException, + IllegalAccessException, + InvocationTargetException { + return (int) (Integer) regDeleteKey.invoke(root, new Object[] + { + hkey, + toCstr(key) + }); // can REG_NOTFOUND, REG_ACCESSDENIED, REG_SUCCESS + } + + private static String readString ( final Preferences root, + final int hkey, + final String key, + final String value ) + throws IllegalArgumentException, + IllegalAccessException, + InvocationTargetException { + final int[] handles = (int[]) regOpenKey.invoke ( root, + new Object[] + { + hkey, + toCstr ( key ), + KEY_READ + } ); + if (handles[1] != REG_SUCCESS) { + return null; + } + final byte[] valb = (byte[]) regQueryValueEx.invoke ( root, + new Object[] + { + handles[0], + toCstr ( value ) + } ); + regCloseKey.invoke ( root, + handles[0]); + return (valb != null ? new String ( valb ).trim ( ) : null); + } + + private static Map readStringValues ( final Preferences root, + final int hkey, + final String key ) + throws IllegalArgumentException, + IllegalAccessException, + InvocationTargetException { + final HashMap results = new HashMap<>(); + final int[] handles = (int[]) regOpenKey.invoke ( root, + new Object[] + { + hkey, + toCstr ( key ), + KEY_READ + } ); + if (handles[1] != REG_SUCCESS) { + return null; + } + final int[] info = (int[]) regQueryInfoKey.invoke ( root, + new Object[] + { + handles[0] + } ); + + final int count = info[2]; // count + final int maxlen = info[3]; // value length max + for (int index = 0; index < count; index++) { + final byte[] name = (byte[]) regEnumValue.invoke ( root, + new Object[] + { + handles[0], + index, + maxlen + 1 + } ); + final String value = readString ( hkey, + key, + new String ( name ) ); + results.put ( new String ( name ).trim ( ), + value ); + } + regCloseKey.invoke ( root, + handles[0]); + return results; + } + + private static List readStringSubKeys ( final Preferences root, + final int hkey, + final String key ) + throws IllegalArgumentException, + IllegalAccessException, + InvocationTargetException { + final List results = new ArrayList<>(); + final int[] handles = (int[]) regOpenKey.invoke ( root, + new Object[] + { + hkey, + toCstr ( key ), + KEY_READ + } ); + if (handles[1] != REG_SUCCESS) { + return null; + } + final int[] info = (int[]) regQueryInfoKey.invoke ( root, + new Object[] + { + handles[0] + } ); + + final int count = info[2]; // count + final int maxlen = info[3]; // value length max + for (int index = 0; index < count; index++) { + final byte[] name = (byte[]) regEnumKeyEx.invoke ( root, + new Object[] + { + handles[0], + index, + maxlen + 1 + } ); + results.add ( new String ( name ).trim ( ) ); + } + regCloseKey.invoke ( root, + handles[0]); + return results; + } + + private static int[] createKey ( final Preferences root, + final int hkey, + final String key ) + throws IllegalArgumentException, + IllegalAccessException, + InvocationTargetException { + return (int[]) regCreateKeyEx.invoke ( root, + new Object[] + { + hkey, + toCstr ( key ) + } ); + } + + private static void writeStringValue ( final Preferences root, + final int hkey, + final String key, + final String valueName, + final String value ) + throws IllegalArgumentException, + IllegalAccessException, + InvocationTargetException { + final int[] handles = (int[]) regOpenKey.invoke ( root, + new Object[] + { + hkey, + toCstr ( key ), + KEY_ALL_ACCESS + } ); + + regSetValueEx.invoke ( root, + handles[0], + toCstr ( valueName ), + toCstr ( value )); + regCloseKey.invoke ( root, + handles[0]); + } + + // utility + private static byte[] toCstr ( final String str ) { + final byte[] result = new byte[str.length ( ) + 1]; + + for (int i = 0; i < str.length ( ); i++) { + result[i] = (byte) str.charAt ( i ); + } + result[str.length ( )] = 0; + return result; + } + + public static void main ( final String[] args) throws IllegalArgumentException, + IllegalAccessException, + InvocationTargetException { + System.out.println ( WindowsRegistry.readString ( HKEY_LOCAL_MACHINE, + "Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Shell Folders", + "Common Desktop" ) ); + System.out.println ( WindowsRegistry.readString ( HKEY_LOCAL_MACHINE, + "Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Shell Folders", + "Common Start Menu" ) ); + } +} diff --git a/util/src/main/java/com/epam/deltix/util/os/WindowsUtils.java b/util/src/main/java/com/epam/deltix/util/os/WindowsUtils.java index e1730940..418c59ec 100644 --- a/util/src/main/java/com/epam/deltix/util/os/WindowsUtils.java +++ b/util/src/main/java/com/epam/deltix/util/os/WindowsUtils.java @@ -14,6 +14,7 @@ * License for the specific language governing permissions and limitations under * the License. */ + package com.epam.deltix.util.os; import com.github.sarxos.winreg.HKey; @@ -90,7 +91,7 @@ public static String regQuery (final HKey hive, try { return registry.readString (hive, keyName, valueName); - } catch (final Throwable e) { + } catch (final Exception e) { return null; } } @@ -211,4 +212,4 @@ public static void main (final String[] args) { System.exit (0); } -} \ No newline at end of file +} diff --git a/util/src/main/java/com/epam/deltix/util/parsers/synthetic/SyntheticInstrumentParser.java b/util/src/main/java/com/epam/deltix/util/parsers/synthetic/SyntheticInstrumentParser.java new file mode 100644 index 00000000..d8cacd93 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/parsers/synthetic/SyntheticInstrumentParser.java @@ -0,0 +1,238 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.parsers.synthetic; + +import com.epam.deltix.dfp.Decimal64; +import com.epam.deltix.qsrv.hf.framework.mdp.impl.parser.generated.SyntheticRuleBaseVisitor; +import com.epam.deltix.qsrv.hf.framework.mdp.impl.parser.generated.SyntheticRuleLexer; +import com.epam.deltix.qsrv.hf.framework.mdp.impl.parser.generated.SyntheticRuleParser; +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.misc.ParseCancellationException; + +import java.util.LinkedHashSet; + +public class SyntheticInstrumentParser { + + private static class Leg { + private final String symbol; + private Decimal64 ratio; + + private Leg(String symbol, String ratio) { + this.symbol = symbol; + this.ratio = Decimal64.parse(ratio); + if (this.ratio.equals(Decimal64.ZERO)) { + throw new IllegalArgumentException("ratio cannot be zero"); + } + } + + private Leg negate() { + this.ratio = this.ratio.negate(); + return this; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Leg leg = (Leg) o; + + return symbol.equals(leg.symbol); + } + + @Override + public int hashCode() { + return symbol.hashCode(); + } + } + + private static class ThrowingErrorListener extends BaseErrorListener { + + @Override + public void syntaxError(Recognizer recognizer, + Object offendingSymbol, + int line, int charPositionInLine, + String msg, + RecognitionException e) + throws ParseCancellationException + { + throw new ParseCancellationException("line " + line + ":" + charPositionInLine + " " + msg); + } + } + + private static final ThrowingErrorListener ERROR_LISTENER = new ThrowingErrorListener(); + + private final SyntheticRuleLexer lexer = new SyntheticRuleLexer(null); + private final SyntheticRuleParser parser = new SyntheticRuleParser(null); + private final InputVisitor inputVisitor = new InputVisitor(); + + public SyntheticInstrumentParser() { + this(true); + } + + private SyntheticInstrumentParser(boolean throwExceptions) { + if (throwExceptions) { + lexer.removeErrorListeners(); + lexer.addErrorListener(ERROR_LISTENER); + parser.removeErrorListeners(); + parser.addErrorListener(ERROR_LISTENER); + } + } + + public static SyntheticInstrumentRule createParserAndParse(String textRule) { + return new SyntheticInstrumentParser().parse(textRule); + } + + public SyntheticInstrumentRule parse(String textRule) { + if (textRule == null || textRule.isEmpty()) + throw new IllegalArgumentException("Synthetic rule cannot be empty"); + + try { + lexer.setInputStream(new ANTLRInputStream(textRule)); + parser.setInputStream(new CommonTokenStream(lexer)); + + inputVisitor.init(); + return inputVisitor.visit(parser.input()); + } catch (ParseCancellationException e) { + throw new IllegalArgumentException(e); + } + } + + private static class InputVisitor extends SyntheticRuleBaseVisitor { + + private final SyntheticRuleVisitor syntheticRuleVisitor = new SyntheticRuleVisitor(); + + void init() { + syntheticRuleVisitor.clearLegs(); + } + + @Override + public SyntheticInstrumentRule visitInput(SyntheticRuleParser.InputContext ctx) { + LinkedHashSet legs = syntheticRuleVisitor.visit(ctx.syntheticRule()); + if (legs.size() < 2) + throw new IllegalArgumentException("Rule must specify at least two legs"); + + String[] symbols = new String[legs.size()]; + Decimal64[] ratios = new Decimal64[legs.size()]; + int i = 0; + for (final Leg leg : legs) { + symbols[i] = leg.symbol; + ratios[i] = leg.ratio; + ++i; + } + + return new SyntheticInstrumentRule(symbols, ratios); + } + } + + private static class SyntheticRuleVisitor extends SyntheticRuleBaseVisitor> { + + private final LinkedHashSet legs = new LinkedHashSet<>(); + private final LegVisitor legVisitor = new LegVisitor(); + + void clearLegs() { + legs.clear(); + } + + @Override + public LinkedHashSet visitPlusSyntheticRule(SyntheticRuleParser.PlusSyntheticRuleContext ctx) { + visit(ctx.syntheticRule()); + legs.add(checkDuplicate(legVisitor.visit(ctx.leg()))); + + return legs; + } + + @Override + public LinkedHashSet visitMinusSyntheticRule(SyntheticRuleParser.MinusSyntheticRuleContext ctx) { + visit(ctx.syntheticRule()); + legs.add(checkDuplicate(legVisitor.visit(ctx.leg()).negate())); + + return legs; + } + + @Override + public LinkedHashSet visitLegSyntheticRule(SyntheticRuleParser.LegSyntheticRuleContext ctx) { + legs.add(checkDuplicate(legVisitor.visit(ctx.leg()))); + + return legs; + } + + private Leg checkDuplicate(final Leg leg) { + if (legs.contains(leg)) + throw new IllegalArgumentException("Duplicate leg symbol: \"" + leg + "\""); + + return leg; + } + } + + private static class LegVisitor extends SyntheticRuleBaseVisitor { + + private final RatioVisitor ratioVisitor = new RatioVisitor(); + + @Override + public Leg visitLegLeft(SyntheticRuleParser.LegLeftContext ctx) { + return new Leg(unquote(ctx.symbol().getText()), ratioVisitor.visit(ctx.unaryRatio())); + } + + @Override + public Leg visitUnarySymbolMinus(SyntheticRuleParser.UnarySymbolMinusContext ctx) { + return visit(ctx.unarySymbol()).negate(); + } + + @Override + public Leg visitUnarySymbolPlus(SyntheticRuleParser.UnarySymbolPlusContext ctx) { + return visit(ctx.unarySymbol()); + } + + @Override + public Leg visitToSymbol(SyntheticRuleParser.ToSymbolContext ctx) { + return new Leg(unquote(ctx.symbol().getText()), "1.0"); + } + } + + private static class RatioVisitor extends SyntheticRuleBaseVisitor { + + @Override + public String visitUnaryRatioMinus(SyntheticRuleParser.UnaryRatioMinusContext ctx) { + return "-" + visit(ctx.unaryRatio()); + } + + @Override + public String visitToRatio(SyntheticRuleParser.ToRatioContext ctx) { + return visit(ctx.ratio()); + } + + @Override + public String visitRatio(SyntheticRuleParser.RatioContext ctx) { + return ctx.getText(); + } + + } + + private static String unquote(final String s) { + if (s.length() < 2) + return s; + + if (s.charAt(0) == '"' && s.charAt(s.length() - 1) == '"') + return s.substring(1, s.length() - 1); + + return s; + } + +} + diff --git a/util/src/main/java/com/epam/deltix/util/parsers/synthetic/SyntheticInstrumentRule.java b/util/src/main/java/com/epam/deltix/util/parsers/synthetic/SyntheticInstrumentRule.java new file mode 100644 index 00000000..f725c659 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/parsers/synthetic/SyntheticInstrumentRule.java @@ -0,0 +1,106 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.parsers.synthetic; + +import com.epam.deltix.dfp.Decimal64; +import com.epam.deltix.util.lang.Util; + +/** + * + * Spread rule can be defined as text using the following BNF grammar: + * 
+ *     <spread-rule> ::= [ <sign> ] <leg> ( <sign> <leg> )+
+ *     <leg> :== [ <ratio> '*' ]  SYMBOL
+ *     <ratio> :== ('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' )+
+ *     <sign> :== '+' | '-'
+ *
+ * SYMBOL can contain any characters except '-', '+', and '*'. Ratio number cannot be equal to zero. Ratio 1 can be skipped, except when SYMBOL starts with a digit. Each symbol may not appear more than once. + *

+ * Examples: + *

    + *
  • A+B (is identical to 1*A+1*B)
    • + *
  • -A+B (is identical to B-A)
    • + *
  • 1*A+2*B-3*C
    • + *
+ * + */ +public class SyntheticInstrumentRule { + + /** + * System-wide constant that defines maximum number of legs a Synthetic Instrument may have. Some OMS code pre-allocates arrays based on this number. + */ + public static final int MAX_NUMBER_OF_LEGS = Util.getIntSystemProperty("QuantServer.maxNumberOfSyntheticInstrumentLegs", 16, 2, 256); + + private static final char RATIO_TO_SYMBOL_SEPARATOR = '*'; + private static final char BUY_LEG_SEPARATOR = '+'; + private static final char SELL_LEG_SEPARATOR = '-'; + + public final String[] symbols; + public final double[] ratios; + public final Decimal64[] decimalRatios; + + public SyntheticInstrumentRule(String[] symbols, Decimal64[] ratios) { + assert symbols.length == ratios.length; + assert symbols.length > 1; + + this.symbols = symbols; + this.ratios = new double[ratios.length]; + for (int i = 0; i < ratios.length; i++) { + this.ratios[i] = ratios[i].toDouble(); + } + this.decimalRatios = ratios; + } + + public SyntheticInstrumentRule(int numberOfLegs) { + if (numberOfLegs < 2) + throw new IllegalArgumentException("Rule must specify at least two legs"); + this.symbols = new String[numberOfLegs]; + this.ratios = new double[numberOfLegs]; + this.decimalRatios = new Decimal64[numberOfLegs]; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < symbols.length; i++) { + double ratio = ratios[i]; + + String symbol = symbols[i]; + if (ratio < 0) { + sb.append(SELL_LEG_SEPARATOR); + ratio = -ratio; + } else { + if (i > 0) + sb.append(BUY_LEG_SEPARATOR); + } + + if (ratio != 1 || Character.isDigit(symbol.charAt(0))) { + if ((ratio - (int)ratio) != 0) + sb.append(ratio); + else + sb.append((int)ratio); + sb.append(RATIO_TO_SYMBOL_SEPARATOR); + } + sb.append(symbol); + } + + return sb.toString(); + } + +} + diff --git a/util/src/main/java/com/epam/deltix/util/text/CharSequenceParser.java b/util/src/main/java/com/epam/deltix/util/text/CharSequenceParser.java index 8068a4dc..8e64e433 100644 --- a/util/src/main/java/com/epam/deltix/util/text/CharSequenceParser.java +++ b/util/src/main/java/com/epam/deltix/util/text/CharSequenceParser.java @@ -14,6 +14,7 @@ * License for the specific language governing permissions and limitations under * the License. */ + package com.epam.deltix.util.text; import com.epam.deltix.util.lang.Util; @@ -38,7 +39,17 @@ public abstract class CharSequenceParser { private static final int FLOAT_MANTISSA_BITMASK = FLOAT_ASSUMED_BIT - 1; private static final int FLOAT_NORM_EXP = FLOAT_BIAS_EXP + FLOAT_MANTISSA_WIDTH; private static final int FLOAT_OVERFLOW_BITMASK = ~FLOAT_MANTISSA_BITMASK - FLOAT_ASSUMED_BIT; - + + private static final double[] SMALL_POWERS_OF_10 = { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, + 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, + 1e20, 1e21, 1e22 + }; + + private static final double[] BIG_POWERS_OF_10 = { + 1e16, 1e32, 1e64, 1e128, 1e256 + }; + public static boolean parseBoolean ( CharSequence sc ) { if ("true".contentEquals ( sc )) { return true; @@ -223,40 +234,43 @@ public static double parseDouble (final CharSequence sc, final int startIncl, f dotSeen = true; else if (ch == 'e' || ch == 'E') { pos++; - checkNotAtEnd (pos, endExcl, sc, startIncl); - ch = sc.charAt (pos); - boolean negativeExp = false; - if (ch == '-') { pos++; - negativeExp = true; - } - else if (ch == '+') + } else if (ch == '+') { pos++; - + } + checkNotAtEnd (pos, endExcl, sc, startIncl); - int exp = parseInt (sc, pos, endExcl); - - for (int ii = 0; ii < exp; ii++) - if (negativeExp) - denominator *= 10; - else - denominator /= 10; + + double powerOf10 = pow10(exp); + if (negativeExp) { + denominator *= powerOf10; + } else { + denominator /= powerOf10; + } + break; } else { final int digit = ch - '0'; - if (digit < 0 || digit > 9) { - if (Util.equals (sc, "NaN")) - return (Double.NaN); - + if (pos == startIncl || (pos == startIncl + 1 && (sc.charAt(startIncl) == '+' || sc.charAt(startIncl) == '-'))) { + if (matchesAt(sc, pos, endExcl, "Infinity")) { + return sign == 0 ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY; + } + else if (matchesAt(sc, pos, endExcl, "Inf")) { + return sign == 0 ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY; + } + else if (matchesAt(sc, pos, endExcl, "NaN")) { + return Double.NaN; + } + } throw new NumberFormatException ( - "Illegal digit at position " + (pos + 1) + " in: " + sc.subSequence (startIncl, endExcl).toString ()); + "Illegal digit at position " + (pos + 1) + " in: " + sc.subSequence (startIncl, endExcl)); } if (overflow) { @@ -283,9 +297,9 @@ else if (ch == '+') ch = sc.charAt (pos); } - if (numerator == 0) - return (0.0); - + if (numerator == 0){ + return 0.0; + } // Build the double first, ignoring the denominator long exp = DOUBLE_NORM_EXP; @@ -304,13 +318,24 @@ else if (ch == '+') final long bits = sign | (exp << DOUBLE_MANTISSA_WIDTH) | numerator; double result = Double.longBitsToDouble (bits); - + if (denominator != 1) result /= denominator; - + return (result); } + private static boolean matchesAt(CharSequence sc, int start, int end, String target) { + if (end - start != target.length()) + return false; + + for (int i = 0; i < target.length(); i++) { + if (sc.charAt(start + i) != target.charAt(i)) + return false; + } + return true; + } + private static void checkNotAtEnd (int pos, final int endExcl, final CharSequence sc, final int startIncl) throws NumberFormatException { @@ -334,7 +359,7 @@ public static float parseFloat (final CharSequence sc, final int startIncl, int pos = startIncl; int numerator = 0; - float denominator = 1; + double denominator = 1; int sign = 0; boolean dotSeen = false; boolean overflow = false; @@ -355,12 +380,46 @@ public static float parseFloat (final CharSequence sc, final int startIncl, if (ch != ',') { if (!dotSeen && ch == '.') dotSeen = true; + else if (ch == 'e' || ch == 'E') { + pos++; + checkNotAtEnd (pos, endExcl, sc, startIncl); + ch = sc.charAt (pos); + boolean negativeExp = false; + + if (ch == '-') { + pos++; + negativeExp = true; + } + else if (ch == '+') + pos++; + + checkNotAtEnd (pos, endExcl, sc, startIncl); + + int exp = parseInt (sc, pos, endExcl); + + double powerOf10 = pow10(exp); + if (negativeExp) { + denominator *= powerOf10; + } else { + denominator /= powerOf10; + } + break; + } else { final int digit = ch - '0'; if (digit < 0 || digit > 9) { - if (Util.equals (sc, "NaN")) - return (Float.NaN); + if (pos == startIncl || (pos == startIncl + 1 && (sc.charAt(startIncl) == '+' || sc.charAt(startIncl) == '-'))) { + if (matchesAt(sc, pos, endExcl, "Infinity")) { + return sign == 0 ? Float.POSITIVE_INFINITY : Float.NEGATIVE_INFINITY; + } + else if (matchesAt(sc, pos, endExcl, "Inf")) { + return sign == 0 ? Float.POSITIVE_INFINITY : Float.NEGATIVE_INFINITY; + } + else if (matchesAt(sc, pos, endExcl, "NaN")) { + return Float.NaN; + } + } throw new NumberFormatException ( "Illegal digit at position " + (pos + 1) + " in: " + @@ -393,9 +452,10 @@ public static float parseFloat (final CharSequence sc, final int startIncl, ch = sc.charAt (pos); } - if (numerator == 0) - return (0.0F); - + if (numerator == 0) { + return (0.0F); + } + // Build the double first, ignoring the denominator int exp = FLOAT_NORM_EXP; @@ -421,7 +481,25 @@ public static float parseFloat (final CharSequence sc, final int startIncl, return (result); } + private static double pow10(int exp) { + if (exp < 0) { + return 1.0 / pow10(-exp); + } + if (exp < SMALL_POWERS_OF_10.length) { + return SMALL_POWERS_OF_10[exp]; + } + + double result = 1.0; + int bitMask = 1; + for (int i = 0; i < BIG_POWERS_OF_10.length; i++) { + if ((exp & (bitMask << (i + 4))) != 0) { + result *= BIG_POWERS_OF_10[i]; + } + } + result *= SMALL_POWERS_OF_10[exp & 0xF]; + return result; + } public static void main (String [] args) { System.out.println (parseDouble (args [0])); } -} \ No newline at end of file +} diff --git a/util/src/main/java/com/epam/deltix/util/time/DefaultTimeSourceProvider.java b/util/src/main/java/com/epam/deltix/util/time/DefaultTimeSourceProvider.java new file mode 100644 index 00000000..22ec5b7b --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/time/DefaultTimeSourceProvider.java @@ -0,0 +1,129 @@ +///* +// * Copyright 2021 EPAM Systems, Inc +// * +// * See the NOTICE file distributed with this work for additional information +// * regarding copyright ownership. Licensed under the Apache License, +// * Version 2.0 (the "License"); you may not use this file except in compliance +// * with the License. You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// * License for the specific language governing permissions and limitations under +// * the License. +// */ +// +//package com.epam.deltix.util.time; +// +//import com.epam.deltix.gflog.api.Log; +//import com.epam.deltix.gflog.api.LogFactory; +//import com.epam.deltix.qsrv.hf.pub.TimeSource; +//import org.jetbrains.annotations.Nullable; +//import org.jetbrains.annotations.VisibleForTesting; +// +///** +// * Time source provided for applications that do not have own source of clock configuration. +// */ +//public class DefaultTimeSourceProvider { +// private static final Log LOG = LogFactory.getLog(DefaultTimeSourceProvider.class); +// +// /** +// * System property that can be used to configure time source. Supported values are defined in {@link #getTimeSourceByName(String)}. +// */ +// public static final String TIME_SOURCE_SYS_PROP = "deltix.util.time.DefaultTimeSourceProvider.clock"; +// +// private static volatile TimeSource configuredInstance; +// +// private DefaultTimeSourceProvider() { +// } +// +// /** +// * Provides default time source. If time source is not configured yet, then it will be configured during this call using system property. +// * +// * @param appName name of application or use case that will use this time source. Will be logged if triggers configuration. +// */ +// public static TimeSource getTimeSourceForApp(String appName) { +// if (configuredInstance == null) { +// synchronized (DefaultTimeSourceProvider.class) { +// if (configuredInstance == null) { +// // ... +// String sysProperty = System.getProperty(TIME_SOURCE_SYS_PROP); +// if (sysProperty == null) { +// configuredInstance = getDefaultFallback(); +// LOG.info("Selected time source: %s (default, implicitly configured by %s)") +// .with(configuredInstance.getClass().getSimpleName()) +// .with(appName); +// } else { +// configuredInstance = getTimeSourceByNameWithFallback(sysProperty); +// LOG.info("Selected time source: %s (implicitly configured by %s)") +// .with(configuredInstance.getClass().getSimpleName()) +// .with(appName); +// } +// } +// } +// } +// return configuredInstance; +// } +// +// /** +// * Sets time source to be used by default. +// * +// *

Will throw exception if time source is already configured (directly or by call to {@link #getTimeSourceForApp(String)}). +// * +// * @param appName name of application that configures this time source. Will be logged. +// * @param timeSource time source to use +// */ +// public static void configure(String appName, TimeSource timeSource) { +// synchronized (DefaultTimeSourceProvider.class) { +// if (configuredInstance != null) { +// throw new IllegalStateException("Time source is already configured"); +// } +// configuredInstance = timeSource; +// LOG.info("Selected time source: %s (explicitly configured by %s)") +// .with(configuredInstance.getClass().getSimpleName()) +// .with(appName); +// } +// } +// +// private static TimeSource getDefaultFallback() { +// return KeeperTimeSource.getInstance(); +// } +// +// /** +// * Converts time source name to time source instance. +// * +// * @return time source instance or null if name is unknown +// */ +// @Nullable +// public static TimeSource getTimeSourceByName(String sourceName) { +// switch (sourceName) { +// case "MonotonicRealTimeSource": +// return MonotonicRealTimeSource.getInstance(); +// case "KeeperTimeSource": +// return KeeperTimeSource.getInstance(); +// default: +// return null; +// } +// } +// +// private static TimeSource getTimeSourceByNameWithFallback(String sysProperty) { +// TimeSource timeSourceByName = getTimeSourceByName(sysProperty); +// if (timeSourceByName == null) { +// LOG.warn().append("Unknown time source name: ").append(sysProperty).append(" (will use default)").commit(); +// timeSourceByName = getDefaultFallback(); +// } +// return timeSourceByName; +// } +// +// /** +// * ONLY FOR TESTING! +// */ +// @VisibleForTesting +// static void unconfigure() { +// synchronized (DefaultTimeSourceProvider.class) { +// configuredInstance = null; +// } +// } +//} diff --git a/util/src/main/java/com/epam/deltix/util/time/MonotonicRealTimeSource.java b/util/src/main/java/com/epam/deltix/util/time/MonotonicRealTimeSource.java new file mode 100644 index 00000000..4cdfe65f --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/time/MonotonicRealTimeSource.java @@ -0,0 +1,83 @@ +///* +// * Copyright 2021 EPAM Systems, Inc +// * +// * See the NOTICE file distributed with this work for additional information +// * regarding copyright ownership. Licensed under the Apache License, +// * Version 2.0 (the "License"); you may not use this file except in compliance +// * with the License. You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// * License for the specific language governing permissions and limitations under +// * the License. +// */ +// +//package com.epam.deltix.util.time; +// +//import com.epam.deltix.clock.Clocks; +//import com.epam.deltix.qsrv.hf.pub.TimeSource; +//import com.epam.deltix.util.annotations.TimestampMs; +//import com.epam.deltix.util.annotations.TimestampNs; +//import net.jcip.annotations.ThreadSafe; +// +//import java.util.concurrent.atomic.AtomicLong; +// +///** +// * Shared non-decreasing realtime time source with up to nanosecond resolution (if available). +// * +// *

Time source that: +// *

    +// *
  • Uses {@link deltix.clock.Clocks#REALTIME} as base time source
    • +// *
  • Guarantied to return monotonously non-decreasing values
    • +// *
  • Guaranties consistent time across multiple threads (instance users)
    • +// *
  • Provided "millis" version is in sync with "nanos" value (rounded down) but is not very efficient
    • +// *
+// * +// *

WARNING: This implementation is monotonic in the sense of non-decreasing returned values. +// * However, it is not monotonic in the same sense as Linux CLOCK_MONOTONIC (or {@link Clocks#MONOTONIC}) +// * that assumes linear growth with time flow. +// */ +//@ThreadSafe +//public class MonotonicRealTimeSource implements TimeSource { +// +// // Shared value +// private static final AtomicLong lastTimeNs = new AtomicLong(Long.MIN_VALUE); +// +// private static final MonotonicRealTimeSource INSTANCE = new MonotonicRealTimeSource(); +// +// private MonotonicRealTimeSource() { +// } +// +// public static MonotonicRealTimeSource getInstance() { +// return INSTANCE; +// } +// +// @Override +// @TimestampMs +// public long currentTimeMillis() { +// // TODO: Consider using System.currentTimeMillis() directly Clocks.REALTIME is not available +// // to avoid extra multiplication and division steps +// return currentTimeNanos() / 1_000_000L; +// } +// +// @SuppressWarnings("DuplicatedCode") +// @Override +// @TimestampNs +// public long currentTimeNanos() { +// long currentTimeNanos = Clocks.REALTIME.time(); +// while (true) { +// long prevVal = lastTimeNs.get(); +// if (prevVal >= currentTimeNanos) { +// // Shared value is already ahead (or same). So we can use it and do not need to update shared value. +// return prevVal; +// } +// // currentTimeNanos > prevVal +// if (lastTimeNs.compareAndSet(prevVal, currentTimeNanos)) { +// return currentTimeNanos; +// } +// } +// } +//} todo: Extract Clock from TB diff --git a/util/src/main/java/com/epam/deltix/util/time/TimerRunner.java b/util/src/main/java/com/epam/deltix/util/time/TimerRunner.java index 19a07a40..0633b680 100644 --- a/util/src/main/java/com/epam/deltix/util/time/TimerRunner.java +++ b/util/src/main/java/com/epam/deltix/util/time/TimerRunner.java @@ -14,8 +14,10 @@ * License for the specific language governing permissions and limitations under * the License. */ + package com.epam.deltix.util.time; +import com.epam.deltix.util.LangUtil; import com.epam.deltix.util.lang.Util; /** @@ -31,10 +33,12 @@ public final void run () { runInternal(); } catch (Throwable e) { + // We catch Throwable to keep existing API behavior for possible onError() overrides. try { onError(e); } catch (Throwable ex) { Util.handleException(ex); + LangUtil.propagateError(ex); } } } @@ -46,10 +50,11 @@ public final void run () { */ protected void onError (Throwable e) { Util.handleException (e); + LangUtil.propagateError(e); } /** * Override this method to perform timer task, instead of overriding run (). */ protected abstract void runInternal () throws Exception; -} \ No newline at end of file +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/ChannelClosedException.java b/util/src/main/java/com/epam/deltix/util/vsocket/ChannelClosedException.java new file mode 100644 index 00000000..3250e643 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/ChannelClosedException.java @@ -0,0 +1,28 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import java.io.IOException; + +/** + * Thrown from read or write methods when channel is explicitly + * closed by the other side (by calling close ()). + */ +public class ChannelClosedException extends IOException { + public ChannelClosedException () { + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/ChannelExecutor.java b/util/src/main/java/com/epam/deltix/util/vsocket/ChannelExecutor.java new file mode 100644 index 00000000..1a50027d --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/ChannelExecutor.java @@ -0,0 +1,310 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.vsocket; + +import com.epam.deltix.thread.affinity.AffinityConfig; +import com.epam.deltix.thread.affinity.AffinityThreadFactoryBuilder; +import com.epam.deltix.util.collections.QuickList; +import com.epam.deltix.util.lang.Util; +import com.epam.deltix.util.memory.MemoryDataOutput; +import com.epam.deltix.util.time.TimeKeeper; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.VisibleForTesting; + +import java.io.IOException; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.locks.LockSupport; +import java.util.logging.Level; + +class ChannelExecutor implements Runnable { + @ApiStatus.Experimental // Temporary option for testing performance effect of using yield on Windows + private static final boolean USE_YIELD_ON_WINDOWS = Boolean.getBoolean("TimeBase.network.executor.windows.useYield"); + + private static volatile ChannelExecutor INSTANCE; + + private static ChannelExecutor createInstance(AffinityConfig affinityConfig) { + ChannelExecutor executor = create(affinityConfig); + executor.thread.start(); + return executor; + } + + @SuppressWarnings("SameParameterValue") + @VisibleForTesting + static ChannelExecutor createNonSharedTestInstance(AffinityConfig affinityConfig) { + return createInstance(affinityConfig); + } + + public static ChannelExecutor getInstance(AffinityConfig affinityConfig) { + // "Double checked lock" (via volatile) + if (INSTANCE == null) { + synchronized (ChannelExecutor.class) { + if (INSTANCE == null) { + INSTANCE = createInstance(affinityConfig); + } + } + } + return INSTANCE; + } + + private final QuickList channels = new QuickList<>(); + private boolean stopped = false; + private final CPUEater cpuEater; // Used only for Windows + private final int idleTime; + private final Thread thread; + + static ChannelExecutor create(AffinityConfig affinityConfig) { + ThreadFactory factory = new AffinityThreadFactoryBuilder(affinityConfig) + .setNameFormat("ChannelExecutor Thread") + .setDaemon(true) + .build(); + + return new ChannelExecutor(factory); + } + + private ChannelExecutor(ThreadFactory factory) { + idleTime = VSProtocol.getIdleTime(); + cpuEater = Util.IS_WINDOWS_OS ? new CPUEater(idleTime) : null; + + this.thread = factory.newThread(this); + } + + public void wakeup() { + LockSupport.unpark(this.thread); + } + + public void shutdown () { + stopped = true; + wakeup (); + } + + public void addChannel(VSChannel channel) { + assert channel != null; + + if (channel == null) + return; + + synchronized (channels) { + channels.linkLast(new Entry(channel)); + } + + wakeup(); + } + + public void removeChannel(VSChannel channel) { + assert channel != null; + + + synchronized (channels) { + Entry entry = channels.getFirst(); + + while (entry != null) { + if (entry.channel.equals(channel)) { + remove(entry); + return; + } else { + entry = entry.next(); + } + } + } + + wakeup(); + } + + @Override + public void run() { + assert Thread.currentThread() == this.thread; + + while (!stopped) { + Entry entry; + boolean isEmpty; + + long bytesSent = 0; + synchronized (channels) { + entry = channels.getFirst(); + isEmpty = entry == null; + + while (entry != null) { + VSChannel channel = entry.channel; + try { + switch (channel.getState()) { + case Connected: { + if (channel.getNoDelay()) { + // Flush + VSOutputStream out = channel.getOutputStream(); + // We do not want to send all at once, we will, re-try send shortly + bytesSent += out.flushAvailable(false); + } + break; + } + case Removed: + case Closed: { + entry = remove(entry); + continue; + } + } + } catch (ChannelClosedException e) { + // ignore + entry = remove(entry); + continue; + } catch (IOException e) { + VSProtocol.LOGGER.log (Level.WARNING, "Exception while flushing data", e); + } + + // Move to the next channel + entry = entry.next(); + } + } + + if (isEmpty) { + // No channels to process => Wait for channels to be added. + LockSupport.park(); + + if (Thread.interrupted ()) { + if (stopped) { + break; + } + } + } else { + // Do not wait if we have sent any data. + // It's very likely that it's time to send more because the sending time is relatively long. + if (bytesSent == 0) { + // Wait till next time to flush channels + idleWait(); + } + } + } + } + + private void idleWait() { + if (!Util.IS_WINDOWS_OS) { + if (USE_YIELD_ON_WINDOWS) { + waitWithYield(idleTime); + } else { + LockSupport.parkNanos(idleTime); + } + } else { + if (TimeKeeper.getMode() == TimeKeeper.Mode.HIGH_RESOLUTION_SYNC_BACK) { + TimeKeeper.parkNanos(idleTime); + } else { + cpuEater.run(); + } + } + } + + private static void waitWithYield(int idleTime) { + long start = System.nanoTime(); + long end = start + idleTime; + while (System.nanoTime() < end) { + Thread.yield(); + } + } + + private Entry remove(Entry entry) { + Entry next = entry.next(); + entry.unlink(); + return next; + } + + private static class Entry extends QuickList.Entry { + final VSChannel channel; + + private Entry(VSChannel channel) { + this.channel = channel; + } + } + + private static class CPUEater { + private final long avgCostOfNanoTimeCall; + private final long cycles; + + private final MemoryDataOutput out = new MemoryDataOutput(); + private static final double value = 345.56787899; + + private CPUEater(long nanos) { + this.avgCostOfNanoTimeCall = nanoTimeCost(); + + if (nanos <= avgCostOfNanoTimeCall) + throw new IllegalArgumentException("Input time is too small: " + nanos); + + // warmup + for (int j = 0; j < 1000; j++) + execute(100); + + long time10 = measureExecution(10); + long time50 = measureExecution(50); + double c = time50 / time10 / 5.0; + + long low = (nanos / time10 * 10); + long high = (long) (low / c); + long count = low + (high - low) / 2; + long increment = Math.abs((high - low) / 4); + + if (increment == 0) + increment = 100; + + long time = measureExecution(count); + while (time < nanos) { + count += increment; + time = measureExecution(count); + } + cycles = low; + } + + private static long nanoTimeCost() { + final int N = 30000; + long enterTime = System.nanoTime(); + for (int i = 0; i < N; i++) { + System.nanoTime(); + } + long exitTime = System.nanoTime(); + return (exitTime - enterTime) / (N + 2); + } + + private void execute(long cycles) { + for (int i = 0; i < cycles; i++) { + out.reset(); + out.writeScaledDouble(value); + } + } + +// // non-deterministic execution time on high cpu load +// private void execute(long cycles) { +// for (int i = 0; i < cycles; i++) { +// try { +// Thread.sleep(0); +// } catch (InterruptedException e) { +// // ignore +// } +// } +// } + + public void run() { + execute(cycles); + } + + private long measureExecution(long cycles) { + long enterTime = System.nanoTime(); + for (int j = 0; j < 20000; j++) + execute(cycles); + long exitTime = System.nanoTime(); + long time = avgCostOfNanoTimeCall + (exitTime - enterTime) / 20000; + //System.out.println("Time of execution(" + cycles + "): " + time); + return time; + } + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/ChannelOutputStream.java b/util/src/main/java/com/epam/deltix/util/vsocket/ChannelOutputStream.java new file mode 100644 index 00000000..715b3bee --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/ChannelOutputStream.java @@ -0,0 +1,374 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.concurrent.UncheckedInterruptedException; +import com.epam.deltix.util.lang.Util; +import net.jcip.annotations.GuardedBy; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Date: Mar 25, 2010 + */ +public class ChannelOutputStream extends VSOutputStream { + @ApiStatus.Experimental // Temporary option for testing performance effect of flushing single packet + private static final boolean SINGLE_SEND_ON_PARTIAL_FLUSH = Boolean.getBoolean("TimeBase.network.channel.singleSendOnPartialFlush"); + + private final int maxCapacity; + private final VSChannelImpl channel; + @GuardedBy("this") + private boolean closed = false; + @GuardedBy("this") + private byte [] buffer; + @GuardedBy("this") + private boolean flushDisabled = false; + + // Total number of accumulated bytes in the buffer. + @GuardedBy("this") + private int size = 0; + + // Number of bytes available to be sent. + // Always: available <= size. + // Can be less than size if flush is disabled. + @GuardedBy("this") + private int available = 0; + + private final AtomicInteger waiting = new AtomicInteger(0); + // TODO: Consider making it volatile instead. All writes are under synchronization + private final AtomicInteger remoteCapacityAvailable = new AtomicInteger(-1); + private final AtomicInteger capacityIncrement = new AtomicInteger(0); + + ChannelOutputStream(VSChannelImpl channel, int bufferSize) { + this.channel = channel; + this.maxCapacity = bufferSize; + this.buffer = new byte [bufferSize]; + } + + @Override + public synchronized void close() throws IOException { + if (!closed) { + flush(); + closed = true; + notifyAll(); + } + } + + @Override + public synchronized void enableFlushing() throws IOException { + flushDisabled = false; + available = size; + + // Previously this check looked like this: "size >= maxCapacity" + // However this is ineffective: the remote capacity is "maxCapacity" at most, + // So attempt to flush more than that almost certainly results in situation + // when we will block on that flush and need to wait for BYTES_AVAILABLE_REPORT from the remote side. + // At the same time we do not want to flush too often (it's costly), + // so we flush only when we have at least half of the buffer filled. + int halfCapacity = maxCapacity >> 1; + if (size >= halfCapacity) { + // If we below of 75% capacity, we can flush buffer partially. + // However, if we are above 75% capacity, we should flush all data + // and block till all accumulated data is sent. + // Otherwise, if the consumer too slow, the buffer will start to grow indefinitely. + // See https://gitlab.deltixhub.com/Deltix/QuantServer/QuantServer/-/issues/1298 + int buffer75percent = halfCapacity + (halfCapacity >> 1); + boolean partialOk = size < buffer75percent; + try { + flushInternal(partialOk, false); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UncheckedInterruptedException(e); + } + } + } + + @Override + public synchronized void disableFlushing() { + flushDisabled = true; + available = size; + } + + synchronized void closeNoFlush() { + if (!closed) { + closed = true; + notifyAll(); + } + } + + @Override + public synchronized void flush() throws IOException { + try { + flushInternal (false, false); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UncheckedInterruptedException(e); + } + } + + @Override + public synchronized int flushAvailable(boolean flushAll) throws IOException { + try { + // we do not want to wait() here + + // TODO: Consider adding "capacityIncrement.get()" here + if (available > 0 && getRemoteCapacity() > VSProtocol.MINSIZE) { + boolean stopAfterSingleSend = SINGLE_SEND_ON_PARTIAL_FLUSH && !flushAll; + return flushInternal(true, stopAfterSingleSend); + } else { + return 0; + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UncheckedInterruptedException(e); + } + } + + // Must be called by the thread that performs flush under lock + @GuardedBy("this") + private int checkCapacity() { + return remoteCapacityAvailable.addAndGet(capacityIncrement.getAndSet(0)); + } + + private int getRemoteCapacity() { + return remoteCapacityAvailable.get(); + } + + @GuardedBy("this") + private int flushInternal (boolean partialOk, boolean stopAfterSingleSend) throws IOException, InterruptedException { + assert partialOk || !stopAfterSingleSend : "stopAfterSingleSend is only allowed with partialOk==true"; + + // Currently we try to send all "available" bytes at once. + // This may be not the best idea with partialOk==true, because this means that this way we may prevent + // loader thread from adding new data to the buffer and blocking him for a long time. + // At the same time it possible that we will get into a blocking send because socket buffer + // is full. + // TODO: Consider sending only once if partialOk==true. Additionally the flush may return number of bytes sent + // so the ChannelExecutor can decide if it should sleep or not. Because if we sent at least some data + // then it's very likely that we spend on this more time, than the ChannelExecutor sleeps. + + int sent = 0; + for (;;) { + int remoteCapacity; + for (;;) { + remoteCapacity = checkCapacity(); + + if (available == 0) + return sent; + + if (closed) + throw new ChannelClosedException(); + + if (remoteCapacity >= VSProtocol.MINSIZE) { + // The only way to proceed to code after the loop + break; + } + + if (partialOk) + return sent; + + // "out" could be asynchronously flushed while this thread + // is in wait (). Therefore, we have to query the state of + // "out" after wait (). + + // TODO: "waiting" counter is not needed anymore. Consider removal. + waiting.incrementAndGet(); + wait (); + waiting.decrementAndGet(); + } + + // No need to get new value of getRemoteCapacity() here, we just got it in the loop above + int packetSize = Math.min(Math.min(available, remoteCapacity), VSProtocol.MAXSIZE); + + channel.send (buffer, 0, packetSize); + + remoteCapacityAvailable.addAndGet(-packetSize); + + size -= packetSize; + available -= packetSize; + sent += packetSize; + + assert size >= 0; + + if (size > 0) { + // Shift remaining data to the buffer start. + // Slow on big buffer sizes! + // TODO: Implement cyclic buffer instead + System.arraycopy(buffer, packetSize, buffer, 0, size); + + if (stopAfterSingleSend) { + // We stop after very first send to allow loader thread to add new data. + + // Move capacityIncrement to remoteCapacityAvailable, + // so threads that calls flushAvailable() or addAvailableCapacity() can see updated value. + checkCapacity(); + + return sent; + } + } + } + } + + @GuardedBy("this") + private void ensureCapacity (int c) { + int cap = buffer.length; + + if (cap < c) { + byte [] save = buffer; + + buffer = new byte [Util.doubleUntilAtLeast (cap, c)]; + + System.arraycopy (save, 0, buffer, 0, size); + } + } + + @Override + public synchronized void write (byte @NotNull [] b, int off, int len) + throws IOException + { + if (closed) + throw new ChannelClosedException(); + + int newSize = size + len; + + if (flushDisabled) { + ensureCapacity (newSize); + System.arraycopy (b, off, buffer, size, len); + size = newSize; + } + else if (newSize <= buffer.length) { + System.arraycopy (b, off, buffer, size, len); + available = size = newSize; + } + else { + try { + // TODO: We do not necessarily need full flush here. We need to get "len" bytes of free space + flushInternal (false, false); + + assert size == 0; + + if (len < maxCapacity) { + System.arraycopy (b, off, buffer, 0, len); + available = size = len; + } else { + send (b, off, len); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UncheckedInterruptedException (e); + } + } + } + + @Override + public synchronized void write(int b) throws IOException { + if (closed) + throw new ChannelClosedException(); + + try { + if (flushDisabled) + ensureCapacity (size + 1); + else if (size >= maxCapacity) + flushInternal (false, false); + + if (!flushDisabled) + available++; + + buffer [size++] = (byte) b; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UncheckedInterruptedException (e); + } + } + + public synchronized void setRemoteCapacity(int capacity) { + remoteCapacityAvailable.set(capacity); + notify(); + } + + private void wakeAfterCapacityAdded() { + synchronized (this) { + // Update remote capacity, so next "addAvailableCapacity" will not need to block + checkCapacity(); + // Wakeup waiting thread + notify(); + } + } + + public void addAvailableCapacity(int capacity) { + capacityIncrement.addAndGet(capacity); + if (getRemoteCapacity() < VSProtocol.MINSIZE) { + wakeAfterCapacityAdded(); + } + } + + /** + * Sends data immediately, without putting it into the buffer. + * Used when the data does not fit into buffer. + */ + @GuardedBy("this") + private int send (byte @NotNull [] data, int offset, int length) + throws IOException + { + int bytes = 0; + + try { + while (length > 0) { + int remoteCapacity; + for (;;) { + remoteCapacity = checkCapacity(); + + if (closed) + throw new ChannelClosedException(); + + if (remoteCapacity >= VSProtocol.MINSIZE) { + // The only way to proceed to code after the loop + break; + } + + //waiting.incrementAndGet(); + wait (); + //waiting.decrementAndGet(); + } + + // No need to get new value of getRemoteCapacity() here, we just got it in the loop above + int packetSize = Math.min(Math.min(length, remoteCapacity), VSProtocol.MAXSIZE); + + channel.send(data, offset, packetSize); + + remoteCapacityAvailable.addAndGet(-packetSize); + + length -= packetSize; + bytes += packetSize; + offset += packetSize; + } + } catch (InterruptedException e) { + throw new UncheckedInterruptedException (e); + } + + return bytes; + } + + @Override + public String toString() { + return "ChannelOutputStream@" + Integer.toHexString(hashCode()) + + " for channel=" + channel; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/ClientConnection.java b/util/src/main/java/com/epam/deltix/util/vsocket/ClientConnection.java new file mode 100644 index 00000000..0d5629bf --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/ClientConnection.java @@ -0,0 +1,58 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.Socket; + +/** + * @author Alexei Osipov + */ +public class ClientConnection { + private final Socket socket; + + private final InputStream in; + private final OutputStream os; + + private final BufferedInputStream bin; + + public ClientConnection(Socket socket) throws IOException { + this.socket = socket; + this.in = socket.getInputStream(); + this.os = socket.getOutputStream(); + this.bin = new BufferedInputStream(this.in, VSocketImpl.INPUT_STREAM_BUFFER_SIZE); + } + + public Socket getSocket() { + return socket; + } + + public InputStream getInputStream() { + return in; + } + + public BufferedInputStream getBufferedInputStream() { + return bin; + } + + public OutputStream getOutputStream() { + return os; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/ClientSocketProvider.java b/util/src/main/java/com/epam/deltix/util/vsocket/ClientSocketProvider.java new file mode 100644 index 00000000..76ffb1da --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/ClientSocketProvider.java @@ -0,0 +1,48 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import javax.net.SocketFactory; +import javax.net.ssl.SSLSocket; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; + +/** + * + */ +public class ClientSocketProvider { + + public static Socket open(SocketFactory factory, String host, int port, int connectTimeout, int timeout) throws IOException { + //create and connect + Socket socket = factory.createSocket(); + + socket.setSoTimeout(timeout); + socket.connect(new InetSocketAddress(host, port), connectTimeout); + + if (socket instanceof SSLSocket) { + //do ssl handshake + ((SSLSocket) socket).startHandshake(); + VSProtocol.LOGGER.fine("VSClient connected to SSL server " + host + ":" + port); + } else { + VSProtocol.LOGGER.fine("VSClient connected to server " + host + ":" + port); + } + + return socket; + } + +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/ConnectionAbortedException.java b/util/src/main/java/com/epam/deltix/util/vsocket/ConnectionAbortedException.java new file mode 100644 index 00000000..6bd74102 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/ConnectionAbortedException.java @@ -0,0 +1,41 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import java.io.IOException; + +/** + * Date: Apr 14, 2010 + * Time: 5:45:37 PM + */ +public class ConnectionAbortedException extends IOException { + + public ConnectionAbortedException() { + } + + public ConnectionAbortedException(String message) { + super(message); + } + + public ConnectionAbortedException(String message, Throwable cause) { + super(message, cause); + } + + public ConnectionAbortedException(Throwable cause) { + super(cause); + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/ConnectionRejectedException.java b/util/src/main/java/com/epam/deltix/util/vsocket/ConnectionRejectedException.java new file mode 100644 index 00000000..8c34cd9c --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/ConnectionRejectedException.java @@ -0,0 +1,34 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import java.io.IOException; + +/** + * + */ +public class ConnectionRejectedException extends IOException { + public final String serverId; + public final int errorCode; + + public ConnectionRejectedException (String serverId, int errorCode) { + super ("Connection rejected by " + serverId + "; error code: " + errorCode); + + this.serverId = serverId; + this.errorCode = errorCode; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/ConnectionStateListener.java b/util/src/main/java/com/epam/deltix/util/vsocket/ConnectionStateListener.java new file mode 100644 index 00000000..f9c13119 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/ConnectionStateListener.java @@ -0,0 +1,48 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.vsocket; + +abstract class ConnectionStateListener { + /** + * Triggered when connection loss causes dispatcher to give up on recovery. + *

+ * Triggered only once per dispatcher lifecycle. + *

+ * Not triggered if dispatcher is stopped normally with {@link VSDispatcher#close()}. + */ + abstract void onDisconnected(); + + /** + * Triggered when the first connection is established. + */ + abstract void onConnected(); + + /** + * Triggered when transport is stopped (e.g. connection lost) but may be recoverable. + * + * @return true if transport is already known to be unrecoverable (and recovery should be stopped right away) + */ + abstract boolean onTransportRecoveryStart(VSocketRecoveryInfo recoveryInfo); + + /** + * Triggered when transport recovery have to stop (because of timeout or dispatcher shutdown). + * + * @return true if transport was permanently lost (can't be recovered anymore) + */ + abstract boolean onTransportRecoveryStop(VSocketRecoveryInfo recoveryInfo); +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/DBConnectionAcceptor.java b/util/src/main/java/com/epam/deltix/util/vsocket/DBConnectionAcceptor.java new file mode 100644 index 00000000..2e64eb2d --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/DBConnectionAcceptor.java @@ -0,0 +1,24 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +/** + * The interface tests if a connection can be accepted for the given client ID or not. + */ +public interface DBConnectionAcceptor { + boolean accept(String clientId); +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/DefaultConnectionAcceptor.java b/util/src/main/java/com/epam/deltix/util/vsocket/DefaultConnectionAcceptor.java new file mode 100644 index 00000000..4e4284af --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/DefaultConnectionAcceptor.java @@ -0,0 +1,29 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +public class DefaultConnectionAcceptor implements DBConnectionAcceptor { + public static final DefaultConnectionAcceptor INSTANCE = new DefaultConnectionAcceptor(); + + private DefaultConnectionAcceptor() { + } + + @Override + public boolean accept(String clientId) { + return true; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/EMA.java b/util/src/main/java/com/epam/deltix/util/vsocket/EMA.java new file mode 100644 index 00000000..23595f0f --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/EMA.java @@ -0,0 +1,78 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.time.TimeKeeper; + +/** + * Exponential moving average: + * EMA(v) = v*(1-k)+ previousEMA*k, where + * k = e^(-timeSincePreviousMeasurement/decayTime) + */ +public class EMA { + private long lastTimestamp = Long.MIN_VALUE; + private double lastAverage = Double.NaN; + private double lastValue = Double.NaN; + private double factor; + + /** + * Creates a running EMA instance + * + * @param decayTimeMillis Time in milliseconds during which a reading's + * weight falls to 1/e. + */ + public EMA (double decayTimeMillis) { + factor = -1 / decayTimeMillis; + } + + public double update (double value) { + register (value); + return (lastAverage); + } + + public void clear () { + lastTimestamp = Long.MIN_VALUE; + lastAverage = Double.NaN; + lastValue = Double.NaN; + } + + public double getLastRegisteredValue () { + return (lastValue); + } + + public double getAverage () { + return (lastAverage); + } + + public void register (double value) { + register (value, TimeKeeper.currentTime); + } + + public void register (double value, long time) { + lastValue = value; + + if (lastTimestamp == Long.MIN_VALUE) + lastAverage = value; + else { + double alpha = Math.exp ((time - lastTimestamp) * factor); + + lastAverage = value * (1 - alpha) + lastAverage * alpha; + } + + lastTimestamp = time; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/IncompatibleClientException.java b/util/src/main/java/com/epam/deltix/util/vsocket/IncompatibleClientException.java new file mode 100644 index 00000000..3590e40e --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/IncompatibleClientException.java @@ -0,0 +1,38 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import java.io.IOException; + +/** + * + */ +public class IncompatibleClientException extends IOException { + public final String serverId; + public final int serverVersion; + + public IncompatibleClientException (String serverId, int serverVersion) { + super ("VS protocol version " + serverVersion + + " of server " + serverId + + " is incompatible; expected [" + VSClient.MIN_COMP_SERVER_VERSION + + " .. " + VSClient.MAX_COMP_SERVER_VERSION + "]" + ); + + this.serverId = serverId; + this.serverVersion = serverVersion; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/MemorySocket.java b/util/src/main/java/com/epam/deltix/util/vsocket/MemorySocket.java new file mode 100644 index 00000000..961c23de --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/MemorySocket.java @@ -0,0 +1,97 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.collections.ByteQueue; +import com.epam.deltix.util.io.ByteQueueInputStream; +import com.epam.deltix.util.io.ByteQueueOutputStream; + +import java.io.BufferedInputStream; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * + */ +public class MemorySocket implements VSocket { + + ByteQueue q; + private InputStream in; + private BufferedInputStream bin; + + private OutputStream out; + private VSocketOutputStream vout; + private VSocketInputStream vin; + private int socketNumber; + + public MemorySocket(int socketNumber) { + this.socketNumber = socketNumber; + q = new ByteQueue(1024 * 1024 * 10); + in = new ByteQueueInputStream(q); + bin = new BufferedInputStream(in); + vin = new VSocketInputStream(bin, getSocketIdStr()); + } + + public MemorySocket(MemorySocket remote, int socketNumber) { + this(socketNumber); + + remote.out = new ByteQueueOutputStream((ByteQueueInputStream) in); + remote.vout = new VSocketOutputStream(remote.out, getSocketIdStr()); + + this.out = new ByteQueueOutputStream((ByteQueueInputStream) remote.in); + this.vout = new VSocketOutputStream(out, getSocketIdStr()); + } + + @Override + public VSocketInputStream getInputStream() { + return vin; + } + + @Override + public VSocketOutputStream getOutputStream() { + return vout; + } + + @Override + public String getRemoteAddress() { + return null; + } + + @Override + public void close() { + + } + + @Override + public int getCode() { + return hashCode(); + } + + @Override + public void setCode(int code) { + } + + @Override + public String toString() { + return getClass().getName() + getSocketIdStr(); + } + + @Override + public int getSocketNumber() { + return socketNumber; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/OffHeapIpcSocket.java b/util/src/main/java/com/epam/deltix/util/vsocket/OffHeapIpcSocket.java new file mode 100644 index 00000000..6448fcef --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/OffHeapIpcSocket.java @@ -0,0 +1,103 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.io.IOUtil; +import com.epam.deltix.util.io.offheap.OffHeap; +import com.epam.deltix.util.lang.Util; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.Socket; + +/** + * + */ +public class OffHeapIpcSocket implements VSocket { + private Socket socket; + private InputStream in; + private OutputStream out; + private VSocketInputStream vin; + private VSocketOutputStream vout; + private int code; + private int socketNumber; + + public OffHeapIpcSocket(Socket socket, int code, boolean isServer, int socketNumber) throws IOException { + this.socket = socket; + this.code = code; + this.socketNumber = socketNumber; + + String inName; + String outName; + if (isServer) { + inName = "__dtx_" + socket.getLocalPort() + "_" + code + "s.tmp"; + outName = "__dtx_" + socket.getLocalPort() + "_" + code + "c.tmp"; + } else { + inName = "__dtx_" + socket.getPort() + "_" + code + "c.tmp"; + outName = "__dtx_" + socket.getPort() + "_" + code + "s.tmp"; + } + + in = OffHeap.createInputStream(inName); + out = OffHeap.createOutputStream(outName); + + vin = new VSocketInputStream(in, getSocketIdStr()); + vout = new VSocketOutputStream(out, getSocketIdStr()); + } + + @Override + public VSocketInputStream getInputStream() { + return vin; + } + + @Override + public VSocketOutputStream getOutputStream() { + return vout; + } + + @Override + public String getRemoteAddress() { + return String.valueOf(code); + } + + @Override + public void close() { + Util.close(in); + Util.close(out); + IOUtil.close(socket); + } + + @Override + public int getCode() { + return code; + } + + @Override + public void setCode(int code) { + this.code = code; + } + + @Override + public String toString() { + return getClass().getName() + getSocketIdStr(); + } + + @Override + public int getSocketNumber() { + return socketNumber; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/ServerRestartedException.java b/util/src/main/java/com/epam/deltix/util/vsocket/ServerRestartedException.java new file mode 100644 index 00000000..14565b4b --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/ServerRestartedException.java @@ -0,0 +1,29 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import java.io.IOException; + +/** + * Signals that Timebase server was restarted during reconnecting. + */ +public class ServerRestartedException extends IOException { + + public ServerRestartedException (String serverId, long time) { + super ("Server " + serverId + " was restarted at " + time); + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/SocketType.java b/util/src/main/java/com/epam/deltix/util/vsocket/SocketType.java new file mode 100644 index 00000000..d1ed09c9 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/SocketType.java @@ -0,0 +1,25 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +/** + * + */ +public enum SocketType { + UNSAFE, + SSL +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/TLSContext.java b/util/src/main/java/com/epam/deltix/util/vsocket/TLSContext.java new file mode 100644 index 00000000..97213070 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/TLSContext.java @@ -0,0 +1,36 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import javax.net.ssl.SSLContext; + +public class TLSContext { + + public SSLContext context; + public final String protocol = "TLS"; + + // disables SSL communication for loopback connections + public boolean preserveLoopback = true; + + // port for SSL connections + public int port; + + public TLSContext(boolean preserveLoopback, int port) { + this.preserveLoopback = preserveLoopback; + this.port = port; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/TransportProperties.java b/util/src/main/java/com/epam/deltix/util/vsocket/TransportProperties.java new file mode 100644 index 00000000..e6158ef6 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/TransportProperties.java @@ -0,0 +1,41 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + + +/** + * + */ +public class TransportProperties { + //public static final String TRANSPORT_DIR_DEF = Home.getPath("temp/dxipc"); + + public final TransportType transportType; + public final String transportDir; + +// public TransportProperties() { +// this(TransportType.SOCKET_TCP); +// } + +// public TransportProperties(TransportType transportType) { +// this(transportType, TRANSPORT_DIR_DEF); +// } + + public TransportProperties(TransportType transportType, String transportDir) { + this.transportType = transportType; + this.transportDir = transportDir; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/TransportRecoveryFailre.java b/util/src/main/java/com/epam/deltix/util/vsocket/TransportRecoveryFailre.java new file mode 100644 index 00000000..26ddcf86 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/TransportRecoveryFailre.java @@ -0,0 +1,25 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +/** + * This exception indicates that transport recovery is failed and any additional attempts will not help. + * + * @author Alexei Osipov + */ +class TransportRecoveryFailre extends Exception { +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/TransportType.java b/util/src/main/java/com/epam/deltix/util/vsocket/TransportType.java new file mode 100644 index 00000000..80cdce00 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/TransportType.java @@ -0,0 +1,27 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +/** + * + */ +public enum TransportType { + SOCKET_TCP, + @Deprecated + AERON_IPC, + OFFHEAP_IPC; +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/VSChannel.java b/util/src/main/java/com/epam/deltix/util/vsocket/VSChannel.java new file mode 100644 index 00000000..5cd5abb9 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/VSChannel.java @@ -0,0 +1,91 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.lang.Disposable; +import com.epam.deltix.util.lang.DisposableListener; +import org.jetbrains.annotations.Nullable; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.InputStream; + +/** + * A virtual socket. + */ +public interface VSChannel extends Disposable { + + public int getLocalId (); + + public int getRemoteId (); + + public String getRemoteAddress(); + + public String getClientAddress(); + + public String getRemoteApplication(); + + public String getClientId(); + + public VSOutputStream getOutputStream (); + + public DataOutputStream getDataOutputStream (); + + public InputStream getInputStream (); + + public DataInputStream getDataInputStream (); + + public VSChannelState getState(); + + public boolean setAutoflush(boolean value); + + public boolean isAutoflush(); + + public void close(boolean terminate); + + public void setAvailabilityListener (Runnable lnr); + + public Runnable getAvailabilityListener (); + + public boolean getNoDelay(); + + public void setNoDelay(boolean value); + + public String encode(String value); + + public String decode(String value); + + void addDisposableListener(DisposableListener listener); + + void removeDisposableListener(DisposableListener listener); + + + /** + * @return value previously set by {@link #setTag(String)} + * + * @apiNote experimental + */ + @Nullable + String getTag(); + + /** + * Sets an arbitrary tag that can be used for debugging purposes. It is not sent to the remote side. + * + * @apiNote experimental + */ + void setTag(@Nullable String tag); +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/VSChannelImpl.java b/util/src/main/java/com/epam/deltix/util/vsocket/VSChannelImpl.java new file mode 100644 index 00000000..f6b83c1d --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/VSChannelImpl.java @@ -0,0 +1,919 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.concurrent.ContextContainer; +import com.epam.deltix.util.concurrent.QuickExecutor; +import com.epam.deltix.util.io.CountingInputStream; +import com.epam.deltix.util.io.GapQueueInputStream; +import com.epam.deltix.util.lang.DisposableListener; +import com.epam.deltix.util.lang.Util; +import com.epam.deltix.util.memory.DataExchangeUtils; +import com.epam.deltix.util.memory.MemoryDataOutput; +import net.jcip.annotations.GuardedBy; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Nullable; + +import javax.annotation.CheckReturnValue; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Comparator; +import java.util.HashSet; +import java.util.PriorityQueue; +import java.util.logging.Level; +import java.util.zip.DataFormatException; +import java.util.zip.Deflater; +import java.util.zip.Inflater; + +/** + * + */ +final class VSChannelImpl implements VSChannel { + @ApiStatus.Experimental // Configurable option for testing optimal value of notifyThreshold in different setups + private static final int MAX_NOTIFY_THRESHOLD = Integer.getInteger("TimeBase.network.channel.maxNotifyThreshold", VSProtocol.MAXSIZE); + + private final ContextContainer contextContainer; + private volatile VSChannelState state = VSChannelState.NotConnected; + + private final VSDispatcher dispatcher; + private final int localId; + private volatile int remoteId = -1; + + // for debug purposes inCapacity and outCapacity can be changed to smaller sizes + private final int inCapacity; // = 1 << 15; + private final int outCapacity; // = 1 << 14; + + private final GapQueueInputStream in; + private final CountingInputStream cin; + private DataInputStream din; + + private final ChannelOutputStream out; + private DataOutputStream dout; + + private boolean autoFlush = false; + private boolean noDelay = false; + private final byte [] buffer8 = new byte[12]; + + private final int index; + private volatile int remoteIndex = -1; + + private volatile Runnable listener; + + private final QuickExecutor.QuickTask lnrNotifier; + + private final boolean compressed; + + @GuardedBy("inflater") + private final Inflater inflater; + @GuardedBy("inflater") + private final MemoryDataOutput infOut; + + // Previously was protected by "deflater" itself however this was redundant as we always sync on "this" + @GuardedBy("this") + private final Deflater deflater; + @GuardedBy("this") + private final MemoryDataOutput defOut; + + private volatile long numBytesSend; // synchronized by "this" + private final Counter numBytesRead = new Counter(); + + private String tag; // Arbitrary tag for debugging purposes. It is not sent to the remote side. + + @GuardedBy("listeners") + private final HashSet> listeners = new HashSet<>(); + +// private final StringBuffer sendLog = new StringBuffer(); +// private final StringBuffer recievedLog = new StringBuffer(); + + private final PriorityQueue commands = new PriorityQueue(2, + new Comparator() { + @Override + public int compare(ChannelCommand a, ChannelCommand b) { + if (a.offset == b.offset) + return Util.compare(a.code, b.code); + + return a.offset > b.offset ? 1 : -1; + } + }); + + static class Counter { + private long count = 0; + + public synchronized long increment(long v) { + return count += v; + } + + public synchronized long value() { + return count; + } + } + + static abstract class ChannelCommand { + + public long offset; + public int code; + + public ChannelCommand(int code, long offset) { + this.offset = offset; + this.code = code; + } + + public abstract void run(); + } + + public class RemoteClosing extends ChannelCommand { + + public RemoteClosing (long offset) { + super(VSProtocol.CLOSING, offset); + + onRemoteClosing(); + } + + @Override + public void run() { + in.finish(); + notifyDataAvailable(); + } + } + + public class RemoteClosed extends ChannelCommand { + public RemoteClosed(long offset) { + super(VSProtocol.CLOSED, offset); + } + + @Override + public void run() { + onRemoteClosed(); + } + } + + public VSChannelImpl (VSDispatcher dispatcher, int inCapacity, int outCapacity, + boolean compressed, int localId, int index, ContextContainer contextContainer) { + if (inCapacity <= 0) + throw new IllegalArgumentException("inCapacity"); + if (outCapacity <= 0) + throw new IllegalArgumentException("outCapacity"); + + this.dispatcher = dispatcher; + this.localId = localId; + this.index = index; + this.compressed = compressed; + this.inCapacity = inCapacity; + this.outCapacity = outCapacity; + + this.contextContainer = contextContainer; + + inflater = compressed ? new Inflater() : null; + infOut = compressed ? new MemoryDataOutput(inCapacity) : null; + deflater = compressed ? new Deflater() : null; + defOut = compressed ? new MemoryDataOutput(outCapacity) : null; + + this.out = new ChannelOutputStream(this, outCapacity); + this.in = new GapQueueInputStream (inCapacity); + + // In case of big buffer we want to notify sender as soon as a full packet can be sent + // or 1/4 of max capacity accumulated + int notifyThreshold = Math.min(inCapacity / 4, MAX_NOTIFY_THRESHOLD); + this.cin = new CountingInputStream(this.in, notifyThreshold) { + + @Override + protected boolean bytesRead(long change) { + try { + sendBytesRead(change); + return true; + } catch (IOException e) { + if (!VSChannelImpl.this.in.isClosed()) + VSProtocol.LOGGER.log (Level.WARNING, "Error sending bytes read.", e); + return false; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + }; + + this.lnrNotifier = + new QuickExecutor.QuickTask (contextContainer.getQuickExecutor()) { + @Override + public void run () { + Runnable consistentListener = listener; + + if (consistentListener != null) + consistentListener.run (); + } + }; + } + + public int getLocalId () { + return (localId); + } + + public int getRemoteId () { + return (remoteId); + } + + public String getRemoteAddress() { + return dispatcher != null ? dispatcher.getRemoteAddress() : null; + } + + @Override + public String getClientAddress() { + return dispatcher != null ? dispatcher.getClientAddress() : null; + } + + public String getRemoteApplication() { + return dispatcher != null ? dispatcher.getApplicationID() : null; + } + + @Override + public String getClientId() { + return dispatcher != null ? dispatcher.getClientId() : null; + } + + public InputStream getInputStream () { + return (cin); + } + + public DataInputStream getDataInputStream () { + if (din == null) + din = new DataInputStream(getInputStream()); + + return (din); + } + + public VSOutputStream getOutputStream () { + return (out); + } + + @Override + public DataOutputStream getDataOutputStream() { + if (dout == null) + dout = new DataOutputStream (out); + + return (dout); + } + + public VSChannelState getState() { + return state; + } + +// public boolean isClosed () { +// return state == VSChannelState.Closed || state == VSChannelState.Removed; +// } + + private boolean isRemoteConnected() { + return state == VSChannelState.Connected; + } + + public void close () { + close(false); + } + + public void close (boolean terminate) { + // Must close "out" before grabbing lock on "this", so that + // another thread sending data releases its lock on "this", + /// held in send (). + Util.close (in); + + //System.out.println(this + " closing having read " + numBytesRead.value()); + + if (isRemoteConnected () && !terminate) { + Util.close (out); // flush all data + } else { +// if (out.size() > 0) +// LOGGER.info(this + ": closing having available bytes=" + out.size()); + out.closeNoFlush(); // do not flush data + } + + synchronized (this) { + if (state == VSChannelState.Removed) + return; + + if (state == VSChannelState.Closed) + return; + + try { + // send 'closing' signal only when connected + if (state != VSChannelState.NotConnected) + sendClosing(); + else + VSProtocol.LOGGER.log(Level.FINE, this + " not connected yet"); + + // if remote in 'disconnected' state - send signal to close + if (state == VSChannelState.RemoteClosed) + sendClosed(); + + } catch (ConnectionAbortedException x) { + VSProtocol.LOGGER.log (Level.FINE, "Error sending disconnect.", x); + } catch (InterruptedException x) { + Thread.currentThread().interrupt(); + VSProtocol.LOGGER.log (Level.FINE, "Sending disconnect interrupted.", x); + } catch (Exception x) { + VSProtocol.LOGGER.log (Level.WARNING, "Error sending disconnect", x); + } + + state = VSChannelState.Closed; + } + + notifyListeners(); + } + + public void processCommand(int cmd, long position) { + assert position >= 0; + + ChannelCommand command = null; + if (cmd == VSProtocol.CLOSING) + command = new RemoteClosing(position); + else if (cmd == VSProtocol.CLOSED) + command = new RemoteClosed(position); + + synchronized (commands) { + if (command != null) + commands.offer(command); + } + +// if (numBytesRead > position) +// System.out.println(this + ": command (" + cmd + ", " + position + ") is out " + numBytesRead); +// else if (numBytesRead < position) +// System.out.println(this + ": command (" + cmd + ", " + position + ") delayed by " + (position - numBytesRead)); + + checkCommands(); + } + + private void checkCommands() { + + long position = numBytesRead.value(); + + if (!commands.isEmpty()) { + ChannelCommand command; + + synchronized (commands) { + command = commands.peek(); + if (command != null && command.offset == position) + command = commands.poll(); + } + + while (command != null && command.offset == position) { + command.run(); + position = numBytesRead.increment(2); + + synchronized (commands) { + command = commands.poll(); + } + } + } + } + + void onRemoteClosed() { + dispatcher.channelClosed (this); + + synchronized (this) { + state = VSChannelState.Removed; + } + + notifyListeners(); + } + + void onRemoteClosing() { + // close output stream - we cannot send any data + out.closeNoFlush(); + + synchronized (this) { + + switch (state) { + case Connected: + state = VSChannelState.RemoteClosed; + break; + + case Closed: + try { + sendClosed(); + } catch (Exception x) { + if (x instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + VSProtocol.LOGGER.log (Level.WARNING, "Error sending disconnect", x); + } + break; + } + } + + notifyListeners(); + } + + void onDisconnected(IOException error) { + // finish input stream - we do not expect any data + if (error != null) + in.putError(error); + + in.finish(); + + // close output stream - we cannot send any data + out.closeNoFlush(); + + synchronized (this) { + switch (state) { + case Connected: + state = VSChannelState.RemoteClosed; + break; + } + } + + // notify availability listener after input close + notifyDataAvailable(); + + // notify disposable listeners + notifyListeners(); + } + +// void onRemoteClosing() { +// // finish input stream - we do not expect any data +// in.finish(); +// +// // close output stream - we cannot send any data +// out.closeNoFlush(); +// +// boolean shouldSendClosed = false; +// +// synchronized (this) { +// switch (state) { +// case Connected: +// state = VSChannelState.RemoteClosed; +// break; +// +// case Closed: +// shouldSendClosed = hasTransport; +// break; +// +// default: +// break; +//// if (hasTransport) +//// throw new IllegalStateException (state.name ()); +// } +// } +// +// // notify availability listener after state change +// notifyDataAvailable(); +// +// if (shouldSendClosed) { +// try { +// sendClosed(); +// } catch (Throwable x) { +// LOGGER.log (Level.WARNING, "Error sending disconnect", x); +// } +// } +// } + +// void receive(long position, byte [] data, int offset, int length) throws IOException { +// // synchronized asserts is NOT allowed - will block transport +// try { +// if (compressed) { +// int size = decompress(data, offset, length); +// in.putData(infOut.getBuffer(), 0, size); +// } else { +// in.putData (data, offset, length); +// System.out.println("receive(" + offset + ", " + length + ")"); +// } +// +// notifyDataAvailable(); +// } catch (EOFException e) { +// // ignore +// } +// } + + void receive(long position, byte[] data, int offset, int length, int code, int socketNumber) throws IOException { + // synchronized asserts is NOT allowed - will block transport + boolean available = false; + //recievedLog.append(position).append(":").append(length).append("\n\r"); + + int unpackedLength = 0; + try { + int queuePosition = (int) (position % inCapacity); + if (compressed) { + synchronized (inflater) { + unpackedLength = decompress(data, offset, length); + available = in.putData(infOut.getBuffer(), 0, unpackedLength, queuePosition); + } + } else { + unpackedLength = length; + available = in.putData (data, offset, unpackedLength, queuePosition); + } + + if (VSProtocol.LOGGER.isLoggable(Level.FINEST)) { + VSProtocol.LOGGER.log(Level.FINEST, "Got data block " + position + ":" + (position + unpackedLength) + " (" + length + "/" + unpackedLength + ") from socket: @" + Integer.toHexString(code) + "#" + socketNumber); + } + } catch (EOFException e) { + // ignore + } finally { + numBytesRead.increment(unpackedLength); + } + + if (available) { + notifyDataAvailable(); + } + // It's necessary to still process commands when "in" gets closed to properly complete + // the channel closing process. Otherwise, commands may stuck in queue and the channel may leak. + // See https://gitlab.deltixhub.com/Deltix/QuantServer/QuantServer/-/issues/1264 + if (available || in.isClosed()) { + checkCommands(); + } + } + + private void notifyDataAvailable() { + if (listener != null) + lnrNotifier.submit (); + } + + void sendBytesRead (long bytes) + throws InterruptedException, IOException + { + VSChannelState state = getState(); + + if (state != VSChannelState.Connected && state != VSChannelState.RemoteClosed) { + if (VSProtocol.LOGGER.isLoggable(Level.FINE)) + VSProtocol.LOGGER.log(Level.FINE, "Skipping BYTES_AVAILABLE_REPORT report: " + bytes + " because channel state is " + state); + return; + } + + state = getState(); + + if (state == VSChannelState.RemoteClosed && VSProtocol.LOGGER.isLoggable(Level.FINE)) + VSProtocol.LOGGER.log(Level.FINE, "Sending BYTES_AVAILABLE_REPORT report: " + bytes + " at state " + state + " with remoteIndex=" + remoteIndex); + + DataExchangeUtils.writeInt (buffer8, 4, (int)bytes); + DataExchangeUtils.writeInt (buffer8, 8, remoteIndex); + + VSTransportChannel tc = dispatcher.checkOut (); + if (VSProtocol.LOGGER.isLoggable(Level.FINEST)) + VSProtocol.LOGGER.log(Level.FINEST, "Sending BYTES_AVAILABLE_REPORT report: " + ((int)bytes) + " from " + tc.getSocketIdStr()); + + try { + tc.write (buffer8, 0, buffer8.length); + } finally { + dispatcher.checkIn (tc); + } + } + + void sendConnect () + throws InterruptedException, IOException + { + byte [] data = new byte[17]; + + DataExchangeUtils.writeUnsignedShort (data, 0, VSProtocol.LISTENER_ID); + DataExchangeUtils.writeUnsignedShort (data, 2, localId); + DataExchangeUtils.writeInt (data, 4, inCapacity); + DataExchangeUtils.writeInt (data, 8, outCapacity); + DataExchangeUtils.writeInt (data, 12, index); + DataExchangeUtils.writeByte(data, 16, compressed ? 1 : 0); + + final VSTransportChannel tc = dispatcher.checkOut (); + try { + tc.write (data, 0, data.length); + } finally { + dispatcher.checkIn (tc); + } + } + + @GuardedBy("this") + void sendClosing() + throws InterruptedException, IOException + { + assert remoteId != -1; + + byte [] data = new byte [16]; + + DataExchangeUtils.writeUnsignedShort (data, 0, remoteId); + DataExchangeUtils.writeUnsignedShort (data, 2, VSProtocol.CLOSING); + DataExchangeUtils.writeInt(data, 4, remoteIndex); + DataExchangeUtils.writeLong (data, 8, numBytesSend); + + final VSTransportChannel tc = dispatcher.checkOut (); + try { + tc.write (data, 0, data.length); + //noinspection NonAtomicOperationOnVolatileField This is safe because writes always happen with lock on "this" + numBytesSend += 2; + } finally { + dispatcher.checkIn (tc); + } + + if (VSProtocol.LOGGER.isLoggable(Level.FINEST)) + VSProtocol.LOGGER.log(Level.FINEST, "Sending CLOSING: remoteIndex=" + remoteIndex + " numBytesSend=" + numBytesSend); + } + + @GuardedBy("this") + void sendClosed () + throws InterruptedException, IOException + { + assert remoteId != -1; + + byte [] data = new byte [16]; + + DataExchangeUtils.writeUnsignedShort (data, 0, remoteId); + DataExchangeUtils.writeUnsignedShort (data, 2, VSProtocol.CLOSED); + DataExchangeUtils.writeInt (data, 4, remoteIndex); + DataExchangeUtils.writeLong(data, 8, numBytesSend); + + final VSTransportChannel tc = dispatcher.checkOut (); + try { + tc.write (data, 0, data.length); + //noinspection NonAtomicOperationOnVolatileField This is safe because writes always happen with lock on "this" + numBytesSend += 2; + } finally { + dispatcher.checkIn (tc); + } + + if (VSProtocol.LOGGER.isLoggable(Level.FINEST)) + VSProtocol.LOGGER.log(Level.FINEST, "Sending CLOSED: remoteIndex=" + remoteIndex + " numBytesSend=" + numBytesSend); + } + + synchronized void send(byte [] data, int offset, int length) + throws InterruptedException, IOException + { + switch (state) { + case Connected: { + VSTransportChannel tc = null; + + try { + tc = dispatcher.checkOut (); + if (compressed) { + int compressed = compress(data, offset, length); + tc.write(remoteId, remoteIndex, numBytesSend, defOut.getBuffer(), 0, compressed, length); + } else { + tc.write(remoteId, remoteIndex, numBytesSend, data, offset, length, length); + } + numBytesSend += length; + //sendLog.append("sending bytes: ").append(numBytesSend); + + } finally { + if (tc != null) + dispatcher.checkIn (tc); + } + break; + } + + case Closed: + case RemoteClosed: + throw new ChannelClosedException (); + + default: + throw new IllegalStateException (state.name ()); + } + } + + void onCapacityIncreased(int capacity) { + out.addAvailableCapacity(capacity); + } + + void onConnectionRequest(int remoteId, int remoteCapacity, int rIndex) + throws InterruptedException, IOException + { + assert this.remoteId == -1; + + this.remoteIndex = rIndex; + this.remoteId = remoteId; + + DataExchangeUtils.writeUnsignedShort (buffer8, 0, remoteId); + DataExchangeUtils.writeUnsignedShort (buffer8, 2, VSProtocol.BYTES_AVAILABLE_REPORT); + + // + // Send CONNECT_ACK + // + byte [] data = new byte [14]; + DataExchangeUtils.writeUnsignedShort (data, 0, remoteId); + DataExchangeUtils.writeUnsignedShort (data, 2, VSProtocol.CONNECT_ACK); + DataExchangeUtils.writeUnsignedShort (data, 4, localId); + DataExchangeUtils.writeInt (data, 6, inCapacity); + DataExchangeUtils.writeInt (data, 10, index); + + final VSTransportChannel tc = dispatcher.checkOut (); + try { + tc.write (data, 0, data.length); + } finally { + dispatcher.checkIn (tc); + } + + synchronized (this) { + state = VSChannelState.Connected; + } + // set remote capacity after sending CONNECT_ACK to prevent closing channel + out.setRemoteCapacity(remoteCapacity); + } + + @CheckReturnValue + boolean assertIndexValid(int dataSize, int incoming) { + boolean valid = index == incoming; + if (!valid) + VSProtocol.LOGGER.log (Level.SEVERE, + this + "[Data:" + dataSize + "] - Wrong channel (remote: " + incoming + "; local: " + index + ")"); + + //LOGGER.info(this + ":" + index + " - [Data:" + dataSize + "]"); + //System.out.println(index + " [Data:" + dataSize + "]"); + + assert valid : "[Data:" + dataSize + "] - Wrong channel (remote: " + incoming + "; local: " + index + ")"; + return valid; + } + + @CheckReturnValue + boolean assertIndexValid(String method, int incoming) { + boolean valid = index == incoming; + if (!valid) + VSProtocol.LOGGER.log (Level.SEVERE, + this + "[" + method + "] - Wrong channel (remote: " + incoming + "; local: " + index + ")"); + + //LOGGER.info(this + ":" + index + " - [" + method + "]"); + //LOGGER.log (Level.INFO, index + " - [" + method + "]"); + assert valid : "[" + method + "] - Wrong channel (remote: " + incoming + "; local: " + index + ")"; + return valid; + } + + void onRemoteConnected(int remoteId, int remoteCapacity, int rIndex) { + assert this.remoteId == -1; + + this.remoteId = remoteId; + this.remoteIndex = rIndex; + + DataExchangeUtils.writeUnsignedShort (buffer8, 0, remoteId); + DataExchangeUtils.writeUnsignedShort (buffer8, 2, VSProtocol.BYTES_AVAILABLE_REPORT); + + boolean wasClosed = state == VSChannelState.Closed; + //assert state == VSChannelState.NotConnected; //TODO: check this + + state = VSChannelState.Connected; + out.setRemoteCapacity(remoteCapacity); + + if (wasClosed) { + //System.out.println(this + " onRemoteConnected for closing channel"); + close(); + } + } + + @Override + public boolean setAutoflush(boolean value) { + autoFlush = value; + return true; + } + + @Override + public boolean isAutoflush() { + return autoFlush; + } + + @Override + public boolean getNoDelay() { + return noDelay; + } + + @Override + public void setNoDelay(boolean value) { + this.noDelay = value; + + if (value) { + // TODO: Ideally we should delegate creation of ChannelExecutor instances to ContextContainer but ContextContainer can't access class ChannelExecutor. + ChannelExecutor.getInstance(contextContainer.getAffinityConfig()).addChannel(this); + } + } + + @Override + public void setAvailabilityListener(Runnable lnr) { + listener = lnr; + } + + @Override + public Runnable getAvailabilityListener() { + return listener; + } + + @Override + public String encode(String value) { + char[] msg = value.toCharArray(); + char[] id = getClientId().toCharArray(); + char[] output = new char[msg.length]; + + for (int i = 0; i < msg.length; i++) { + int index = i % id.length; + output[i] = (char) (msg[i] ^ id[index]); + } + + return new String(output); + } + + @Override + public String decode(String value) { + char[] msg = value.toCharArray(); + char[] id = getClientId().toCharArray(); + char[] output = new char[msg.length]; + + for (int i = 0; i < msg.length; i++) + output[i] = (char)(msg[i] ^ id[i % id.length]); + + return new String(output); + } + + @GuardedBy("this") + private int compress(byte[] data, int offset, int length) { + + deflater.reset(); + deflater.setLevel(length < 128 ? Deflater.NO_COMPRESSION : 3); + deflater.setInput(data, offset, length); + deflater.finish(); + + defOut.reset(0); + byte[] buffer = defOut.getBuffer(); + int count = deflater.deflate(buffer); + + while (!deflater.finished()) { + if (count >= buffer.length) { + defOut.ensureSize(defOut.getSize() + defOut.getSize() / 2); + buffer = defOut.getBuffer(); + } + count += deflater.deflate(buffer, count, buffer.length - count); + } + + return count; + } + + @GuardedBy("inflater") + private int decompress(byte[] data, int offset, int length) { + + inflater.reset(); + inflater.setInput(data, offset, length); + infOut.reset(0); + + try { + int count = inflater.inflate(infOut.getBuffer()); + while (!inflater.finished()) { + infOut.ensureSize(infOut.getSize() + infOut.getSize() / 2); + count += inflater.inflate(infOut.getBuffer(), count, infOut.getSize() - count); + } + + return count; + } catch (final DataFormatException e) { + throw new RuntimeException(e); + } + } + + @Override + public String toString () { + return ("VSChannel [local: " + localId + "; remote: " + (remoteId == Integer.MIN_VALUE ? "(none)" : remoteId) + "] (" + index + ")"); + } + + int getIndex() { + return index; + } + + @Override + public void addDisposableListener(DisposableListener listener) { + synchronized (listeners) { + listeners.add(listener); + } + } + + @Override + public void removeDisposableListener(DisposableListener listener) { + synchronized (listeners) { + listeners.remove(listener); + } + } + + private void notifyListeners() { + DisposableListener[] list; + + synchronized (listeners) { + //noinspection unchecked,ToArrayCallWithZeroLengthArrayArgument + list = listeners.toArray(new DisposableListener[listeners.size()]); + } + + for (DisposableListener dl : list) { + dl.disposed(this); + } + } + + @Override + @Nullable + public String getTag() { + return tag; + } + + @Override + public void setTag(@Nullable String tag) { + this.tag = tag; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/VSChannelState.java b/util/src/main/java/com/epam/deltix/util/vsocket/VSChannelState.java new file mode 100644 index 00000000..596ed072 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/VSChannelState.java @@ -0,0 +1,56 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +/** + * Date: Mar 30, 2010 + */ +public enum VSChannelState { + /** + * Just created no handshake yet. Writing to output stream is possible, + * but might block waiting for handshake (with remote capacity report). + * Reading from input stream will obviously block waiting for data to be + * sent over. + */ + NotConnected, + + /** + * Normal connected state. + */ + Connected, + + /** + * Remote endpoint has been closed. Writing to output stream will result in + * a {@link ChannelClosedException} being thrown. Reading from input stream + * will return all data that was sent prior to remote endpoint being closed, + * then EOF. + */ + RemoteClosed, + + /** + * Local endpoint has been closed. Either reading or writing will + * immediately throw a {@link ChannelClosedException}. + */ + Closed, + + /** + * Local close has been confirmed by remote side; therefore its id can + * now be re-used. To the caller, this state's behavior is identical to + * {@link #Closed}. + */ + Removed +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/VSClient.java b/util/src/main/java/com/epam/deltix/util/vsocket/VSClient.java new file mode 100644 index 00000000..6b83f6b5 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/VSClient.java @@ -0,0 +1,788 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.concurrent.ContextContainer; +import com.epam.deltix.util.concurrent.QuickExecutor; +import com.epam.deltix.util.time.TimeKeeper; +import com.epam.deltix.qsrv.hf.spi.conn.DisconnectEventListener; +import com.epam.deltix.util.io.GUID; +import com.epam.deltix.util.io.IOUtil; +import com.epam.deltix.util.io.offheap.OffHeap; +import com.epam.deltix.util.lang.Disposable; +import com.epam.deltix.util.lang.DisposableListener; +import com.epam.deltix.util.time.GlobalTimer; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.VisibleForTesting; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocket; +import java.io.*; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.util.Date; +import java.util.TimerTask; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.logging.Level; + +/** + * + */ +public class VSClient extends ConnectionStateListener implements Disposable, DisposableListener { + public static final int MIN_ALLOWED_SERVER_VERSION = 1014; + + public static final int MIN_COMP_SERVER_VERSION = VSProtocol.VERSION; + public static final int MAX_COMP_SERVER_VERSION = VSProtocol.VERSION; + + public static final String SSL_TERMINATION_PROPERTY = "TimeBase.network.VSClient.sslTermination"; + public static final boolean SSL_TERMINATION = Boolean.getBoolean(SSL_TERMINATION_PROPERTY); + + //private static final int MAX_TRANSPORT_RECONNECT_ATTEMPTS = Integer.getInteger("TimeBase.network.VSClient.maxTransportReconnectAttempts", 5); + private static final int TRANSPORT_RECONNECT_ATTEMPT_INTERVAL = Integer.getInteger("TimeBase.network.VSClient.transportReconnectAttemptInterval", 1000); + + private static final int RE_ATTEMPT_EXTRA_DELAY = 10; // Extra delay to avoid situation when we re-schedule task due to timer jitter + + private String host; + private int port; + private int numTransportChannels = 3; + + private volatile VSDispatcher dispatcher; + // Protects "dispatcher" field and interactions with quick executor + private final Object dispatcherLock = new Object(); + + private String clientId; + private long serverTime = -1; + + private volatile DisconnectEventListener listener; + private int reconnectInterval; + private VSCompression serverCompression; + + private int soTimeout = Integer.getInteger("TimeBase.network.VSClient.soTimeout", 5000); + private int timeout = Integer.getInteger("TimeBase.network.VSClient.timeout", 5000); + + private boolean enableSSL = false; + private final boolean sslTermination; + private int sslPort = 0; + private SSLContext sslContext; + + + private final ContextContainer contextContainer; + + private int protocolVersion = VSProtocol.VERSION; + + private volatile boolean closed = false; + + // Elements should be sorted (when possible) by time of last reconnection attempt however there is no strict enforcement for this. + // New broken sockets should be added to the head of the queue. + // Broken sockets that failed to reconnect should be added to the tail of the queue. + private final ConcurrentLinkedDeque broken = new ConcurrentLinkedDeque<>(); + + private final QuickExecutor.QuickTask reconnector; + + private QuickExecutor.QuickTask createReconnectorTask(final QuickExecutor quickExecutor) { + return new QuickExecutor.QuickTask(quickExecutor) { + @Override + public void run() { + for (;;) { + long currentTime = TimeKeeper.currentTime; + + VSocketRecoveryInfo socketRecovery = broken.peek(); + + if (dispatcher == null || socketRecovery == null) + break; + + long lastReconnectAttemptTs = socketRecovery.getLastReconnectAttemptTs(); + if (lastReconnectAttemptTs > currentTime - TRANSPORT_RECONNECT_ATTEMPT_INTERVAL) { + // It's too early to recover this socket + scheduleReconnectAttempt(lastReconnectAttemptTs + TRANSPORT_RECONNECT_ATTEMPT_INTERVAL + RE_ATTEMPT_EXTRA_DELAY); + return; + } + + boolean recoveryAttemptEnded = false; + synchronized (socketRecovery) { + if (!socketRecovery.startRecoveryAttempt()) { + if (socketRecovery.isRecoveryAttemptInProgress()) { + throw new IllegalStateException(); + } else { + // Recovery for that socket already ended + continue; + } + } + } + try { + if (!broken.remove(socketRecovery)) { + // The socket just was removed from broken list by other thread + continue; + } + + VSocket socket = socketRecovery.getSocket(); + + // Start reconnect attempt + int attemptNumber; + synchronized (socketRecovery) { + attemptNumber = socketRecovery.addReconnectAttempt(currentTime); + } + + + boolean success = false; + boolean transportLost = false; + try { + // Try to reconnect - long operation + VSocket vSocket = openTransport(socket); + if (vSocket != null) { + success = true; + dispatcher.addTransportChannel(vSocket); + synchronized (socketRecovery) { + if (socketRecovery.tryMarkRecoverySucceeded()) { + socketRecovery.notifyAll(); + } else { + VSProtocol.LOGGER.log(Level.WARNING, "Reconnect succeeded but recovery process is already cancelled"); + } + } + VSProtocol.LOGGER.log(Level.INFO, "Reconnect success, connection " + socket.getSocketIdStr() + ", address " + socket.getRemoteAddress() + " after " + attemptNumber + " attempts"); + } else { + VSProtocol.LOGGER.log(Level.INFO, "Reconnect failed (no error), connection " + socket.getSocketIdStr() + ", address " + socket.getRemoteAddress() + ", attempt " + attemptNumber); + } + } catch (ConnectionRejectedException e) { + // Explicit reject from server. That means that we should not try to reconnect anymore. + transportLost = true; + VSProtocol.LOGGER.log(Level.INFO, "Reconnect rejected by server, connection " + socket.getSocketIdStr() + ", address " + socket.getRemoteAddress() + ", attempt " + attemptNumber); + } catch (IOException e) { + VSProtocol.LOGGER.log(Level.INFO, "Reconnect failed (" + e.getMessage() + "), connection " + socket.getSocketIdStr() + ", address " + socket.getRemoteAddress() + ", attempt " + attemptNumber); + } catch (TransportRecoveryFailre transportRecoveryFailre) { + transportLost = true; + } + + if (!success) { + if (currentTime >= socketRecovery.getRecoveryDeadlineTs()) { + // At this time socket is discarded on the server side so we should give up now + VSProtocol.LOGGER.log(Level.WARNING, "Transport " + socket.getSocketIdStr() + " was not recovered after " + attemptNumber + " attempts (timeout reached)"); + transportLost = true; + } + if (transportLost) { + // We failed to recover the connection so we have to disconnect entire transport because we might loss some data + synchronized (socketRecovery) { + socketRecovery.stopRecoveryAttempt(); + assert !socketRecovery.isRecoverySucceeded(); + socketRecovery.markRecoveryFailed(); + socketRecovery.notifyAll(); + } + recoveryAttemptEnded = true; + //VSProtocol.LOGGER.log(Level.WARNING, "Disconnecting client due to failure to recover transport channel " + socket.getSocketIdStr() + " after " + attemptNumber + " attempts"); + + //VSClient.this.close(false); + return; + } else { + // We failed to recover but we can try again later + // Add to the last position so it will be last to try + broken.addLast(socketRecovery); + } + } + + } finally { + if (!recoveryAttemptEnded) { + synchronized (socketRecovery) { + socketRecovery.stopRecoveryAttempt(); + socketRecovery.notifyAll(); + } + } + } + } + } + + @Override + protected boolean killSupported() { + return true; + } + }; + } + + private void scheduleReconnectAttempt(long nextAttemptTimestamp) { + GlobalTimer.INSTANCE.schedule(new TimerTask() { + @Override + public void run() { + reconnector.submit(); + } + }, new Date(nextAttemptTimestamp)); + } + + @VisibleForTesting // Should by used in tests ONLY. TODO: Delete? + public VSClient (String host, int port, String ownerID) throws IOException { + this(host, port, ownerID, false, ContextContainer.getContextContainerForClientTests()); + } + + @VisibleForTesting // Should by used in tests ONLY. TODO: Create a factory method with name like "createClientForTests" + public VSClient (String host, int port) throws IOException { + this(host, port, null, false, ContextContainer.getContextContainerForClientTests()); + } + + public VSClient(String host, int port, @Nullable String ownerID, boolean enableSSL, ContextContainer contextContainer) throws IOException { + this(host, port, ownerID, enableSSL, SSL_TERMINATION, contextContainer); + } + + public VSClient(String host, int port, @Nullable String ownerID, boolean enableSSL, boolean sslTermination, + ContextContainer contextContainer) throws IOException { + this.host = host; + this.port = port; + this.enableSSL = enableSSL; + this.sslTermination = sslTermination; + this.contextContainer = contextContainer; + this.reconnector = createReconnectorTask(contextContainer.getQuickExecutor()); + + if (ownerID == null) + this.clientId = new GUID().toStringWithPrefix (InetAddress.getLocalHost().getHostAddress() + ":"); + else + this.clientId = new GUID().toStringWithPrefix(InetAddress.getLocalHost().getHostAddress() + ":" + ownerID + ":"); + } + + public void setClientAddress(String address, String ownerID) { + this.clientId = new GUID().toStringWithPrefix(address + ":" + ownerID + ":"); + } + + public int getSoTimeout () { + return soTimeout; + } + + public void setSoTimeout (int soTimeout) { + this.soTimeout = soTimeout; + } + + public int getTimeout() { + return timeout; + } + + public void setTimeout(int timeout) { + this.timeout = timeout; + } + + public String getHost () { + return host; + } + + public void setHost (String host) { + this.host = host; + } + + public int getNumTransportChannels () { + return numTransportChannels; + } + + public void setNumTransportChannels (int numChannels) { + numTransportChannels = numChannels; + } + + public int getPort () { + return port; + } + + public void setPort (int port) { + this.port = port; + } + + public void setSslContext(SSLContext sslContext) { + this.sslContext = sslContext; + } + + public long getServerStartTime() { + return serverTime; + } + + public int getReconnectInterval() { + return reconnectInterval; + } + + /** + * Checks if client is connected. + * Will not wait but will return true even if reconnecting and there is no immediately available transports. + * + *

It returns true during reconnecting phase because it would be inconsistent to return false, + * considering that reconnecting state does not trigger "disconnected" event. + * + *

In most cases you should use {@link #tryGetConnectionStatus()} instead. + * + * @return true if connected or reconnecting, false otherwise + */ + public boolean isConnected() { + return dispatcher != null && dispatcher.isConnectedOrReconnecting(); + } + + /** + * Checks if client is fully connected right now. + * Will not wait and will return false if reconnecting. + * + * @return true if connected and NOT reconnecting, false otherwise + */ + public boolean isConnectedAndNotReconnecting() { + return dispatcher != null && dispatcher.isConnectedAndNotReconnecting(); + } + + /** + * Return true, if it has CONNECTED state. + * Return false, if it has DISCONNECTED/DISCONNECTING state. + * Otherwise, waits until status gets CONNECTED or DISCONNECTED. + * + * @return true if connected, false if disconnected + */ + public boolean tryGetConnectionStatus() { + if (dispatcher == null) { + return false; + } else { + return dispatcher.tryGetConnectionStatus(); + } + } + + public void connect () throws IOException { + if (dispatcher != null) + throw new IllegalStateException("Already connected"); + + VSocket socket = openTransport(); // try to connect + + synchronized (dispatcherLock) { + contextContainer.getQuickExecutor().reuseInstance(); + + dispatcher = new VSDispatcher(clientId, true, contextContainer); + dispatcher.setLingerInterval(reconnectInterval); + dispatcher.addDisposableListener(this); + } + dispatcher.addTransportChannel (socket); + + for (int ii = 1; ii < numTransportChannels; ii++) + dispatcher.addTransportChannel (openTransport()); + + dispatcher.setStateListener(this); + } + + public VSDispatcher getDispatcher() { + return dispatcher; + } + + public void increaseNumTransportChannels() throws IOException { + numTransportChannels++; + + if (dispatcher != null) + dispatcher.addTransportChannel (openTransport ()); + } + + @NotNull + private Socket setupSocket() throws IOException { + // If SSL termination is enabled, then we start with SSL socket and will NOT try to perform upgrade. + // This is necessary because intermediate proxies will get confused + // if we start with non-SSL socket and then upgrade to SSL after negotiation with TB. + boolean startWithSSL = enableSSL && sslTermination; + + InetSocketAddress socketAddress = new InetSocketAddress(host, port); + + Socket socket = null; + boolean success = false; + try { + if (startWithSSL) { + VSProtocol.LOGGER.info("SSL termination enabled: creating SSL socket on [" + host + ":" + port + "]"); + socket = sslContext.getSocketFactory().createSocket(); + } else { + socket = new Socket(); + } + + socket.setSoTimeout(soTimeout); + socket.setTcpNoDelay(true); + + // Sets socket buffer sizes. + // Please note that later socket also will be additionally configured in VSocketImpl.setUpSocket() method. + // However, that happens only after socket gets connected. + // It's important to configure receive buffer size before connection is established + // to allow it to use TCP window size greater than 64kb. + // That's why we have to do that here. + VSocketImpl.configureBufferSizes(socket); + + // Connect + socket.connect(socketAddress, timeout); + + InputStream is = socket.getInputStream(); + OutputStream os = socket.getOutputStream(); + + // We should not request SSL from TB server if SSL termination is enabled + boolean requestSSL = enableSSL && !sslTermination; + + os.write(0); //first byte of VS protocol + os.write(VSProtocol.getHeader(requestSSL)); + os.flush(); + + int serverResponse = is.read(); + if (serverResponse == VSProtocol.CONN_RESP_SSL_NOT_SUPPORTED) { + assert !startWithSSL; + throw new IOException("Server not supported SSL."); + } else if (serverResponse != VSProtocol.CONN_RESP_OK) { + throw new RuntimeException("Unexpected server response: " + serverResponse); + } + + int serverHeader = is.read(); + if (serverHeader == VSProtocol.SSL_HEADER) { + + if (startWithSSL) { + throw new IllegalStateException("SSL termination is enabled but server attempts to upgrade to SSL"); + } + + // Upgrade non-SSL socket to SSL + socket = sslContext.getSocketFactory().createSocket( + socket, socket.getInetAddress().getHostAddress(), socket.getPort(), true); + ((SSLSocket) socket).setUseClientMode(true); + ((SSLSocket) socket).startHandshake(); + enableSSL = true; // We now use SSL socket, even if client have not requested it. + VSProtocol.LOGGER.info("Socket upgraded to SSL socket! Now connection is secured."); + } else if (serverHeader == VSProtocol.HEADER) { + if (enableSSL && !startWithSSL) { + // Normally we should not get here: + // 1. If SSL termination is enabled, then we don't request SSL from TB server + // 2. If we have enableSSL = true, then we should request it from TB server and get explicit reject if it's not supported + VSProtocol.LOGGER.info("Connection isn't secured."); + enableSSL = false; + } + } else { + throw new RuntimeException("Unexpected server header: " + serverHeader); + } + + success = true; + return socket; + } finally { + if (!success) { + IOUtil.close(socket); + } + } + } + + public void setProtocolVersion(int version) { + if (version < VSClient.MIN_ALLOWED_SERVER_VERSION || version > VSClient.MAX_COMP_SERVER_VERSION) + throw new IllegalArgumentException("Protocol version should be in range [" + + VSClient.MIN_ALLOWED_SERVER_VERSION + ", " + VSClient.MAX_COMP_SERVER_VERSION + "]"); + + this.protocolVersion = version; + } + + /** Used for re-connecting existing VSocket */ + @SuppressFBWarnings(value = "UNENCRYPTED_SOCKET", justification = "Timebase ports should be protected from public access by SSL-terminating NLB") + private VSocket openTransport (VSocket stopped) throws IOException, TransportRecoveryFailre { + Socket s = null; + boolean ok = false; + + try { + s = setupSocket(); + + ClientConnection cc = new ClientConnection(s); + + DataOutputStream dos = new DataOutputStream (cc.getOutputStream()); + DataInputStream dis = new DataInputStream (cc.getBufferedInputStream()); + + //check version compatibility + dos.writeInt (protocolVersion); + dos.writeUTF(clientId); + dos.flush(); + + int spv = dis.readInt (); + + String sid = dis.readUTF (); + if (spv != protocolVersion && (spv < MIN_COMP_SERVER_VERSION || spv > MAX_COMP_SERVER_VERSION)) + throw new IncompatibleClientException (sid, spv); + + sslPort = dis.readInt(); + + if (protocolVersion > 1014) + processTransportHandshake(dis); + + //send other sync data + dos.writeBoolean(false); // restore + dos.writeInt (stopped.getCode()); // socket id + dos.writeLong(stopped.getInputStream().getBytesRead()); // number of read bytes + dos.flush (); + + s.setSoTimeout(0); + + int resp = dis.readByte (); + + if (resp != VSProtocol.CONN_RESP_OK) { + throw new ConnectionRejectedException (sid, resp); + } else { + long time = dis.readLong(); + if (serverTime != -1 && serverTime != time) + throw new ServerRestartedException(sid, time); + else + serverTime = time; + + this.reconnectInterval = dis.readInt(); + + String compression = dis.readUTF(); + this.serverCompression = Enum.valueOf(VSCompression.class, compression); + + long numBytesRecieved = dis.readLong(); // number of bytes recieved by remote side + ok = numBytesRecieved != -1; + + if (ok) { + stopped.getOutputStream().confirm(numBytesRecieved); + return VSocketFactory.get(cc, stopped); + } else { + // TODO: We have to close client here + if (VSProtocol.LOGGER.isLoggable(Level.WARNING)) { + boolean hadUnconfirmedData = stopped.getOutputStream().hasUnconfirmedData(); + String transportTag = stopped.getCode() + " / " + Integer.toHexString(stopped.getCode()); + VSProtocol.LOGGER.warning("Failed to restore transport " + transportTag + ". Data loss: " + (hadUnconfirmedData ? "yes" : "uncertain")); + } + throw new TransportRecoveryFailre(); + } + } + + } finally { + if (!ok) + IOUtil.close (s); + } + } + + @SuppressFBWarnings(value = "UNENCRYPTED_SOCKET", justification = "Timebase ports should be protected from public access by SSL-terminating NLB") + VSocket openTransport () throws IOException { + Socket s = null; + boolean ok = false; + TransportType transportType; + ClientConnection cc; + + try { + s = setupSocket(); + cc = new ClientConnection(s); + + DataOutputStream dos = new DataOutputStream (cc.getOutputStream()); + DataInputStream dis = new DataInputStream (cc.getBufferedInputStream()); + + //check version compatibility + dos.writeInt (protocolVersion); + dos.writeUTF(clientId); + dos.flush (); + int spv = dis.readInt (); + String sid = dis.readUTF (); + + if (spv != protocolVersion && (spv < MIN_COMP_SERVER_VERSION || spv > MAX_COMP_SERVER_VERSION)) + throw new IncompatibleClientException (sid, spv); + + sslPort = dis.readInt(); + + transportType = (protocolVersion > 1014) ? processTransportHandshake(dis) : TransportType.SOCKET_TCP; + + //send other sync data + dos.writeBoolean(true); + dos.writeInt(s.hashCode()); // socket id + dos.writeLong(0L); // number of read bytes + dos.flush (); + + s.setSoTimeout (0); + + int resp = dis.readByte (); + + if (resp != VSProtocol.CONN_RESP_OK) { + throw new ConnectionRejectedException (sid, resp); + } else { + long time = dis.readLong(); + if (serverTime != -1 && serverTime != time) + throw new ServerRestartedException(sid, time); + else + serverTime = time; + + this.reconnectInterval = dis.readInt(); + + String compression = dis.readUTF(); + long numBytesRecieved = dis.readLong(); + assert numBytesRecieved == 0; // new connections should have = 0; + this.serverCompression = Enum.valueOf(VSCompression.class, compression); + } + + ok = true; + } finally { + if (!ok) + IOUtil.close (s); + } + + return VSocketFactory.get(cc, transportType); + } + + private TransportType processTransportHandshake(DataInputStream dis) throws IOException { + TransportType transportType = TransportType.values()[dis.readInt()]; + if (transportType == TransportType.AERON_IPC) { + throw new RuntimeException("Legacy version of Aeron IPC is not supported"); + } else if (transportType == TransportType.OFFHEAP_IPC) { + OffHeap.start(dis.readUTF(), false); + } + + return transportType; + } + + public VSChannel openChannel () throws IOException { + return openChannel(VSProtocol.CHANNEL_BUFFER_SIZE, VSProtocol.CHANNEL_BUFFER_SIZE, false); + } + + public VSChannel openChannel (int inCapacity, int outCapacity, boolean compressed) throws IOException { + if (serverCompression == VSCompression.OFF) + compressed = false; + else if (serverCompression == VSCompression.ON) + compressed = true; + + VSChannelImpl vsc = dispatcher.newChannel (inCapacity, outCapacity, compressed); + + try { + vsc.sendConnect (); + } catch (InterruptedException x) { + Thread.currentThread().interrupt(); + throw new InterruptedIOException (); + } + + return (vsc); + } + + @Override + public void close () { + close(true); + } + + private void close(boolean waitForChannelsToFinish) { + boolean triggerDisconnectEvent; + synchronized (dispatcherLock) { + closed = true; + + VSDispatcher d = dispatcher; + + // If dispatcher is null, then we already disconnected or even never were connected. + // If dispatcher is in shutdown state, then disconnect event already was triggered. + // Note that this check does not give 100% guarantee that disconnect event will be triggered no more than once + // because of race between checking isShutdownState() and calling d.setStateListener(null). + // However, in practice this should be sufficient. + triggerDisconnectEvent = d != null && !d.isShutdownState(); + + if (d != null) { + d.setStateListener(null); + d.removeDisposableListener(this); + d.close(waitForChannelsToFinish); + + contextContainer.getQuickExecutor().shutdownInstance(); + } + + dispatcher = null; + } + // Trigger a disconnect event, so any disconnect listeners can be notified. + // https://gitlab.deltixhub.com/Deltix/QuantServer/QuantServer/-/issues/1269 + // However, this also results that onDisconnect event will be triggered even if no "unexpected disconnect" actually happened. + // So while VSDispatcher does not trigger disconnect event if it shut down gracefully, VSClient.close() will still trigger it. + // TODO: Decide if we want to call .onDisconnected() in case of normal shutdown. + // TODO: This should be reviewed after TickDBClient refactor. We may want to completely remove this call + // as updated VSDispatcher already triggers disconnect event on unexpected disconnects + // and state change that is caused by TickDBClient closing the connection may be handled in TickDBClient itself. + if (triggerDisconnectEvent) { + DisconnectEventListener listenerRef = listener; + if (listenerRef != null) { + listenerRef.onDisconnected(); + } + } + } + + public void setDisconnectedListener(DisconnectEventListener listener) { + this.listener = listener; + } + + @Override + boolean onTransportRecoveryStart(VSocketRecoveryInfo recoveryInfo) { + if (dispatcher != null) { + // TODO: Ensure that we can't get duplicate instance of socket in the broken list + broken.addFirst(recoveryInfo); + + reconnector.submit(); + return false; + } else { + return true; + } + } + + @Override + boolean onTransportRecoveryStop(VSocketRecoveryInfo recoveryInfo) { + try { + synchronized (recoveryInfo) { + while (recoveryInfo.isRecoveryAttemptInProgress()) { + recoveryInfo.wait(); + } + if (!recoveryInfo.isRecoveryEnded()) { + recoveryInfo.markRecoveryFailed(); + } + boolean removed = broken.remove(recoveryInfo); + if (!removed && !recoveryInfo.isRecoverySucceeded()) { + VSProtocol.LOGGER.warning("Transport for " + recoveryInfo.getSocket().getCode() + " is missing from broken transport list"); + } + + boolean recoveryFailed = recoveryInfo.isRecoveryFailed(); + if (recoveryFailed) { + if (VSProtocol.LOGGER.isLoggable(Level.FINE)) { + VSProtocol.LOGGER.fine("Transport for " + recoveryInfo.getSocket().getCode() + " was permanently lost"); + } + } + return recoveryFailed || (!removed && !recoveryInfo.isRecoverySucceeded()); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return true; + } + } + + @Override + void onConnected() { + if (listener != null) + listener.onReconnected(); + } + + @Override + void onDisconnected() { + reconnector.kill(); + + if (listener != null) + listener.onDisconnected(); + } + + public long getLatency() { + return dispatcher != null ? dispatcher.getLatency() : Long.MAX_VALUE; + } + + @Override + public void disposed (VSDispatcher d) { + synchronized (dispatcherLock) { + closed = true; + + if (d == dispatcher) { + d.setStateListener(null); + d.removeDisposableListener(this); + contextContainer.getQuickExecutor().shutdownInstance(); + + dispatcher = null; + } + } + } + + public boolean isSSLEnabled() { + return enableSSL; + } + + public int getSSLPort() { + return sslPort; + } + + @Override + public String toString () { + return ("VSClient (" + host + ":" + port + ")"); + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/VSCompression.java b/util/src/main/java/com/epam/deltix/util/vsocket/VSCompression.java new file mode 100644 index 00000000..7cb54ab4 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/VSCompression.java @@ -0,0 +1,24 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +/** + * + */ +public enum VSCompression { + ON, OFF, AUTO +} \ No newline at end of file diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/VSConnectionListener.java b/util/src/main/java/com/epam/deltix/util/vsocket/VSConnectionListener.java new file mode 100644 index 00000000..fd0c9504 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/VSConnectionListener.java @@ -0,0 +1,31 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.concurrent.QuickExecutor; + +import java.io.IOException; + +/** + * + */ +public interface VSConnectionListener { + public void connectionAccepted ( + QuickExecutor executor, + VSChannel serverChannel + ) throws IOException; +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/VSDispatcher.java b/util/src/main/java/com/epam/deltix/util/vsocket/VSDispatcher.java new file mode 100644 index 00000000..1a7fd3ea --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/VSDispatcher.java @@ -0,0 +1,970 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.concurrent.ContextContainer; +import com.epam.deltix.util.concurrent.QuickExecutor; +import com.epam.deltix.thread.affinity.AffinityThreadFactoryBuilder; +import com.epam.deltix.util.annotations.TimestampMs; +import com.epam.deltix.util.collections.generated.ObjectHashSet; +import com.epam.deltix.util.lang.Disposable; +import com.epam.deltix.util.lang.DisposableListener; +import com.epam.deltix.util.lang.Util; +import com.epam.deltix.util.memory.DataExchangeUtils; +import com.epam.deltix.util.time.TimeKeeper; +import com.epam.deltix.util.time.TimerRunner; +import net.jcip.annotations.GuardedBy; +import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.VisibleForTesting; + +import java.io.EOFException; +import java.io.IOException; +import java.net.SocketException; +import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Stack; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Phaser; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Level; + +/** + * + */ +public final class VSDispatcher implements Disposable { + + private final ContextContainer contextContainer; + private final ThreadFactory transportChannelThreadFactory; + + + private final Date creationDate = new Date (); + private Timer timer; + + @GuardedBy("transportChannels") + private final ObjectHashSet transportChannels = + new ObjectHashSet<> (); + + /** + * Set to non-null value during transport channel recovery. + * If multiple transport channels are being recovered, this future will be completed with value "true" if all of + * them are recovered successfully, or "false" if at least one of them failed to recover. + */ + + @GuardedBy("freeChannels") + // TODO: Replace by Deque + private final Stack freeChannels = + new Stack <> (); + + private final ArrayList channels = + new ArrayList <> (10); + //private volatile boolean hasAvailableTransport = false; + + volatile VSConnectionListener connectionListener = null; + volatile ConnectionStateListener stateListener; + + private int activeChannels; + private int lingerInterval; // How long Dispatcher will wait for reconnection to happen + + private String address; + private final String clientAddress; + private String applicationID; + + // State of Dispatcher on remote side + private volatile boolean remoteConnected = true; + private volatile long throughput = 0; + private volatile long totalBytes = 0; // number of bytes sent + private final EMA average = new EMA(1000 * 60); // 1 minute + + /** + * State transitions: + *

    + *
  • INITIAL -> CONNECTED : when first transport channel is added + *
  • CONNECTED -> CONNECTING : when at least one transport channel is lost + *
  • CONNECTING -> CONNECTED : when ALL transport channels are recovered + *
  • CONNECTING -> DISCONNECTING : when recovery of at least one transport channel fails (explicitly or because of timeout) + *
  • DISCONNECTING -> DISCONNECTED : when all transport channels are closed after failed recovery + *
+ */ + private final AtomicReference state = new AtomicReference<>(VSDispatcherState.INITIAL); + + // Latch that gets counted down when dispatcher is fully closed + private final CountDownLatch closeLatch = new CountDownLatch(1); + + // Works both as a counter of recovering transport channels + // and as a barrier to wait until all recovering transports finish recovering. + private final Phaser recoveringTransports = new Phaser() { + @Override + protected boolean onAdvance(int phase, int registeredParties) { + return false; // never terminate automatically + } + }; + + + private TimerTask flusher = new TimerRunner() { + private VSChannelImpl[] list = new VSChannelImpl[10]; + private VSTransportChannel[] transports = new VSTransportChannel[5]; + private long runs = 0; + + @Override + protected void runInternal() { + + int size; + synchronized (channels) { + if ((size = channels.size()) > 0) + list = channels.toArray(list); + } + + for (int i = 0; i < size; i++) { + VSChannelImpl channel = list[i]; + try { + if (channel != null && channel.isAutoflush()) { + // For low latency channels (noDelay==true) we do not want to flush all + // the accumulated data at once because the remaining data will be sent + // by ChannelExecutor shortly. This allows to get more steady rate. + // For regular channels (noDelay==false) we want to send all the data + // (there is nobody else to do that). + channel.getOutputStream().flushAvailable(!channel.getNoDelay()); + } + } catch (ChannelClosedException e) { + // ignore + } catch (ConnectionAbortedException e) { + VSProtocol.LOGGER.log (Level.WARNING, "Client unexpectedly drop connection. Remote address: " + channel.getRemoteAddress()); + } catch (Exception e) { + VSProtocol.LOGGER.log (Level.WARNING, "Exception while flushing data. Remote address: " + channel.getRemoteAddress(), e); + } + } + + try { + // do keep-alive assuming that this task runs every millisecond + if (runs++ % VSProtocol.KEEP_ALIVE_INTERVAL == 0) { + synchronized (transportChannels) { + transports = transportChannels.toArray(transports); + size = transportChannels.size(); + } + + long bytes = 0; + for (int i = 0; i < size; i++) { + VSTransportChannel transport = transports[i]; + transport.keepAlive(); + bytes += transport.socket.getOutputStream().getBytesWritten(); + bytes += transport.socket.getInputStream().getBytesRead(); + } + + throughput = (bytes - totalBytes) / VSProtocol.KEEP_ALIVE_INTERVAL * 1000; + average.register(throughput); + totalBytes = bytes; + } + } catch (Exception ex) { + VSProtocol.LOGGER.log (Level.WARNING, "Exception while sending transport keep-alive.", ex); + } + } + }; + + private final String clientId; + private final boolean isClient; + private volatile int index = 0; + + private final HashSet> listeners = new HashSet<> (); + + /** + * Constructs a dispatcher instance for the specified client. + */ + public VSDispatcher(String clientId, boolean isClient, ContextContainer contextContainer) { + this.clientId = clientId; + + String[] parts = clientId.split(":"); + this.clientAddress = parts.length > 1 ? parts[0] : null; + + this.isClient = isClient; + this.contextContainer = contextContainer; + + timer = new Timer ("Flush Timer (" + this + ")", true); + timer.scheduleAtFixedRate (flusher, 1L, 1L); + + this.transportChannelThreadFactory = new AffinityThreadFactoryBuilder(contextContainer.getAffinityConfig()) + .setNameFormat("VSDispatcher(" + clientId + ") Transport %d") + .setPriority(Thread.MAX_PRIORITY) + .setDaemon(true) + .build(); + } + + /** + * Return current peak throughput (bytes per second) + * @return number of bytes per second + */ + public long getThroughput() { + return throughput; + } + + /** + * Return average throughput (bytes per second) + * @return number of bytes per second + */ + public double getAverageThroughput() { + return average.getAverage(); + } + + public int getReconnectInterval() { + return lingerInterval; + } + + public String getApplicationID() { + if (applicationID == null) { + applicationID = getApplicationID(clientId); + } + return applicationID; + } + + public static String getApplicationID(String clientId) { + String[] parts = clientId.split(":"); + return parts.length == 4 ? parts[2] : ""; + } + + public void setApplicationID(String applicationID) { + this.applicationID = applicationID; + } + + public void setLingerInterval(int reconnectInterval) { + this.lingerInterval = reconnectInterval; + } + + public String getClientId () { + return clientId; + } + + public Date getCreationDate () { + return (creationDate); + } + + public int getNumTransportChannels () { + synchronized (transportChannels) { + return (transportChannels.size ()); + } + } + + public boolean hasTransportChannels() { + synchronized (transportChannels) { + return !transportChannels.isEmpty(); + } + } + + public boolean isConnectedOrReconnecting() { + VSDispatcherState value = state.get(); + return value == VSDispatcherState.CONNECTED || value == VSDispatcherState.RECONNECTING; + } + + public boolean isConnectedAndNotReconnecting() { + VSDispatcherState value = state.get(); + return value == VSDispatcherState.CONNECTED; + } + + public void addTransportChannel (VSocket socket) + throws IOException + { + VSDispatcherState currentState = state.get(); + if (currentState == VSDispatcherState.DISCONNECTING || currentState == VSDispatcherState.DISCONNECTED) { + VSProtocol.LOGGER.log (Level.WARNING, "Attempt to add transport channel while dispatcher is disconnecting. Remote address: " + socket.getRemoteAddress() + ". Dispatcher: " + this); + } + + VSTransportChannel tc = new VSTransportChannel(this, socket, transportChannelThreadFactory); + tc.checkedOut = true; // Initially this channel is not in "freeChannels" so it is effectively "checked out" + + // Is that fist transport channel? + boolean fistConnected = state.compareAndSet(VSDispatcherState.INITIAL, VSDispatcherState.CONNECTED); + + // start transport + tc.start (); + + synchronized (transportChannels) { + + if (address == null) + address = socket.getRemoteAddress(); + + transportChannels.add (tc); + transportChannels.notify(); + } + + checkIn(tc); + + // This may be triggered only once per dispatcher lifetime + if (fistConnected && stateListener != null) { + stateListener.onConnected(); + } + } + + public void setConnectionListener (VSConnectionListener connectionListener) { + this.connectionListener = connectionListener; + } + + void setStateListener(ConnectionStateListener stateListener) { + this.stateListener = stateListener; + } + + @VisibleForTesting + VSDispatcherState getInternalState() { + return state.get(); + } + + /** + * Executed in the context of transport channel thread (VSTransportChannel.run() method) when error occurs on transport. + * + *

Corresponding transport channel will be closed after this method returns. + * + *

This method is expected to block until logical transport gets recovered or declared unrecoverably broken. + * In case of recovery failure, expected to trigger dispatcher shutdown, as loss of single transport channel + * means loss of data and inconsistent state for client and server. + * + *

Multiple transport channels may be lost concurrently, so this method may be executed concurrently. + * In that case, threads may compete for changing dispatcher state. + */ + void transportStopped (VSTransportChannel channel, Throwable ex) { + if (!(ex instanceof Exception)) { + // This means major failure, possibly OOM or other serious error. + VSProtocol.LOGGER.log(Level.SEVERE, "Critical error on transport channel. Remote address: " + channel.socket.getRemoteAddress() + ". Dispatcher: " + this, ex); + // Just close dispatcher right away + close(); + return; + } + + Level disconnectLogLevel = ex instanceof EOFException ? Level.FINE : Level.INFO; + if (VSProtocol.LOGGER.isLoggable(disconnectLogLevel)) { + VSProtocol.LOGGER.log(disconnectLogLevel, "Transport channel has stopped. Remote address: " + channel.socket.getRemoteAddress() + ". Error: " + ex.getClass().getSimpleName() + ". Message: " + ex.getMessage()); + } + + long startTime = System.currentTimeMillis(); + long endTime = startTime + lingerInterval; + + boolean registered = false; // true if we have registered this transport in "recoveringTransports" phaser + boolean wasCheckedIn; + + try { + synchronized (transportChannels) { + VSDispatcherState currentState = state.get(); + if (currentState == VSDispatcherState.DISCONNECTING || currentState == VSDispatcherState.DISCONNECTED) { + // Dispatcher is already closing or closed, no need to recover transport + return; + } + + if (!transportChannels.remove(channel)) // check if that channel is already removed + return; + + // From this point we consider that we are recovering this transport channel. + + // Counter incremented before state change, + // so that should be impossible to see CONNECTING with 0 recovering transports and still pending recovery attempt. + recoveringTransports.register(); + registered = true; + state.compareAndSet(VSDispatcherState.CONNECTED, VSDispatcherState.RECONNECTING); + + synchronized (freeChannels) { + wasCheckedIn = freeChannels.remove(channel); + assert wasCheckedIn == !channel.checkedOut; + freeChannels.notifyAll(); + } + + if (!wasCheckedIn) { + // Try to wait for the channel to become checked in + wasCheckedIn = waitForTransportCheckIn(channel, startTime, endTime); + if (!wasCheckedIn) { + if (VSProtocol.LOGGER.isLoggable(Level.INFO)) { + VSProtocol.LOGGER.log(Level.INFO, "Error waiting to reconnect (transport was not checked in)."); + } + } + } + } + ConnectionStateListener stateListener = this.stateListener; + + boolean transportIsUnrecoverablyBroken = false; + try { + // trying to recover transport + VSocketRecoveryInfo recoveryInfo = new VSocketRecoveryInfo(channel.socket, endTime); + + long now = System.currentTimeMillis(); + if (wasCheckedIn && (now < endTime) && !isShutdownState()) { + + if (stateListener != null) { + if (stateListener.onTransportRecoveryStart(recoveryInfo)) { + transportIsUnrecoverablyBroken = true; + } + } + + if (remoteConnected && !transportIsUnrecoverablyBroken) { + // System.out.println("WAITED: remoteConnected=" + remoteConnected + " transportIsUnrecoverablyBroken=" + transportIsUnrecoverablyBroken); + try { + // We loop here waiting for recovery to complete or timeout to expire or dispatcher to be closed. + synchronized (recoveryInfo) { + long timeToWait; + while ((timeToWait = endTime - now) > 0 && recoveryInfo.isWaitingForRecovery() && remoteConnected && !isShutdownState()) { + recoveryInfo.wait(timeToWait); + if (recoveryInfo.isWaitingForRecovery() && remoteConnected) { + now = System.currentTimeMillis(); + } + } + if (recoveryInfo.isRecoveryFailed()) { + transportIsUnrecoverablyBroken = true; + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + if (VSProtocol.LOGGER.isLoggable(Level.FINE)) + VSProtocol.LOGGER.log(Level.FINE, "Error waiting to reconnect.", e); + } + } else { + //System.out.println("NOT WAITED: remoteConnected=" + remoteConnected + " transportIsUnrecoverablyBroken=" + transportIsUnrecoverablyBroken); + } + } else { + transportIsUnrecoverablyBroken = true; + if (VSProtocol.LOGGER.isLoggable(Level.FINE)) { + VSProtocol.LOGGER.log(Level.FINE, "Cancelled recovery of failed connection because of timeout on waiting for check-in from other thread. Remote address: " + getRemoteAddress()); + } + } + + // In general, VSDispatcher don't have to shut down if transport recovery fails, + // because other transport channels may remain functional. + // However, in our current design, loss of single transport channel means loss of data + // and inconsistent state for client and server, so we have to shut down the dispatcher. + // The decision to shut down the dispatcher is delegated to the state listener. + if (stateListener != null) { + if (stateListener.onTransportRecoveryStop(recoveryInfo)) { + transportIsUnrecoverablyBroken = true; + } + } + } finally { + if (transportIsUnrecoverablyBroken || isShutdownState()) { + // We lost this transport channel and were unable to recover it (because of explicit error, timeout or triggered shutdown state). + // This means it is not possible to recover from this state, and we have to properly close the dispatcher. + // We need to close all remaining connections and explicitly notify user about that. + + // Record a copy of state listener reference before updating state because it may be changed concurrently. + // If save "stateListener" before state update, then we can be sure that + // if we had non-null listener before state update, then we will have non-null listener for thread that gets "triggerDisconnectedEvent". + var savedStateListener = this.stateListener; + + // Try to set state to DISCONNECTING, before decrementing recoveringTransports counter, + // so other thread will not switch into CONNECTED state if this was the last recovering transport. + + VSDispatcherState stateBeforeUpdate = state.getAndUpdate(prevState -> { + switch (prevState) { + case CONNECTED: + // Should not happen + return VSDispatcherState.DISCONNECTING; + case RECONNECTING: + return VSDispatcherState.DISCONNECTING; + case DISCONNECTING: + return prevState; // remain in DISCONNECTING + case DISCONNECTED: + return prevState; // remain in DISCONNECTED + default: + throw new IllegalStateException("Unexpected dispatcher state: " + prevState); + } + }); + // Disconnected event should be triggered only if we changed the state. + // So that event should be triggered only once per dispatcher lifetime. + // Also, it disables trigger of onDisconnected event if dispatcher is closed normally via direct call to close(). + boolean triggerDisconnectedEvent = stateBeforeUpdate == VSDispatcherState.CONNECTED || stateBeforeUpdate == VSDispatcherState.RECONNECTING; + + registered = false; + recoveringTransports.arriveAndDeregister(); + + processChannelRecoveryFailure(ex, triggerDisconnectedEvent, savedStateListener); + } else { + // Successfully recovered this transport channel + registered = false; + recoveringTransports.arriveAndDeregister(); + int remaining = recoveringTransports.getUnarrivedParties(); + if (!recoveringTransports.isTerminated() && remaining == 0) { + // All lost transport channels are recovered, try to update state to CONNECTED + state.getAndUpdate(prev -> { + if (prev == VSDispatcherState.RECONNECTING) { + return VSDispatcherState.CONNECTED; + } else { + return prev; // Keep state unchanged + } + }); + } + } + } + + } finally { + if (registered) { + // In case of any unexpected error, ensure that we release the counter + recoveringTransports.arriveAndDeregister(); + } + } + } + + /** + * Triggered when transport channel recovery has failed and dispatcher must be disconnected. + * @param triggerClose if true, then current thread is the one that first detected unrecoverable transport failure and responsible for shutdown + */ + private void processChannelRecoveryFailure(Throwable ex, boolean triggerClose, @Nullable ConnectionStateListener stateListenerCopy) { + boolean wasConnected = remoteConnected; + + // notify all waiting for transport that connection is lost + onRemoteClosed(); + + if (ex instanceof SocketException || ex instanceof EOFException || ex instanceof SocketTimeoutException) { + if (VSProtocol.LOGGER.isLoggable(Level.FINE)) + VSProtocol.LOGGER.log(Level.FINE, "Exception on transport channel. Remote address: " + getRemoteAddress(), ex); + } else { + VSProtocol.LOGGER.log(Level.SEVERE, "Exception on transport channel. Remote address: " + getRemoteAddress(), ex); + } + + if (wasConnected) { + VSProtocol.LOGGER.log(Level.WARNING, "Disconnecting due to unrecoverable transport channel loss. Remote address: " + getRemoteAddress(), ex); + } else { + VSProtocol.LOGGER.log(Level.FINER, "Disconnecting (re-triggered) due to unrecoverable transport channel loss. Remote address: " + getRemoteAddress(), ex); + } + + // and then notify all channels that we lost transport + IOException iex = ex instanceof IOException ? (IOException)ex : null; + synchronized (channels) { + for (VSChannelImpl vsChannel : channels) { + if (vsChannel != null) { + // TODO: Review. Calling onDisconnected while holding "channels" lock may lead to deadlocks + vsChannel.onDisconnected(iex); + } + } + } + + if (triggerClose) { + // notify state listener that connections lost + if (stateListenerCopy != null) { + if (VSProtocol.LOGGER.isLoggable(Level.FINER)) { + VSProtocol.LOGGER.log(Level.FINER, "Notifying state listener about disconnection. Remote address: " + getRemoteAddress()); + } + stateListenerCopy.onDisconnected(); + } + + close(); + } else { + // If this thread is not responsible for closing dispatcher, + // just wait until dispatcher gets closed by other thread. + boolean success; + try { + success = closeLatch.await(lingerInterval + 1_000, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Failed waiting for dispatcher to close after transport recovery failure.", e); + } + if (!success) { + VSProtocol.LOGGER.log(Level.WARNING, "Timeout waiting for dispatcher to close after transport recovery failure. Remote address: " + getRemoteAddress()); + // No other thread closed the dispatcher in a timely manner, close it ourselves. Even if it may break order between .onDisconnected() and .disposed() events. + close(); + } + } + } + + boolean isShutdownState() { + VSDispatcherState value = state.get(); + return value == VSDispatcherState.DISCONNECTING || value == VSDispatcherState.DISCONNECTED; + } + + /** + * Return true, if it has CONNECTED state. + * Return false, if it has INITIAL, DISCONNECTED or DISCONNECTING state. + * Otherwise, waits at least {@link #lingerInterval} until status gets CONNECTED or DISCONNECTED. + * + * @return true if connected, false if disconnected + */ + public boolean tryGetConnectionStatus() { + // Get current phase before checking state + int phase = recoveringTransports.getPhase(); + + // Read current status + switch (state.get()) { + case CONNECTED: + return true; + case INITIAL: + case DISCONNECTED: + case DISCONNECTING: + return false; + case RECONNECTING: + // Wait below + } + + // What for the phase to change + recoveringTransports.awaitAdvance(phase); + + // Check new status + switch (state.get()) { + case CONNECTED: + return true; + case INITIAL: + case DISCONNECTED: + case DISCONNECTING: + return false; + case RECONNECTING: + default: { + // Special case: we are still in reconnection state, even after phase advanced. + if (recoveringTransports.isTerminated()) { + return false; + } + // It may be possible that the waiting transport count was just decremented to zero + // but state was not updated yet. Check that. + return recoveringTransports.getUnarrivedParties() == 0; + } + } + } + + /** + * Waits for the specified channel to become checked in. + * + * @param channel channel to wait for + * @param now current time + * @param endTime completion deadline (will stop after this time even if channel still checked out) + * @return true if channel was checked in + */ + @GuardedBy("transportChannels") + private boolean waitForTransportCheckIn(VSTransportChannel channel, long now, long endTime) { + assert Thread.holdsLock(transportChannels); + + boolean checkedIn = false; + try { + while (now < endTime && !checkedIn && !isShutdownState()) { + transportChannels.wait(endTime - now); + now = System.currentTimeMillis(); + synchronized (freeChannels) { + checkedIn = !channel.checkedOut; + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + if (VSProtocol.LOGGER.isLoggable(Level.FINE)) { + VSProtocol.LOGGER.log(Level.FINE, "Error waiting to reconnect.", e); + } + } + return checkedIn; + } + + public void closeTransport() throws IOException, InterruptedException { + synchronized (transportChannels) { + Iterator iterator = transportChannels.iterator(); + + if (iterator.hasNext()) { + VSTransportChannel next = iterator.next(); + next.socket.close(); + } + } + } + + void checkIn (VSTransportChannel tc) { + synchronized (transportChannels) { + if (transportChannels.contains(tc)) { + synchronized (freeChannels) { + tc.checkedOut = false; + freeChannels.add (tc); + freeChannels.notify(); + } + } else { + // This was removed from dispatcher, possibly channel recovery in progress. + synchronized (freeChannels) { + tc.checkedOut = false; + } + // Notifies thread that waits in {@link #transportStopped(VSTransportChannel, Throwable)}. + transportChannels.notifyAll(); + VSProtocol.LOGGER.log (Level.INFO, "Adding closed channel - ignored."); + } + } + } + + VSTransportChannel checkOut () + throws InterruptedException, ConnectionAbortedException + { + synchronized (freeChannels) { + for (;;) { + if (isShutdownState() && !remoteConnected) { + throw new ConnectionAbortedException("Connection aborted from remote side [" + getRemoteAddress() + "]"); + } + + if (!freeChannels.isEmpty ()) { + VSTransportChannel channel = freeChannels.pop(); + channel.checkedOut = true; + return channel; + } + + freeChannels.wait (); + } + } + } + + private int getActiveChannels() { + synchronized (channels){ + return activeChannels; + } + } + + public void close(boolean wait) { + if (wait) { + if (getActiveChannels() > 0) + synchronized (channels) { + try { + channels.wait(VSProtocol.SHUTDOWN_TIMEOUT); + } catch (InterruptedException e) { + if (VSProtocol.LOGGER.isLoggable(Level.FINE)) + VSProtocol.LOGGER.log (Level.FINE, "Error waiting to shutdown", e); + } + } + } + + if (wait && getActiveChannels() > 0) { +// synchronized (channels) { +// for (VSChannelImpl channel : channels) { +// if (channel != null) +// System.out.println(channel); +// } +// } + VSProtocol.LOGGER.log(Level.INFO, "Disconnect by timeout having opened " + getActiveChannels() + " channels"); + } + + close(); + } + + void onRemoteClosed() { + remoteConnected = false; + + // notify all waiting threads in checkOut() + synchronized (freeChannels) { + freeChannels.notifyAll(); + } + } + + private void sendClosing() { + // TODO: In theory we can try to check if there are any free transport channels and try to use them to send + // the close message. But in practice, if we are not connected anymore, then we are not very likely to succeed. + if (!remoteConnected || state.get() != VSDispatcherState.CONNECTED) + return; + + VSTransportChannel channel = null; + try { + byte[] buffer = new byte[2]; + DataExchangeUtils.writeUnsignedShort(buffer, 0, VSProtocol.DISPATCHER_CLOSE); + channel = checkOut(); + channel.write(buffer); + } catch (Exception e) { + VSProtocol.LOGGER.log (Level.INFO, "Error sending dispatcher close"); + } finally { + if (channel != null) + checkIn (channel); + } + } + + @Override + public void close () { + + sendClosing(); + + // Change state to DISCONNECTING if it was not DISCONNECTED already. + // This state change disables triggering of stateListener.onDisconnected() on transport channel error. + // So normal dispatcher.close() will not trigger onDisconnected() event. + state.getAndUpdate(prevState -> { + if (prevState == VSDispatcherState.DISCONNECTED) { + return VSDispatcherState.DISCONNECTED; + } else { + return VSDispatcherState.DISCONNECTING; + } + }); + + synchronized (transportChannels) { + for (VSTransportChannel tc : transportChannels) + Util.close (tc); + + transportChannels.clear (); + transportChannels.notify(); + } + + remoteConnected = false; + + // disable free channels to prevent locking on code below + synchronized (freeChannels) { + freeChannels.clear (); + freeChannels.notifyAll (); + } + + // notify channels that no transport available + VSChannel[] virtualChannels = getVirtualChannels(); + for (VSChannel vsChannel : virtualChannels) + if (vsChannel != null) + ((VSChannelImpl)vsChannel).onDisconnected(null); + + synchronized (channels) { + channels.clear(); + } + + TimerTask task = flusher; + if (task != null) + task.cancel(); + flusher = null; // for GC + + Timer t = timer; + if (t != null) + t.cancel(); // stop timer thread + timer = null; // for GC + + VSDispatcherState prevState = state.getAndUpdate(x -> VSDispatcherState.DISCONNECTED); + if (prevState != VSDispatcherState.DISCONNECTED) { + // This may be triggered only once per dispatcher lifetime + notifyDisposedEventListeners(); + } + + recoveringTransports.forceTermination(); + + closeLatch.countDown(); + } + + VSChannelImpl newChannel (int inCapacity, int outCapacity, boolean compressed) { + VSChannelImpl vsc; + + if (!remoteConnected) { + throw new IllegalStateException("Attempt to create new channel after disconnect"); + } + + synchronized (channels) { + int localId = channels.indexOf (null); + boolean extend = localId < 0; + + if (extend) + localId = channels.size (); + + if (localId >= VSProtocol.LISTENER_ID) + throw new IllegalStateException ("Too many channels are open"); + + index += isClient ? -1 : 1; + vsc = new VSChannelImpl (this, inCapacity, outCapacity, compressed, localId, index, contextContainer); + + if (extend) { + channels.add (vsc); + } else { + assert (channels.get(localId) == null); + channels.set (localId, vsc); + } + + activeChannels++; + channels.notify(); + } + + return (vsc); + } + + public VSChannel [] getVirtualChannels () { + synchronized (channels) { + //noinspection ToArrayCallWithZeroLengthArrayArgument + return (channels.toArray (new VSChannel [channels.size ()])); + } + } + + public String getRemoteAddress() { + return address; + } + + public String getClientAddress() { + return "/" + clientAddress + ":"; + } + + VSChannelImpl getChannel (int id) { + synchronized (channels) { + if (id >= channels.size() ) { + // Client requested a channel that never existed + if (VSProtocol.LOGGER.isLoggable(Level.WARNING)) { + VSProtocol.LOGGER.warning("Attempt to use invalid channel id: " + id + ", max valid value: " + (channels.size() - 1)); + } + return null; + } + + return (channels.get (id)); + } + } + + long getLatency() { + VSTransportChannel tc = null; + try { + return (tc = checkOut()).getLatency(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return 0; + } catch (ConnectionAbortedException e) { + return 0; + } finally { + if (tc != null) + checkIn(tc); + } + } + + void channelClosed (VSChannelImpl vsc) { + int id = vsc.getLocalId(); + + synchronized (channels) { + if (vsc.equals(channels.get(id))) { + channels.set(id, null); + + activeChannels--; + channels.notify(); + } else { + VSProtocol.LOGGER.log(Level.SEVERE, "Trying to remove wrong channel."); + } + } + } + + public void addDisposableListener(DisposableListener listener) { + synchronized (listeners) { + listeners.add(listener); + } + } + + public void removeDisposableListener(DisposableListener listener) { + synchronized (listeners) { + listeners.remove(listener); + } + } + + @SuppressWarnings("unchecked") + private DisposableListener[] getListeners() { + DisposableListener[] list; + + synchronized (listeners) { + //noinspection ToArrayCallWithZeroLengthArrayArgument + list = listeners.toArray(new DisposableListener[listeners.size()]); + } + + return list; + } + + private void notifyDisposedEventListeners() { + DisposableListener[] list = getListeners(); + + for (var aList : list) { + aList.disposed(this); + } + } + + public QuickExecutor getQuickExecutor() { + return contextContainer.getQuickExecutor(); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "@" + Integer.toHexString(hashCode()) + " for clientId='" + clientId; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/VSDispatcherState.java b/util/src/main/java/com/epam/deltix/util/vsocket/VSDispatcherState.java new file mode 100644 index 00000000..5f92c0a2 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/VSDispatcherState.java @@ -0,0 +1,26 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.vsocket; + +enum VSDispatcherState { + INITIAL, // No connection attempt made yet + CONNECTED, // At least one connection established, no transports in "recovery" state + RECONNECTING, // At least one transport in "recovery" state, trying to reconnect + DISCONNECTING, // Disconnect process initiated, waiting for all shutdown-related actions to complete + DISCONNECTED // Can be set only at the end of VSDispatcher.close() method +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/VSOutputStream.java b/util/src/main/java/com/epam/deltix/util/vsocket/VSOutputStream.java new file mode 100644 index 00000000..a63ccb76 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/VSOutputStream.java @@ -0,0 +1,45 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * Date: Mar 29, 2010 + */ +public abstract class VSOutputStream extends OutputStream { + + public abstract void enableFlushing() throws IOException; + + /* + * Disables flushing internal buffer when it reach capacity. + * Buffer will be extended when new data is written until flashing will be enabled again + */ + + public abstract void disableFlushing(); + + /* + * Flushes portion of data that can recieved on the remote side immediately. + * + * @param flushAll attempts to flush all available data, even if it requires multiple send operations + * and blocking for longer time. + */ + public abstract int flushAvailable(boolean flushAll) throws IOException; + + //public abstract int available(); +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/VSProtocol.java b/util/src/main/java/com/epam/deltix/util/vsocket/VSProtocol.java new file mode 100644 index 00000000..2b6544e1 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/VSProtocol.java @@ -0,0 +1,74 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import java.util.logging.Logger; + +/** + * + */ +public class VSProtocol { + public static final Logger LOGGER = Logger.getLogger ("deltix.vsocket"); + + public static final int VERSION = 1015; + public static final int KEEP_ALIVE_INTERVAL = 1000; + + public static final int HEADER = 0xD1; + public static final int SSL_HEADER = 0xD2; + + public static final int CONN_RESP_OK = 0; + public static final int CONN_RESP_INCOMPATIBLE_CLIENT = 1; + public static final int CONN_RESP_SSL_NOT_SUPPORTED = 2; + public static final int CONN_RESP_CONNECTION_REJECTED = 3; + + static final int LISTENER_ID = 0xFFFF; + static final int KEEP_ALIVE = 0xFFFE; + static final int PING = 0xFFFD; + + static final int MAXSIZE = 0xFF00; // max packet size to send + static final int MINSIZE = 7; // min packet size to send + + static final int CONNECT_ACK = MAXSIZE + 1; + static final int CLOSING = MAXSIZE + 2; + static final int CLOSED = MAXSIZE + 3; + static final int BYTES_AVAILABLE_REPORT = MAXSIZE + 4; + static final int DISPATCHER_CLOSE = MAXSIZE + 5; + static final int BYTES_RECIEVED = MAXSIZE + 6; + //static final int BYTES_READ = MAXSIZE + 7; + + static final long SHUTDOWN_TIMEOUT = 5000; + static final long RECONNECT_TIMEOUT = 500; + public static final int LINGER_INTERVAL = 10000; + public static final int IDLE_TIME = 20000; + + public static final int CHANNEL_BUFFER_SIZE = 1 << 17; // optimized for local connections + public static final int CHANNEL_MAX_BUFFER_SIZE = 1 << 19; // optimized for remote connections + + public static int getIdleTime() { + + String delay = System.getProperty("VSProtocol.idleTime"); + try { + return delay != null ? Integer.parseInt(delay) : IDLE_TIME; + } catch (NumberFormatException e) { + return IDLE_TIME; + } + } + + public static int getHeader(boolean ssl) { + return ssl ? SSL_HEADER : HEADER; + } +} \ No newline at end of file diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/VSServer.java b/util/src/main/java/com/epam/deltix/util/vsocket/VSServer.java new file mode 100644 index 00000000..6826eccc --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/VSServer.java @@ -0,0 +1,151 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.concurrent.ContextContainer; +import com.epam.deltix.util.concurrent.QuickExecutor; +import com.epam.deltix.util.io.IOUtil; +import org.jetbrains.annotations.VisibleForTesting; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketException; +import java.util.logging.Level; + +/** + * + */ +public class VSServer extends Thread { + private final ContextContainer contextContainer; + private final VSServerFramework framework; + private ServerSocket serverSocket; + + public VSServer () throws IOException { + this (0); + } + + public VSServer (int port) throws IOException { + this(port, null, null, null); + } + + public VSServer(int port, TLSContext ctx) throws IOException { + this(port, null, ctx, null); + } + + public VSServer(int port, TransportProperties transportProperties) throws IOException { + this(port, null, null, transportProperties); + } + + public VSServer(int port, InetAddress address, TLSContext sslProperties, TransportProperties transportProperties) throws IOException { + this(VSServerSocketFactory.createServerSocket(port, address), sslProperties, transportProperties); + } + + public VSServer (ServerSocket serverSocket, TLSContext ctx, TransportProperties transportProperties) { + super ("VSServer on " + serverSocket); + // TODO: Probably we should get contextContainer from constructor parameters + this.contextContainer = new ContextContainer(); + this.contextContainer.setQuickExecutorName("VSServer Executor"); + this.serverSocket = serverSocket; + + contextContainer.getQuickExecutor().reuseInstance(); + this.framework = new VSServerFramework ( + contextContainer.getQuickExecutor(), VSProtocol.LINGER_INTERVAL, VSCompression.AUTO, contextContainer); + try { + if (ctx != null) + this.framework.initSSLSocketFactory(ctx); + } catch (Exception ex) { + VSProtocol.LOGGER.log(Level.WARNING, "Failed to init SSL", ex); + } + + this.framework.initTransport(transportProperties); + } + + public QuickExecutor getExecutor () { + return framework.getExecutor (); + } + + public int getLocalPort () { + return (serverSocket.getLocalPort ()); + } + + public void setConnectionListener (VSConnectionListener lnr) { + framework.setConnectionListener (lnr); + } + + public int getSoTimeout () throws IOException { + return serverSocket.getSoTimeout(); + } + + public void setSoTimeout (int readTimeout) throws SocketException { + serverSocket.setSoTimeout(readTimeout); + } + + @VisibleForTesting + public void setTransportsLimit(short transportsLimit) { + this.framework.setTransportsLimit(transportsLimit); + } + + @Override + public void run () { + Socket s = null; + for (;;) { + try { + s = serverSocket.accept(); + + //long t0 = System.nanoTime(); + if (framework.handleHandshake(s)) + s = null; + else + s.close(); + //long t1 = System.nanoTime(); + //System.out.println("New connection was processed in: " + TimeUnit.NANOSECONDS.toMillis(t1 - t0)); + + } catch (IOException iox) { + IOUtil.close(s); + + if (!serverSocket.isClosed()) { + VSProtocol.LOGGER.log( + Level.WARNING, + "Exception while accepting connections", + iox + ); + } else { + break; + } + } + } + + if (serverSocket != null && !serverSocket.isClosed()) + IOUtil.close (serverSocket); + } + + public void close () { + IOUtil.close (serverSocket); + interrupt (); + contextContainer.getQuickExecutor().shutdownInstance(); + if (framework != null) + framework.close(); + } + + @VisibleForTesting + VSDispatcher[] getDispatchers() { + return framework.getDispatchers(); + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/VSServerFramework.java b/util/src/main/java/com/epam/deltix/util/vsocket/VSServerFramework.java new file mode 100644 index 00000000..7908f223 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/VSServerFramework.java @@ -0,0 +1,600 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.concurrent.ContextContainer; +import com.epam.deltix.util.concurrent.QuickExecutor; +import com.epam.deltix.util.vsocket.transport.Connection; +import com.epam.deltix.util.vsocket.transport.SocketConnectionFactory; +import com.epam.deltix.util.collections.generated.IntegerToObjectHashMap; +import com.epam.deltix.util.io.offheap.OffHeap; +import com.epam.deltix.util.lang.DisposableListener; +import com.epam.deltix.util.tomcat.ConnectionHandshakeHandler; +import net.jcip.annotations.GuardedBy; + +import java.io.BufferedInputStream; +import java.io.Closeable; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.Socket; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; + +/** + * + */ +public class VSServerFramework implements ConnectionHandshakeHandler, DisposableListener, Closeable { + public static volatile VSServerFramework INSTANCE = null; + + static final int MIN_COMPATIBLE_CLIENT_VERSION = 1014; + static final int MAX_COMPATIBLE_CLIENT_VERSION = VSProtocol.VERSION; + + public final static int MAX_CONNECTIONS = 100; + public final static short MAX_SOCKETS_PER_CONNECTION = 8; + + private final Map dispatchers = + new HashMap <> (); + + private final QuickExecutor executor; + private final ContextContainer contextContainer; + + private volatile VSConnectionListener connectionListener; + private final int connectionsLimit; + private short transportsLimit; + private final long time; + private final int lingerInterval; + private final VSCompression compression; + + private TLSContext tlsContext; + + private TransportType transportType = TransportType.SOCKET_TCP; + + private final DBConnectionAcceptor connectionAcceptor; + + public static final Comparator comparator = new Comparator () { + + @Override + public int compare (VSDispatcher o1, VSDispatcher o2) { + return (o1.getCreationDate ().compareTo (o2.getCreationDate ())); + } + }; + + public VSServerFramework(QuickExecutor executor, int lingerInterval, + VSCompression compression, int connectionsLimit, short socketsPerConnection, ContextContainer contextContainer, DBConnectionAcceptor connectionAcceptor) { + this.connectionAcceptor = connectionAcceptor; + this.executor = executor; + this.lingerInterval = lingerInterval; + this.time = System.currentTimeMillis(); + this.compression = compression; + this.connectionsLimit = connectionsLimit; + this.transportsLimit = socketsPerConnection; + this.contextContainer = contextContainer; + INSTANCE = this; + } + + public VSServerFramework(QuickExecutor executor, int lingerInterval, VSCompression compression, ContextContainer contextContainer) { + this(executor, lingerInterval, compression, MAX_CONNECTIONS, MAX_SOCKETS_PER_CONNECTION, contextContainer, DefaultConnectionAcceptor.INSTANCE); + } + + public QuickExecutor getExecutor () { + return executor; + } + + public VSDispatcher [] getDispatchers () { + VSDispatcher [] ret; + + synchronized (dispatchers) { + ret = new VSDispatcher[dispatchers.size()]; + int index = 0; + for (Connector value : dispatchers.values()) + ret[index++] = value.dispatcher; + } + + Arrays.sort (ret, comparator); + return (ret); + } + + public int getDispatchersCount() { + synchronized (dispatchers) { + return dispatchers.size(); + } + } + + public VSDispatcher getDispatcher(String id) { + synchronized (dispatchers) { + Connector connector = dispatchers.get(id); + return connector != null ? connector.dispatcher : null; + } + } + + /* + Returns current throughput: bytes per second + */ + + public long getThroughput() { + long throughput = 0; + synchronized (dispatchers) { + for (Connector value : dispatchers.values()) + throughput += value.dispatcher.getAverageThroughput(); + } + + return throughput; + } + + public void setConnectionListener (VSConnectionListener lnr) { + connectionListener = lnr; + } + + public void initSSLSocketFactory(TLSContext context) { + this.tlsContext = context; + } + + public void initTransport(TransportProperties transportProperties) { + if (transportProperties != null) { + transportType = transportProperties.transportType; + if (transportType == TransportType.AERON_IPC) { + throw new RuntimeException("Legacy version of Aeron IPC is not supported"); + } else if (transportType == TransportType.OFFHEAP_IPC) { + OffHeap.start(transportProperties.transportDir, true); + } + } + } + + public boolean handleHandshake (Socket s) throws IOException { + s.setSoTimeout (0); + s.setTcpNoDelay(true); + s.setKeepAlive(true); + + BufferedInputStream bis = new BufferedInputStream(s.getInputStream(), VSocketImpl.INPUT_STREAM_BUFFER_SIZE); + return handleHandshake( + SocketConnectionFactory.createConnection(s, bis, s.getOutputStream()) + ); + } + + /** Handles inbound HTTP connection handshake when TB is running inside Tomcat */ + @Override + public boolean handleHandshake(Socket s, BufferedInputStream is, OutputStream os) throws IOException { + s.setSoTimeout (0); + s.setTcpNoDelay(true); + s.setKeepAlive(true); + + return handleHandshake( + SocketConnectionFactory.createConnection(s, is, os) + ); + } + + /** + * Handle the initial transport-level handshake with a client. + * + * @param c Socket to perform the handshake with. + * @return true if the connection is accepted and socket added to the + * set of transport channels. false if socket should be closed. + * + * @throws IOException + */ + public boolean handleHandshake (Connection c) throws IOException { + Level handshakeTimeLogLevel = Level.FINE; + long handshakeStart = VSProtocol.LOGGER.isLoggable(handshakeTimeLogLevel) ? System.currentTimeMillis() : 0; + try { + return handleHandshakeInternal(c); + } finally { + if (VSProtocol.LOGGER.isLoggable(handshakeTimeLogLevel)) { + long handshakeEnd = System.currentTimeMillis(); + VSProtocol.LOGGER.log(handshakeTimeLogLevel, "handleHandshake took " + (handshakeEnd - handshakeStart) + " ms"); + } + } + } + + /** + * See {@link #handleHandshake(Connection)}. + */ + private boolean handleHandshakeInternal (Connection c) throws IOException { + + processSSLHandshake(c); + + DataInputStream dis = new DataInputStream (c.getInputStream()); + DataOutputStream dout = new DataOutputStream (c.getOutputStream()); + + int clientVersion = dis.readInt (); + + boolean isCompatible = clientVersion >= MIN_COMPATIBLE_CLIENT_VERSION && clientVersion <= MAX_COMPATIBLE_CLIENT_VERSION; + + dout.writeInt (isCompatible ? clientVersion : VSProtocol.VERSION); + dout.writeUTF(String.valueOf(VSProtocol.VERSION)); + dout.flush (); + + String clientId = dis.readUTF (); + //dout.writeUTF (Version.VERSION_STRING); + + if (!isCompatible) { + VSProtocol.LOGGER.severe ( + "Connection from " + clientId + " rejected due to incompatible protocol version #" + + clientVersion + " (accepted: " + + MIN_COMPATIBLE_CLIENT_VERSION + " .. " + + MAX_COMPATIBLE_CLIENT_VERSION + ")" + ); + + dout.writeByte (VSProtocol.CONN_RESP_INCOMPATIBLE_CLIENT); + dout.flush (); + return (false); + } + + if (!connectionAcceptor.accept(clientId)) { + dout.writeByte(VSProtocol.CONN_RESP_CONNECTION_REJECTED); + dout.flush(); + return false; + } + + dout.writeInt(tlsContext == null ? 0 : tlsContext.port); + + if (clientVersion > 1014) + processTransportHandshake(c); + + boolean isNew = dis.readBoolean(); + int sCode = dis.readInt(); + long received = dis.readLong(); + + Connector connector = process(clientId); + if (connector == null) { + dout.writeByte (VSProtocol.CONN_RESP_CONNECTION_REJECTED); + dout.flush (); + return (false); + } + + if (VSProtocol.LOGGER.isLoggable (Level.FINE)) + VSProtocol.LOGGER.fine ("Accepted connection from " + clientId); + + VSocketRecoveryInfo brokenSocketRecoveryInfo = isNew ? null : connector.remove(sCode); + VSocket broken = null; + + if (brokenSocketRecoveryInfo != null) { + synchronized (brokenSocketRecoveryInfo) { + // Prevent other threads against attempting to recover same socket + if (brokenSocketRecoveryInfo.startRecoveryAttempt()) { + broken = brokenSocketRecoveryInfo.getSocket(); + } else { + VSProtocol.LOGGER.fine("Recovery attempt failed because another attempt for this thread in progress"); + } + } + } else { + if (!isNew) { + // Client attempts to recover a connection but there are no connection with such "sCode" on the server side. + // That may happen if server was restarted. In such case we should reject the connection + // and force a client to do a full reconnect. + VSProtocol.LOGGER.warning("Connection restore failed for transport (" + sCode + ") for " + clientId + " because server side transport is not found"); + + dout.writeByte(VSProtocol.CONN_RESP_CONNECTION_REJECTED); + dout.flush(); + return false; + } + } + + boolean success = false; + try { + + String transportTag = sCode + " / " + Integer.toHexString(sCode); + if (broken != null) { + VSProtocol.LOGGER.info("Restoring connection (" + transportTag + ") for " + clientId); + broken.getOutputStream().confirm(received); + } else { + if (!isNew) { + VSProtocol.LOGGER.warning("Connection restore failed for transport (" + transportTag + ") for " + clientId + " because server side transport is not found"); + } + } + + dout.writeByte(VSProtocol.CONN_RESP_OK); + dout.writeLong(time); + dout.writeInt(lingerInterval); + dout.writeUTF(compression.toString()); + + // writing -1 means socket wasn't found + dout.writeLong(broken != null ? broken.getInputStream().getBytesRead() : (isNew ? 0 : -1)); + dout.flush(); + + VSocket socket = broken != null ? c.create(broken) : c.create(sCode); + + if (!connector.addTransportChannel(socket)) { + VSProtocol.LOGGER.info("Connection(" + transportTag + ") rejected for " + clientId); + return (false); + } + + if (broken != null) + VSProtocol.LOGGER.info("Connection(" + transportTag + ") restored for " + clientId); + + success = true; + return (true); + } finally { + if (broken != null) { + // assert brokenSocketRecoveryInfo != null; + synchronized (brokenSocketRecoveryInfo) { + brokenSocketRecoveryInfo.stopRecoveryAttempt(); + if (success) { + if (brokenSocketRecoveryInfo.tryMarkRecoverySucceeded()) { + brokenSocketRecoveryInfo.notifyAll(); + } + } + } + } + } + } + + private Connector process(String clientId) { + Connector connector; + + synchronized (dispatchers) { + connector = dispatchers.get(clientId); + + if (connector == null && dispatchers.size() >= connectionsLimit) { + VSProtocol.LOGGER.severe ( + "Connection from " + clientId + " rejected due to connections limit = (" + connectionsLimit + ")" + ); + return null; + } + + if (connector == null) { + VSDispatcher dispatcher = new VSDispatcher (clientId, false, contextContainer); + dispatcher.setConnectionListener(connectionListener); + dispatcher.setLingerInterval(lingerInterval); + dispatcher.addDisposableListener(this); + dispatchers.put (clientId, (connector = new Connector(dispatcher, transportsLimit))); + } + } + + return connector; + } + + private void processSSLHandshake(Connection c) throws IOException { + boolean enableSSL = (tlsContext != null); + boolean sslForLoopback = (tlsContext != null) && !tlsContext.preserveLoopback; + + BufferedInputStream is = c.getInputStream(); + OutputStream os = c.getOutputStream(); + + //initial byte (0 for VS protocol) + int b = is.read(); + assert b == 0; + + int clientHeader = is.read(); + if (clientHeader == VSProtocol.SSL_HEADER && !enableSSL) { + os.write(VSProtocol.CONN_RESP_SSL_NOT_SUPPORTED); + os.flush(); + throw new IOException("Client wants SSL but server have not prepared for handshake."); + } + os.write(VSProtocol.CONN_RESP_OK); + + //server choice + boolean sslConnection = false; + if (enableSSL && sslForLoopback) + sslConnection = true; + else if (enableSSL && !sslForLoopback && !c.isLoopback()) + sslConnection = true; + else if (enableSSL && !sslForLoopback && c.isLoopback() && clientHeader == VSProtocol.SSL_HEADER) + sslConnection = true; + + //write server decision (SSL or NON-SSL) + if (sslConnection) { + os.write(VSProtocol.SSL_HEADER); + c.upgradeToSSL(tlsContext.context.getSocketFactory()); + VSProtocol.LOGGER.fine("Socket upgraded to SSL socket! Now connection is secured."); + } else { + os.write(VSProtocol.HEADER); + } + } + + private void processTransportHandshake(Connection c) throws IOException { + DataOutputStream dout = new DataOutputStream (c.getOutputStream()); + + TransportType type = (!c.isLoopback() || transportType == null) ? TransportType.SOCKET_TCP : transportType; + dout.writeInt(type.ordinal()); + + c.setTransportType(type); + if (type == TransportType.AERON_IPC) + throw new RuntimeException("Legacy version of Aeron IPC is not supported"); + else if (type == TransportType.OFFHEAP_IPC) + dout.writeUTF(OffHeap.getOffHeapDir()); + } + + public VSCompression getCompression() { + return compression; + } + + @Override + public void disposed(VSDispatcher resource) { + synchronized (dispatchers) { + Connector c = dispatchers.remove(resource.getClientId()); + if (c != null) + c.close(); + } + } + + @Override + public void close() { + if (transportType == TransportType.AERON_IPC) + throw new RuntimeException("Legacy version of Aeron IPC is not supported"); + } + + void setTransportsLimit(short transportsLimit) { + this.transportsLimit = transportsLimit; + } + + static class Connector extends ConnectionStateListener implements Closeable { + // May contain null values. Null value indicates that transport is still considered active (not stopped). + private final IntegerToObjectHashMap stopped = + new IntegerToObjectHashMap<>(); + + private final VSDispatcher dispatcher; + private final int limit; // limit of transport channels + + Connector(VSDispatcher d, int limit) { + this.dispatcher = d; + this.limit = limit; + dispatcher.setStateListener(this); + } + + boolean addTransportChannel(VSocket socket) throws IOException { + + synchronized (stopped) { + + // not accept new channels, only that can be restored + if (!stopped.containsKey(socket.getCode()) && stopped.size() >= limit) + return false; + + // Note: we may override previous state of socket here (in case of recovery) + stopped.put(socket.getCode(), ACTIVE); + stopped.notifyAll(); + } + + dispatcher.addTransportChannel(socket); + return true; + } + + @Override + boolean onTransportRecoveryStart(VSocketRecoveryInfo recoveryInfo) { + VSocket socket = recoveryInfo.getSocket(); + + int code = socket.getCode(); + synchronized (stopped) { + VSocketRecoveryInfo prevValue = stopped.get(code, null); + if (prevValue != ACTIVE) { + String clientId = dispatcher.getClientId(); + if (prevValue == null) { + VSProtocol.LOGGER.severe("Transport for " + clientId + " has stopped. It can't be recovered because the recovery state is invalid."); + } else { + VSProtocol.LOGGER.warning("Transport for " + clientId + " has stopped. It can't be recovered because a recovery attempt is already initiated."); + } + return true; + } + stopped.put(code, recoveryInfo); + stopped.notifyAll(); + return false; + } + } + + @Override + boolean onTransportRecoveryStop(VSocketRecoveryInfo recoveryInfo) { + try { + synchronized (recoveryInfo) { + while (recoveryInfo.isRecoveryAttemptInProgress()) { + recoveryInfo.wait(); + } + if (recoveryInfo.isRecoverySucceeded()) { + return false; + } + recoveryInfo.markRecoveryFailed(); + + VSocket socket = recoveryInfo.getSocket(); + int code = socket.getCode(); + synchronized (stopped) { + VSocketRecoveryInfo currentValue = stopped.get(code, null); + if (currentValue == recoveryInfo) { + stopped.remove(code); + stopped.notifyAll(); + recoveryInfo.notifyAll(); + return true; + } + } + + recoveryInfo.notifyAll(); + return false; + } + } catch (InterruptedException e) { + return true; + } + } + + @Override + public void close() { + dispatcher.setStateListener(null); + + synchronized (stopped) { + stopped.clear(); + stopped.notifyAll(); + } + } + + /** + * Attempts to get transport for a recovery attempt. + */ + public VSocketRecoveryInfo remove(int code) { + try { + synchronized (stopped) { + VSocketRecoveryInfo currentValue; + while (true) { + currentValue = stopped.get(code, null); + // request for restoring connection may came faster that socket is stopped + if (currentValue == ACTIVE) { + stopped.wait(); + } else { + break; + } + } + + // assert currentValue != ACTIVE + + if (currentValue == null) { + // This transport never existed OR permanently removed + return null; + } + + return currentValue; + } + } catch (InterruptedException e) { + return null; + } + } + + @Override + void onDisconnected() { + synchronized (stopped) { + stopped.clear(); + stopped.notifyAll(); + } + } + + @Override + void onConnected() { + } + } + + // Special marker values + + // This value indicates that the transport for corresponding code is active (not stopped yet) + private static final FakeRecoveryInfo ACTIVE = new FakeRecoveryInfo("ACTIVE"); + + /** + * Special class to represent special values in {@link Connector#stopped} map. + */ + private static class FakeRecoveryInfo extends VSocketRecoveryInfo { + private final String label; + + FakeRecoveryInfo(String label) { + super(null, Long.MAX_VALUE); + this.label = label; + } + + @Override + public String toString() { + return "FakeVSocket{" + label + '}'; + } + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/VSServerSocketFactory.java b/util/src/main/java/com/epam/deltix/util/vsocket/VSServerSocketFactory.java new file mode 100644 index 00000000..44431dd1 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/VSServerSocketFactory.java @@ -0,0 +1,49 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import javax.net.ssl.SSLServerSocketFactory; +import java.io.IOException; +import java.net.InetAddress; +import java.net.ServerSocket; + +/** + * + */ +public class VSServerSocketFactory { + public static ServerSocket createServerSocket(int port) throws IOException { + return createServerSocket(port, null, null); + } + + public static ServerSocket createServerSocket(int port, InetAddress address) throws IOException { + return createServerSocket(port, address, null); + } + + public static ServerSocket createServerSocket(int port, InetAddress address, TLSContext ctx) throws IOException { + ServerSocket socket = null; + if (ctx != null) { + //SSL server socket + SSLServerSocketFactory ssf = ctx.context.getServerSocketFactory(); + socket = ssf.createServerSocket(port, 0, address); + } else { + socket = new ServerSocket(port, 0, address); // if backlog = 0 server will decide automatically + } + + return socket; + } + +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/VSTransportChannel.java b/util/src/main/java/com/epam/deltix/util/vsocket/VSTransportChannel.java new file mode 100644 index 00000000..412ab7a2 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/VSTransportChannel.java @@ -0,0 +1,373 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.LangUtil; +import com.epam.deltix.util.concurrent.QuickExecutor; +import com.epam.deltix.util.lang.Disposable; +import com.epam.deltix.util.lang.Util; +import com.epam.deltix.util.memory.DataExchangeUtils; + +import javax.annotation.Nonnull; +import javax.annotation.concurrent.GuardedBy; +import java.io.DataInputStream; +import java.io.IOException; +import java.util.concurrent.ThreadFactory; +import java.util.logging.Level; + +/** + * Similar to DataSocket, but stripped of much special logic. + */ +class VSTransportChannel implements Runnable, Disposable { + // Set to true whenever the channel is checked out + @GuardedBy("dispatcher.freeChannels") + boolean checkedOut = false; + + private byte[] buffer = new byte[4096]; + private final byte[] header = new byte[16]; + + private final VSDispatcher dispatcher; + final VSocket socket; + + private final VSocketInputStream vin; + private final DataInputStream din; + private final VSocketOutputStream out; + + private final byte[] keepAlive = new byte[2]; + private final byte[] bytesReport = new byte[10]; + private final byte[] ping = new byte[2]; + + private volatile boolean closed = false; + volatile long latency = Long.MAX_VALUE; + + // Value at which the "completeTask" should be triggered next time. + // Checked and updated by transport thread. + private long nextReportValue = VSocketOutputStream.REPORT_THRESHOLD; + + private final Thread thread; + + private final QuickExecutor.QuickTask completeTask; + + @Nonnull + private QuickExecutor.QuickTask createCompleteTask(QuickExecutor quickExecutor) { + return new QuickExecutor.QuickTask(quickExecutor) { + // Bytes that already reported. Checked and updated by completeTask. + private long reported = 0; + + @Override + public void run() throws InterruptedException { + long bytesRead; + synchronized (out) { + bytesRead = vin.getBytesRead(); + if (bytesRead == reported) { + // We already reported this value + return; + } + DataExchangeUtils.writeLong(bytesReport, 2, bytesRead); + out.write(bytesReport, 0, bytesReport.length); + reported = bytesRead; + } + if (VSProtocol.LOGGER.isLoggable(Level.FINEST)) { + VSProtocol.LOGGER.log(Level.FINEST, "Sent BYTES_RECIEVED report: " + bytesRead + " from " + socket.getSocketIdStr()); + } + } + }; + } + + VSTransportChannel(VSDispatcher dispatcher, final VSocket socket, ThreadFactory threadFactory) throws IOException { + this.dispatcher = dispatcher; + this.socket = socket; + this.completeTask = createCompleteTask(dispatcher.getQuickExecutor()); + + DataExchangeUtils.writeUnsignedShort(keepAlive, 0, VSProtocol.KEEP_ALIVE); + DataExchangeUtils.writeUnsignedShort(ping, 0, VSProtocol.PING); + DataExchangeUtils.writeUnsignedShort(bytesReport, 0, VSProtocol.BYTES_RECIEVED); + + this.vin = socket.getInputStream(); + this.din = new DataInputStream (vin); + this.out = socket.getOutputStream(); + + this.thread = threadFactory.newThread(this); + this.thread.setName("VSTransportChannel for " + socket); + } + + private synchronized void onException (Throwable x) { + dispatcher.transportStopped (this, x); + close(); + } + + public void write(int id, int index, long position, byte[] data, int offset, int length, int unpackedLength) { + assert id >= 0 && length > 0; + + if (length == 0) + VSProtocol.LOGGER.log (Level.WARNING, "Writing zero length packet"); + + if (out.isBroken()) { + if (VSProtocol.LOGGER.isLoggable(Level.FINEST)) { + VSProtocol.LOGGER.log(Level.FINEST, "Write to a broken transport " + socket.getSocketIdStr()); + } + } + + if (VSProtocol.LOGGER.isLoggable(Level.FINEST)) { + VSProtocol.LOGGER.log(Level.FINEST, "Sending data block " + position + ":" + (position + unpackedLength) + " (" + length + "/" + unpackedLength + ") to socket: " + socket.getSocketIdStr()); + } + + synchronized (out) { + DataExchangeUtils.writeUnsignedShort(header, 0, id); + DataExchangeUtils.writeUnsignedShort(header, 2, length); + DataExchangeUtils.writeInt(header, 4, index); + DataExchangeUtils.writeLong(header, 8, position); + + out.writeTwoArrays (header, 0, header.length, data, offset, length); + } + } + + public void write (byte [] data) { + write (data, 0, data.length); + } + + public void write (byte [] data, int offset, int length) { + if (length == 0) + VSProtocol.LOGGER.log (Level.WARNING, "Zero length packet"); + + synchronized (out) { + out.write (data, offset, length); + } + } + + public void keepAlive () { + write(keepAlive); + } + + private long ping () { + long l = latency = System.nanoTime(); + write(ping); + return l; + } + + @Override + public void run () { + int index; + long offset; + Thread currentThread = Thread.currentThread(); + assert currentThread == this.thread; + + try { + for (;;) { + vin.complete(); + + if (currentThread.isInterrupted()) + throw new InterruptedException(); + + long bytesRead = vin.getBytesRead(); + if (bytesRead >= nextReportValue) { + nextReportValue = bytesRead + VSocketOutputStream.REPORT_THRESHOLD; + completeTask.submit(); + } + + int destId = din.readUnsignedShort (); + //System.out.println(this.socket + ": signal = " + destId); + + if (destId == VSProtocol.LISTENER_ID) { // Virtual connection request + int code = din.readUnsignedShort (); + + int inCapacity = din.readInt(); + int outCapacity = din.readInt(); + int rIndex = din.readInt(); + boolean compressed = din.readByte() == 1; + + VSChannelImpl local = dispatcher.newChannel (outCapacity, inCapacity, compressed); + try { + local.onConnectionRequest(code, inCapacity, rIndex); + dispatcher.connectionListener.connectionAccepted (dispatcher.getQuickExecutor(), local); + } catch (Throwable x) { + // No reason to shutdown this transport channel + VSProtocol.LOGGER.log (Level.SEVERE, "Exception sending ACK", x); + local.close (); + LangUtil.propagateError(x); + } + } + else if (destId == VSProtocol.BYTES_RECIEVED) { + long size = din.readLong (); + out.confirm(size); + if (VSProtocol.LOGGER.isLoggable(Level.FINEST)) { + VSProtocol.LOGGER.log(Level.FINEST, "Got BYTES_RECIEVED report: " + size + " in " + socket.getSocketIdStr()); + } + } + else if (destId == VSProtocol.KEEP_ALIVE) { + // keep alive signal - do nothing + } + else if (destId == VSProtocol.PING) { + if (latency != Long.MAX_VALUE) + latency = System.nanoTime() - latency; + else + write(ping); + } + else if (destId == VSProtocol.DISPATCHER_CLOSE) { + dispatcher.onRemoteClosed(); + } + else { + int code = din.readUnsignedShort (); + VSChannelImpl c = dispatcher.getChannel (destId); + + if (c == null && code != VSProtocol.CLOSING) { + if (code == VSProtocol.BYTES_AVAILABLE_REPORT) { + if (VSProtocol.LOGGER.isLoggable(Level.FINE)) { + VSProtocol.LOGGER.log(Level.FINE, "Got BYTES_AVAILABLE_REPORT for missing (recently closed?) channel " + destId); + } + } else { + VSProtocol.LOGGER.log(Level.SEVERE, code + ": No local channel for " + destId); + } + } + + switch (code) { + case VSProtocol.CONNECT_ACK: + int remoteId = din.readUnsignedShort (); + int remoteCapacity = din.readInt (); + int remoteIndex = din.readInt (); + + assert c != null; + + c.onRemoteConnected(remoteId, remoteCapacity, remoteIndex); + break; + + case VSProtocol.CLOSING: + index = din.readInt(); + offset = din.readLong(); + + if (c != null) { + boolean valid = c.assertIndexValid("CLOSING", index); + if (valid) { + c.processCommand(code, offset); + } + } + + break; + + case VSProtocol.CLOSED: + index = din.readInt(); + offset = din.readLong(); + + if (c != null) { + boolean valid = c.assertIndexValid("CLOSED", index); + if (valid) { + c.processCommand(code, offset); + } + } + + break; + + case VSProtocol.BYTES_AVAILABLE_REPORT : + int available = din.readInt(); + index = din.readInt(); + + assert (available >= 0); + + if (c != null) { + boolean valid = index == c.getIndex(); + // make sure that we mark that bytes reports as read, + vin.complete(); + // because code below may not return + if (valid) { + c.onCapacityIncreased(available); + if (VSProtocol.LOGGER.isLoggable(Level.FINEST)) { + VSProtocol.LOGGER.log(Level.FINEST, "Got BYTES_AVAILABLE_REPORT report: " + available + " in " + socket.getSocketIdStr()); + } + } else { + VSProtocol.LOGGER.log(Level.FINE, "Got BYTES_AVAILABLE_REPORT for wrong channel " + destId); + } + } + break; + + default: + index = din.readInt(); + offset = din.readLong(); + + if (c == null) { + VSProtocol.LOGGER.log (Level.INFO, "Skipping bytes (no channel): " + code); + din.skipBytes (code); + } else if (!c.assertIndexValid(code, index)) { + VSProtocol.LOGGER.log (Level.WARNING, "Skipping bytes (wrong channel): " + code); + din.skipBytes (code); + } else { + int capacity = buffer.length; + + if (capacity < code) + buffer = new byte [Util.doubleUntilAtLeast (capacity, code)]; + + if (code == 0) + VSProtocol.LOGGER.log (Level.WARNING, "Unknown zero-length data for channel: " + destId); + + din.readFully (buffer, 0, code); + c.receive(offset, buffer, 0, code, socket.getCode(), socket.getSocketNumber()); + } + } + } + } + } catch (InterruptedException e) { + if (VSProtocol.LOGGER.isLoggable(Level.FINE)) + VSProtocol.LOGGER.log (Level.FINE, "Interrupted" , e); + Thread.currentThread().interrupt(); + } catch (Throwable x) { + if (currentThread.isInterrupted()) + VSProtocol.LOGGER.log (Level.FINE, this + ": Interrupted."); + else + onException (x); + + LangUtil.propagateError(x); + } finally { + closed = true; + } + } + + public long getLatency() { + long l = ping(); + while (latency == l && !closed) + try { + Thread.sleep(1); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return Long.MAX_VALUE; + } + + if (closed) + return Long.MAX_VALUE; + + return latency; + } + + @Override + public void close () { + //flusher.interrupt(); + this.thread.interrupt(); + Util.close (socket); + } + + public void start() { + thread.start(); + } + + @Override + public String toString() { + return getClass().getName() +"@" + Integer.toHexString(hashCode()) + " of socket " + socket.getSocketIdStr(); + } + + public String getSocketIdStr() { + return socket.getSocketIdStr(); + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/VSocket.java b/util/src/main/java/com/epam/deltix/util/vsocket/VSocket.java new file mode 100644 index 00000000..1ffdc8aa --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/VSocket.java @@ -0,0 +1,49 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.lang.Disposable; + +/** + * + */ +public interface VSocket extends Disposable { + + VSocketInputStream getInputStream(); + + VSocketOutputStream getOutputStream(); + + String getRemoteAddress(); + + //void restore(VSocket from); + + void close(); + + int getCode(); + + void setCode(int code); + + /** + * Serial number of this socket. + * Should be used only for easier identification of the socket during debug. + */ + int getSocketNumber(); + + default String getSocketIdStr() { + return '@' + Integer.toHexString(getCode()) + "#" + getSocketNumber(); + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/VSocketFactory.java b/util/src/main/java/com/epam/deltix/util/vsocket/VSocketFactory.java new file mode 100644 index 00000000..4d482af3 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/VSocketFactory.java @@ -0,0 +1,106 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.Socket; +import java.util.concurrent.atomic.AtomicInteger; + +public class VSocketFactory { + private static final AtomicInteger socketNumberGen = new AtomicInteger(); + + public enum Transport { + Socket, + SharedMemory, + Memory + } + + public static volatile Transport transport; + + public static VSocket get(ClientConnection cc, TransportType transportType) throws IOException { + int socketNumber = nextSocketNumber(); + +// synchronized (cache) { +// String remoteID = getRemoteID(s); +// if (!cache.containsKey(remoteID)) { +// MemorySocket socket = new MemorySocket(); +// MemorySocket remote = new MemorySocket(socket); +// cache.put(getRemoteID(s), remote); +// cache.put(getID(s), socket); +// +// return socket; +// } else { +// return cache.get(getID(s)); +// } +// } + + Socket s = cc.getSocket(); + if (transportType == TransportType.AERON_IPC) { + throw new RuntimeException("Legacy version of Aeron IPC is not supported"); + } + else if (transportType == TransportType.OFFHEAP_IPC) + return new OffHeapIpcSocket(s, s.hashCode(), false, socketNumber); + else + return new VSocketImpl(cc, socketNumber); + } + + public static VSocket get(ClientConnection cc, VSocket stopped) throws IOException { + Socket s = cc.getSocket(); + int socketNumber = nextSocketNumber(); + VSocket socket; + if (stopped instanceof VSocketImpl) { + socket = new VSocketImpl(cc, socketNumber); + } else if (stopped instanceof OffHeapIpcSocket) { + socket = new OffHeapIpcSocket(s, stopped.getCode(), false, socketNumber); + } else { + throw new IllegalStateException("Unknown VSocket implementation:" + stopped); + } + + socket.setCode(stopped.getCode()); + + //System.out.println("Restore socket from " + stopped); + stopped.getOutputStream().writeTo(socket.getOutputStream()); + return socket; + } + + public static VSocket get(Socket socket, BufferedInputStream in, OutputStream out) + throws IOException + { + int socketNumber = nextSocketNumber(); + //return get(socket); + + return new VSocketImpl(socket, in, out, socket.hashCode(), socketNumber); + } + +// public static VSocket get(Connection c) throws IOException { +// return c.create(); +// } + + private static String getID(Socket s) { + return s.getLocalAddress().toString() + ":" + s.getPort() + "\\" + s.getLocalPort(); + } + + private static String getRemoteID(Socket s) { + return s.getLocalAddress().toString() + ":" + s.getLocalPort() + "\\" + s.getPort(); + } + + public static int nextSocketNumber() { + return socketNumberGen.incrementAndGet(); + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/VSocketImpl.java b/util/src/main/java/com/epam/deltix/util/vsocket/VSocketImpl.java new file mode 100644 index 00000000..9d74634c --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/VSocketImpl.java @@ -0,0 +1,211 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.vsocket.transport.Connection; +import com.epam.deltix.util.io.IOUtil; +import com.epam.deltix.util.lang.Util; +import org.jetbrains.annotations.ApiStatus; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.Socket; +import java.net.SocketAddress; +import java.net.SocketException; +import java.util.logging.Level; + +/** + * Date: Mar 5, 2010 + */ +public class VSocketImpl implements VSocket { + // Buffer size should be at least big enough to fit maximum packet size (VSProtocol.MAX_SIZE). + // Controls buffer sized. "Default buffer size" lets you set both send and receive buffer size using single argument. + public static final int SOCKET_DEFAULT_BUFFER_SIZE = Integer.getInteger("TimeBase.network.socket.bufferSize", 1 << 16); + public static final int SOCKET_RECEIVE_BUFFER_SIZE = Integer.getInteger("TimeBase.network.socket.receiveBufferSize", SOCKET_DEFAULT_BUFFER_SIZE); + public static final int SOCKET_SEND_BUFFER_SIZE = Integer.getInteger("TimeBase.network.socket.sendBufferSize", SOCKET_DEFAULT_BUFFER_SIZE); + + @ApiStatus.Experimental // Temporary option for testing performance effect of using buffered reader of different size + // 8kb size matches to the previous value. However it's very likely that we need 64k or 128k buffer size to match the maximum "logical packet" size (VSProtocol.MAXSIZE). + // TODO: Consider increasing default value to 64kb. + public static final int INPUT_STREAM_BUFFER_SIZE = Integer.getInteger("TimeBase.network.streamBufferSize", 8 * 1024); + + public static final boolean PRINT_VSOCKET_SETTINGS = Boolean.getBoolean("TimeBase.network.printSettings"); + + static { + if (PRINT_VSOCKET_SETTINGS) { + System.out.println("SOCKET_RECEIVE_BUFFER_SIZE: " + SOCKET_RECEIVE_BUFFER_SIZE); + System.out.println("SOCKET_SEND_BUFFER_SIZE: " + SOCKET_SEND_BUFFER_SIZE); + System.out.println("INPUT_STREAM_BUFFER_SIZE: " + INPUT_STREAM_BUFFER_SIZE); + } + } + + private static final int IPTOS_THROUGHPUT = 0x08; + private static final int DEFAULT_TRAFFIC_CLASS = IPTOS_THROUGHPUT; + /** Value for {@link Socket#setTrafficClass(int)} */ + public static final int SOCKET_TRAFFIC_CLASS = validateTrafficClassValue(Integer.getInteger("TimeBase.network.socket.tos", DEFAULT_TRAFFIC_CLASS)); + + private final Socket socket; + private final InputStream in; + private final BufferedInputStream bin; + + private final OutputStream out; + private final VSocketOutputStream vout; + private final VSocketInputStream vin; + private String remoteAddress; + private int code; + private final int socketNumber; + + + /** + * Configures socket. + * + *

Current implementation of {@link VSClient} executes this on a connected socket. + * While this is allowed to change socket buffer sizes after the connection is established, + * it may have different effects on different platforms.

+ * + *

Most importantly, TCP Window size may be limited by 64k if receive buffer size + * is set after the connection is established.

+ */ + private void setUpSocket () { + try { + socket.setTcpNoDelay (true); + socket.setSoTimeout (0); + socket.setKeepAlive (true); + + // This is likely to have no effect as corresponding TOS field is deprecated. + // See https://en.wikipedia.org/wiki/Type_of_service + // and https://en.wikipedia.org/wiki/Differentiated_services + socket.setTrafficClass(SOCKET_TRAFFIC_CLASS); + + SocketAddress address = socket.getRemoteSocketAddress(); + remoteAddress = address != null ? address.toString() : null; + + // We do not change socket buffer sizes here because + // we expect that they are already set using configureBufferSizes(...) method or manually. + } catch (IOException x) { + VSProtocol.LOGGER.log (Level.WARNING, null, x); + } + } + + /** + * Configures buffer sizes for a socket. + * + *

It's important to configure decent buffer sizes. + * It should be at least big enough to fit maximum packet size ({@link VSProtocol#MAXSIZE}). + * Otherwise, there is a possible situation when TimeBase client and server may run into a deadlock, + * when transport thread gets stuck in blocking write into socket.

+ * + *

Please note that if socket is already connected, then OS may ignore these values.

+ * + * Also please note that in case of server-side socket, the "receive" buffer size should be set on + * {@link java.net.ServerSocket} using {@link java.net.ServerSocket#setReceiveBufferSize(int)} method. + * On the client socket, the "receive" buffer size should be set before the connection is established. + * Otherwise, TCP window size will be limited by 64k. + */ + public static void configureBufferSizes(Socket socket) throws SocketException { + socket.setReceiveBufferSize(SOCKET_RECEIVE_BUFFER_SIZE); + socket.setSendBufferSize(SOCKET_SEND_BUFFER_SIZE); + } + + public VSocketImpl(ClientConnection cc, int socketNumber) { + this.socket = cc.getSocket(); + this.in = cc.getInputStream (); + this.bin = cc.getBufferedInputStream(); + this.out = cc.getOutputStream(); + this.code = this.socket.hashCode(); + this.socketNumber = socketNumber; + String socketIdStr = getSocketIdStr(); + this.vout = new VSocketOutputStream(out, socketIdStr); + this.vin = new VSocketInputStream(bin, socketIdStr); + setUpSocket (); + } + + public VSocketImpl(Socket s, BufferedInputStream in, OutputStream out, int code, int socketNumber) { + this.socket = s; + this.in = this.bin = in; + this.out = out; + this.code = code; + this.socketNumber = socketNumber; + String socketIdStr = getSocketIdStr(); + this.vout = new VSocketOutputStream(out, socketIdStr); + this.vin = new VSocketInputStream(bin, socketIdStr); + setUpSocket (); + } + + public VSocketImpl(Connection c, int code, int socketNumber) { + this.in = this.bin = c.getInputStream(); + this.out = c.getOutputStream(); + this.socketNumber = socketNumber; + this.code = code; + String socketIdStr = getSocketIdStr(); + this.vout = new VSocketOutputStream(out, socketIdStr); + this.vin = new VSocketInputStream(bin, socketIdStr); + this.socket = null; + //setUpSocket (); + } + + @Override + public int getCode() { + return code; + } + + @Override + public void setCode(int code) { + this.code = code; + } + + @Override + public VSocketInputStream getInputStream() { + return vin; + } + + @Override + public VSocketOutputStream getOutputStream() { + return vout; + } + + @Override + public String getRemoteAddress() { + return remoteAddress; + } + + @Override + public void close() { + IOUtil.close (socket); + Util.close (in); + Util.close (out); + } + + @Override + public String toString() { + return getClass().getName() + getSocketIdStr(); + } + + @Override + public int getSocketNumber() { + return socketNumber; + } + + private static int validateTrafficClassValue(int value) { + if (value < 0 || value > 255) { + throw new IllegalArgumentException("Invalid value for TimeBase.network.socket.tos: " + value); + } + return value; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/VSocketInputStream.java b/util/src/main/java/com/epam/deltix/util/vsocket/VSocketInputStream.java new file mode 100644 index 00000000..50094ef4 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/VSocketInputStream.java @@ -0,0 +1,114 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * Date: Jun 22, 2010 + * + * @author alex + */ +public class VSocketInputStream extends FilterInputStream { + private final String socketIdStr; + + private volatile long bytes = 0; + private volatile long completed = 0; + private volatile long mark; + + public VSocketInputStream(InputStream delegate, String socketIdStr) { + super(delegate); + this.socketIdStr = socketIdStr; + } + + void complete() { + completed = bytes; + } + + long getBytesRead() { + return completed; + } + + long getTotalBytes() { + return bytes; + } + +// void setBytesRead(long value) { +// completed = bytes = value; +// } + + @Override + public int read () throws IOException { + int count = super.read (); + + if (count >= 0) + bytes ++; + + return (count); + } + + + @Override + public int read(byte[] b) throws IOException { + int n = super.read(b); + + if (n >= 0) + bytes += n; + + return (n); + } + + @Override + public int read (byte[] b, int off, int len) throws IOException { + int n = super.read (b, off, len); + if (n > 0) + bytes += n; + + return (n); + } + + @Override + public void mark (int readlimit) { + super.mark (readlimit); + mark = bytes; + } + + @Override + public long skip(long n) throws IOException { + long skipped = super.skip(n); + bytes += skipped; + + return skipped; + } + + + @Override + public void reset () throws IOException { + super.reset (); + + if (bytes != mark) + completed = bytes = mark; + } + + @Override + public String toString() { + return getClass().getName() + socketIdStr; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/VSocketOutputStream.java b/util/src/main/java/com/epam/deltix/util/vsocket/VSocketOutputStream.java new file mode 100644 index 00000000..199876ef --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/VSocketOutputStream.java @@ -0,0 +1,228 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.collections.ByteQueue; +import net.jcip.annotations.GuardedBy; +import org.jetbrains.annotations.ApiStatus; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.logging.Level; + +/** + * + */ +public class VSocketOutputStream extends OutputStream { + private final String socketIdStr; + + @ApiStatus.Experimental + public static int CAPACITY = Integer.getInteger("TimeBase.network.socketOutputStream.bufferCapacity", 1024 * 512); + public static int INCREMENT = CAPACITY / 4; + /** Controls how often {@link VSProtocol#BYTES_RECIEVED} message will be sent from {@link VSTransportChannel} */ + @ApiStatus.Experimental + public static int REPORT_THRESHOLD = Integer.getInteger("TimeBase.network.socketOutputStream.reportThreshold", CAPACITY / 4); + + @GuardedBy("buffer") + private final ByteQueue buffer; + @GuardedBy("out") + private final OutputStream out; + long confirmed; + + // How many bytes we need to skip from the output buffer to make it in-sync with last "confirmed" position + private int bufferDebt = 0; + + // Set to true if we got any IOException from output stream + private volatile boolean broken = false; + + public VSocketOutputStream(OutputStream out, String socketIdStr) { + this.out = out; + this.socketIdStr = socketIdStr; + this.buffer = new ByteQueue(CAPACITY); + } + + @Override + public void write(byte[] b) { + write(b, 0, b.length); + } + + @Override + public void write(int b) { + try { + synchronized (out) { + out.write(b); + } + } catch (IOException e) { + broken = true; + //throw new deltix.util.io.UncheckedIOException(e); + } finally { + dump(b); + } + } + + @Override + public void write(byte[] b, int off, int len) { + try { + synchronized (out) { + out.write(b, off, len); + } + } catch (IOException e) { + broken = true; + //throw new deltix.util.io.UncheckedIOException(e); + } finally { + dump(b, off, len); + } + } + + /** + * Same as calling {@link #write(byte[], int, int)} two times but a little bit more effective. + */ + @SuppressWarnings("SameParameterValue") + public void writeTwoArrays(byte[] b1, int off1, int len1, byte[] b2, int off2, int len2) { + try { + synchronized (out) { + out.write(b1, off1, len1); + out.write(b2, off2, len2); + } + } catch (IOException e) { + broken = true; + //throw new deltix.util.io.UncheckedIOException(e); + } finally { + dumpTwoArrays(b1, off1, len1, b2, off2, len2); + } + } + + private void dump(byte[] b, int off, int len) { + synchronized (buffer) { + dumpInternal(b, off, len); + } + } + + /** + * Same as calling {@link #dump(byte[], int, int)} two times but a little bit more effective. + */ + private void dumpTwoArrays(byte[] b1, int off1, int len1, byte[] b2, int off2, int len2) { + synchronized (buffer) { + dumpInternal(b1, off1, len1); + dumpInternal(b2, off2, len2); + } + } + + @GuardedBy("buffer") + private void dumpInternal(byte[] b, int off, int len) { + // assert Thread.holdsLock(buffer); + int overflow = buffer.size() + len - buffer.capacity(); + if (overflow > 0) { + int incrementsToAdd = divideRoundUp(overflow, INCREMENT); + buffer.addCapacity(INCREMENT * incrementsToAdd); + } + + buffer.offer(b, off, len); + } + + static int divideRoundUp(int val, int divisor) { + int result = val / divisor; + if (val > result * divisor) { + result += 1; + } + return result; + } + + private void dump(int b) { + synchronized (buffer) { + if (buffer.size() + 1 > buffer.capacity()) + buffer.addCapacity(INCREMENT); + + buffer.offer(b); + } + } + + @Override + public void flush() throws IOException { + try { + synchronized (out) { + out.flush(); + } + } catch (IOException e) { + broken = true; + throw e; + } + } + + @Override + public void close() throws IOException { + synchronized (out) { + out.close(); + } + } + + /* + Returns number of bytes written to stream + */ + public long getBytesWritten() { + return confirmed + buffer.size(); + } + + public void confirm(long size) { + + synchronized (buffer) { + int length = (int) (size - confirmed); + int bufferSize = buffer.size(); + if (length > bufferSize) { + bufferDebt = length - bufferSize; + length = bufferSize; + } else { + bufferDebt = 0; + } + + // buffer is not synchronized with out, so + // confirmed bytes signal maybe processed before dump() occurred + if (length > 0) { + buffer.skip(length); + confirmed += length; + } + } + } + + boolean hasUnconfirmedData() { + return buffer.size() - bufferDebt > 0; + } + + public void writeTo(VSocketOutputStream out) throws IOException { + synchronized (buffer) { + if (bufferDebt > buffer.size()) { + throw new IllegalStateException("We confirmed more than we have in buffer"); + } else { + buffer.skip(bufferDebt); + } + if (VSProtocol.LOGGER.isLoggable(Level.FINE)) { + VSProtocol.LOGGER.fine("Sending output buffer of " + out.socketIdStr + " with size " + buffer.size()); + } + buffer.poll(out); + } + } + + public boolean isBroken() { + return broken; + } + + @Override + public String toString() { + return getClass().getName() + socketIdStr; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/VSocketRecoveryInfo.java b/util/src/main/java/com/epam/deltix/util/vsocket/VSocketRecoveryInfo.java new file mode 100644 index 00000000..c239bde9 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/VSocketRecoveryInfo.java @@ -0,0 +1,119 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.annotations.TimestampMs; +import net.jcip.annotations.GuardedBy; + +/** + * @author Alexei Osipov + */ +class VSocketRecoveryInfo { + private final VSocket socket; + + private int reconnectAttempts = 0; + private long lastReconnectAttemptTs = Long.MIN_VALUE; + @TimestampMs + private final long recoveryDeadlineTs; + + private boolean recoveryFailed; + private boolean recoverySucceeded; + + // If true, it means that currently a thread actively attempts to recover corresponding channel + private boolean recoveryAttemptInProgress; + + VSocketRecoveryInfo(VSocket socket, long recoveryDeadlineTs) { + this.socket = socket; + this.recoveryDeadlineTs = recoveryDeadlineTs; + } + + int addReconnectAttempt(long reconnectAttemptTimestamp) { + reconnectAttempts++; + lastReconnectAttemptTs = reconnectAttemptTimestamp; + return reconnectAttempts; + } + + int getReconnectAttempts() { + return reconnectAttempts; + } + + long getLastReconnectAttemptTs() { + return lastReconnectAttemptTs; + } + + long getRecoveryDeadlineTs() { + return recoveryDeadlineTs; + } + + VSocket getSocket() { + return socket; + } + + void markRecoveryFailed() { + recoveryFailed = true; + } + + @GuardedBy("this") + boolean tryMarkRecoverySucceeded() { + if (!isRecoveryEnded()) { + recoverySucceeded = true; + return true; + } else { + return false; + } + } + + boolean isRecoveryFailed() { + return recoveryFailed; + } + + boolean isRecoverySucceeded() { + return recoverySucceeded; + } + + boolean isRecoveryEnded() { + return recoverySucceeded || recoveryFailed; + } + + boolean isWaitingForRecovery() { + return recoveryAttemptInProgress || !isRecoveryEnded(); + } + + boolean startRecoveryAttempt() { + if (recoveryAttemptInProgress || isRecoveryEnded()) { + // Only one attempt at a time + return false; + } else { + recoveryAttemptInProgress = true; + return true; + } + } + + void stopRecoveryAttempt() { + if (recoveryAttemptInProgress) { + // Only one attempt at a time + recoveryAttemptInProgress = false; + } else { + throw new IllegalStateException("Attempt to stop recovery multiple times"); + } + } + + public boolean isRecoveryAttemptInProgress() { + return recoveryAttemptInProgress; + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/transport/Connection.java b/util/src/main/java/com/epam/deltix/util/vsocket/transport/Connection.java new file mode 100644 index 00000000..ec52bcd1 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/transport/Connection.java @@ -0,0 +1,51 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket.transport; + +import com.epam.deltix.util.vsocket.TransportType; +import com.epam.deltix.util.vsocket.VSocket; + +import javax.net.ssl.SSLSocketFactory; +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetAddress; + +/** + * + */ +public interface Connection { + + public OutputStream getOutputStream(); + + public BufferedInputStream getInputStream(); + + public VSocket create(int code) throws IOException; + + public VSocket create(VSocket stopped) throws IOException; + + public InetAddress getRemoteAddress(); + + public void close(); + + public boolean isLoopback(); + + public void setTransportType(TransportType transportType); + + public void upgradeToSSL(SSLSocketFactory sslSocketFactory) throws IOException; + +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/transport/DatagramConnection.java b/util/src/main/java/com/epam/deltix/util/vsocket/transport/DatagramConnection.java new file mode 100644 index 00000000..ecc77013 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/transport/DatagramConnection.java @@ -0,0 +1,116 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket.transport; + +import com.epam.deltix.util.vsocket.TransportType; +import com.epam.deltix.util.vsocket.VSProtocol; +import com.epam.deltix.util.vsocket.VSocket; +import com.epam.deltix.util.vsocket.VSocketFactory; +import com.epam.deltix.util.vsocket.VSocketImpl; + +import javax.net.ssl.SSLSocketFactory; +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.DatagramSocket; +import java.net.InetAddress; +import java.net.SocketException; +import java.net.UnknownHostException; +import java.util.logging.Level; + +/** + * + */ +public class DatagramConnection implements Connection { + public DatagramSocket socket; + private BufferedInputStream in; + private OutputStream out; + int remotePort = -1; + + public DatagramConnection(DatagramSocket socket, InetAddress address) throws UnknownHostException { + this.socket = socket; + this.in = new BufferedInputStream(new DatagramInputStream(this)); + this.out = new DatagramOutputStream(this, address); + setUpSocket(); + } + +// public DatagramConnection(DatagramSocket socket, InetSocketAddress address) throws UnknownHostException { +// this.socket = socket; +// this.in = new BufferedInputStream(new DatagramInputStream(this)); +// this.out = new DatagramOutputStream(this, address.getAddress()); +// this.remotePort = address.getPort(); +// setUpSocket(); +// } + + private void setUpSocket () { + try { + socket.setSoTimeout (0); + + //socket.setReceiveBufferSize(1 << 14); + //socket.setSendBufferSize(1 << 14); + } catch (SocketException e) { + VSProtocol.LOGGER.log (Level.WARNING, null, e); + } + } + + @Override + public OutputStream getOutputStream() { + return out; + } + + @Override + public BufferedInputStream getInputStream() { + return in; + } + + @Override + public VSocket create(int code) { + return new VSocketImpl(this, code, VSocketFactory.nextSocketNumber()); + } + + @Override + public InetAddress getRemoteAddress() { + return null; + } + + @Override + public void close() { + socket.close(); + } + + @Override + public VSocket create(VSocket stopped) throws IOException { + VSocket socket = create(stopped.getCode()); + stopped.getOutputStream().writeTo(socket.getOutputStream()); + return socket; + } + + @Override + public boolean isLoopback() { + return socket.getInetAddress().isLoopbackAddress(); + } + + @Override + public void setTransportType(TransportType transportType) { + /* not implemented for this class */ + } + + @Override + public void upgradeToSSL(SSLSocketFactory sslSocketFactory) throws IOException { + /* not implemented for this class */ + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/transport/DatagramInputStream.java b/util/src/main/java/com/epam/deltix/util/vsocket/transport/DatagramInputStream.java new file mode 100644 index 00000000..fff2a187 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/transport/DatagramInputStream.java @@ -0,0 +1,91 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket.transport; + +/** + * + */ + +import com.epam.deltix.util.collections.ByteQueue; +import com.epam.deltix.util.memory.DataExchangeUtils; + +import java.io.InputStream; +import java.net.DatagramPacket; +import java.net.DatagramSocket; + +public class DatagramInputStream extends InputStream { + private ByteQueue buffer = new ByteQueue(1024 * 1024); + + private DatagramSocket ds; + private DatagramConnection connection; + + private byte[] data = new byte[1024 * 64]; + + public DatagramInputStream(DatagramConnection connection) { + this.connection = connection; + this.ds = connection.socket; + } + + private boolean fillBuffer() { + + DatagramPacket pack = new DatagramPacket(data, data.length); + try { + ds.receive(pack); + if (connection.remotePort == -1) + connection.remotePort = pack.getPort(); + + } catch (Exception e) { + e.printStackTrace(System.out); + return false; + } + + int s = DataExchangeUtils.readInt(pack.getData(), 0); + //System.out.println("sequence: " + s); + buffer.offer(pack.getData(), 4, pack.getLength() - 4); + + return true; + } + + public boolean markSupported() { + return false; + } + + public int read() { + + if (!fillBuffer()) + return -1; + + return buffer.poll(); + } + + public int read(byte buf[]) { + + return read(buf, 0, buf.length); + } + + public int read(byte buf[], int pos, int len) { + + if (!fillBuffer()) + return -1; + + int count = Math.min(len, buffer.size()); + + buffer.poll(buf, pos, count); + + return count; + } +} \ No newline at end of file diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/transport/DatagramOutputStream.java b/util/src/main/java/com/epam/deltix/util/vsocket/transport/DatagramOutputStream.java new file mode 100644 index 00000000..97667c71 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/transport/DatagramOutputStream.java @@ -0,0 +1,82 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket.transport; + +import com.epam.deltix.util.memory.MemoryDataOutput; + +import java.io.OutputStream; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.InetAddress; + +/** + * + */ +public class DatagramOutputStream extends OutputStream { + private byte[] buffer; + private DatagramSocket ds; + + private int sequence = 0; + private DatagramConnection c; + private MemoryDataOutput out = new MemoryDataOutput(); + DatagramPacket packet = new DatagramPacket(out.getBuffer(), 0, 0); + + public DatagramOutputStream(DatagramConnection connection, InetAddress ip) { + this.c = connection; + + this.ds = connection.socket; + packet.setAddress(ip); + } + + public synchronized void write(int b) { + out.reset(); + out.writeInt(sequence++); + out.writeByte(b); + + packet.setData(out.getBuffer(), 0, out.getSize()); + packet.setPort(c.remotePort); + + try { + ds.send(packet); + } catch (Exception e) { + e.printStackTrace(System.out); + } + } + + public void write(byte buf[]) { + write(buf, 0, buf.length); + } + + public synchronized void write(byte buf[], int pos, int len) { + if (len == 0) + return; + + out.reset(); + out.writeInt(sequence++); + out.write(buf, pos,len); + + packet.setData(out.getBuffer(), 0, out.getSize()); + packet.setPort(c.remotePort); + + try { + ds.send(packet); + //System.out.println("Send packet, size = " + out.getSize()); + } catch (Exception e) { + e.printStackTrace(System.out); + } + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/transport/SocketConnection.java b/util/src/main/java/com/epam/deltix/util/vsocket/transport/SocketConnection.java new file mode 100644 index 00000000..49c105e5 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/transport/SocketConnection.java @@ -0,0 +1,127 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket.transport; + +import com.epam.deltix.util.io.IOUtil; +import com.epam.deltix.util.vsocket.OffHeapIpcSocket; +import com.epam.deltix.util.vsocket.TransportType; +import com.epam.deltix.util.vsocket.VSProtocol; +import com.epam.deltix.util.vsocket.VSocket; +import com.epam.deltix.util.vsocket.VSocketFactory; +import com.epam.deltix.util.vsocket.VSocketImpl; + +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.Socket; +import java.util.logging.Level; + +/** + * + */ +public class SocketConnection implements Connection { + protected Socket socket; + private BufferedInputStream in; + private OutputStream out; + + private TransportType transportType = TransportType.SOCKET_TCP; + + public SocketConnection(Socket socket, BufferedInputStream in, OutputStream out) { + this.socket = socket; + this.in = in; + this.out = out; + } + + @Override + public OutputStream getOutputStream() { + return out; + } + + @Override + public BufferedInputStream getInputStream() { + return in; + } + + @Override + public VSocket create(int code) throws IOException { + return create(code, transportType); + } + + @Override + public VSocket create(VSocket stopped) throws IOException { + VSocket s = create(stopped.getCode()); + if (VSProtocol.LOGGER.isLoggable(Level.INFO)) { + VSProtocol.LOGGER.log(Level.INFO, "Restore socket " + s + " from: " + stopped); + } + stopped.getOutputStream().writeTo(s.getOutputStream()); + return s; + } + + private VSocket create(int code, TransportType type) throws IOException { + int socketNumber = VSocketFactory.nextSocketNumber(); + if (type == TransportType.AERON_IPC) + throw new RuntimeException("Legacy version of Aeron IPC is not supported"); + else if (type == TransportType.OFFHEAP_IPC) + return new OffHeapIpcSocket(socket, code, true, socketNumber); + else + return new VSocketImpl(socket, in, out, code, socketNumber); + } + + @Override + public InetAddress getRemoteAddress() { + return socket.getInetAddress(); + } + + @Override + public void close() { + IOUtil.close(socket); + } + + @Override + public boolean isLoopback() { + return socket.getInetAddress().isLoopbackAddress(); + } + + @Override + public void setTransportType(TransportType transportType) { + this.transportType = transportType; + } + + @Override + public void upgradeToSSL(SSLSocketFactory sslSocketFactory) throws IOException { + //upgrade socket + if (sslSocketFactory == null) + throw new IOException("SSLSocketFactory isn't initialized."); + + socket = sslSocketFactory.createSocket( + socket, + socket.getInetAddress().getHostAddress(), + socket.getPort(), false); + + //do handshake + ((SSLSocket) socket).setUseClientMode(false); + ((SSLSocket) socket).startHandshake(); + + //upgrade streams + + in = new BufferedInputStream(socket.getInputStream(), VSocketImpl.INPUT_STREAM_BUFFER_SIZE); + out = socket.getOutputStream(); + } +} diff --git a/util/src/main/java/com/epam/deltix/util/vsocket/transport/SocketConnectionFactory.java b/util/src/main/java/com/epam/deltix/util/vsocket/transport/SocketConnectionFactory.java new file mode 100644 index 00000000..196d1410 --- /dev/null +++ b/util/src/main/java/com/epam/deltix/util/vsocket/transport/SocketConnectionFactory.java @@ -0,0 +1,31 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket.transport; + +import java.io.BufferedInputStream; +import java.io.OutputStream; +import java.net.Socket; + +/** + * + */ +public class SocketConnectionFactory { + + public static Connection createConnection(Socket s, BufferedInputStream bis, OutputStream os) { + return new SocketConnection(s, bis, os); + } +} diff --git a/util/src/main/resources/antlr/SyntheticRule.g4 b/util/src/main/resources/antlr/SyntheticRule.g4 new file mode 100644 index 00000000..34b26d34 --- /dev/null +++ b/util/src/main/resources/antlr/SyntheticRule.g4 @@ -0,0 +1,50 @@ +grammar SyntheticRule; + +SYMBOL : [a-zA-Z_][a-zA-Z0-9_]*; +SYMBOL_FROM_NUM : [0-9]+[a-zA-Z_]+[a-zA-Z0-9_]*; +QUOTED_SYMBOL : '"'~('\r'|'\n'|'"')+'"'; +INT : [0-9]+; +FLOAT : [0-9]+'.'[0-9]+; +WHITESPACE : [ \t\r]+ -> skip; + +PLUS : '+'; +MINUS : '-'; +MULTIPLICATION : '*'; + +input : + syntheticRule EOF + ; + +syntheticRule : + syntheticRule PLUS leg # PlusSyntheticRule + | syntheticRule MINUS leg # MinusSyntheticRule + | leg # LegSyntheticRule + ; + +leg : + unaryRatio MULTIPLICATION symbol # LegLeft + | unarySymbol # LegUnarySymbol + ; + +unarySymbol : + MINUS unarySymbol # UnarySymbolMinus + | PLUS unarySymbol # UnarySymbolPlus + | symbol # ToSymbol + ; + +unaryRatio : + MINUS unaryRatio # UnaryRatioMinus + | PLUS unaryRatio # UnaryRatioPlus + | ratio # ToRatio + ; + +symbol : + SYMBOL + | QUOTED_SYMBOL + | SYMBOL_FROM_NUM + ; + +ratio : + INT + | FLOAT + ; diff --git a/util/src/test/java/com/epam/deltix/qsrv/hf/spi/conn/ReconnectableImplTest.java b/util/src/test/java/com/epam/deltix/qsrv/hf/spi/conn/ReconnectableImplTest.java new file mode 100644 index 00000000..ef1b7617 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/qsrv/hf/spi/conn/ReconnectableImplTest.java @@ -0,0 +1,92 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.qsrv.hf.spi.conn; + +import com.epam.deltix.util.io.IOUtil; +import com.epam.deltix.util.lang.Util; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.logging.Level; + +/** + * Based on deltix.temp.utiltests.DisconnectableTest in QS repo. + * + *

This is not really a test, but more an usage example of ReconnectableImpl.

+ * + *

Run, disconnect from the network, reconnect, and see the output.

+ */ +public class ReconnectableImplTest { + + public static void main(String[] args) throws InterruptedException { + MyReconnector reconnector = new MyReconnector(); + ReconnectableImpl mgr = new ReconnectableImpl(); + mgr.setLogLevel(Level.INFO); + mgr.setReconnector(reconnector); + mgr.connected(); + + while (true) { + reconnector.checkedPing(mgr); + Thread.sleep(3000); + } + } + + private static class MyReconnector implements ReconnectableImpl.Reconnector { + @Override + public boolean tryReconnect(int numAttempts, long timeSinceDisconnected, ReconnectableImpl helper) throws Exception { + ping(); + + System.out.println (" Success"); + helper.connected(); + + return (true); + } + + private void ping() throws IOException { + URL url = new URL("http://www.deltixlab.com"); + + InputStream is = url.openStream(); + + try { + IOUtil.readBytes(is); + } catch (InterruptedException x) { + throw new RuntimeException("unexpected", x); + } finally { + Util.close(is); + } + } + + private void checkedPing(ReconnectableImpl mgr) { + System.out.println ("Checked ping..."); + + if (!mgr.isConnected()) { + System.out.println (" not connected."); + return; + } + + try { + ping(); + System.out.println (" All is well."); + } catch (IOException iox) { + System.out.println (" Lost it!"); + mgr.disconnected(); + mgr.scheduleReconnect(); + } + } + } +} diff --git a/util/src/test/java/com/epam/deltix/util/io/CSVWriterTest.java b/util/src/test/java/com/epam/deltix/util/io/CSVWriterTest.java new file mode 100644 index 00000000..126b5a07 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/io/CSVWriterTest.java @@ -0,0 +1,114 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.io; + +import com.epam.deltix.util.csvx.CSVXReader; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; + +import static org.junit.Assert.assertEquals; + + +public class CSVWriterTest { + + @Test + public void printCellWithComaDefaultSeparator() throws IOException { + + StringWriter out = new StringWriter(); + CSVWriter writer = new CSVWriter(out); + writer.writeCells("line, with escape", "line without escape", "line with, multiple escape, characters"); + String actual = extractBuffer(out); + String expected = "\"line, with escape\",line without escape,\"line with, multiple escape, characters\""; + assertEquals(expected, actual); + writer.writeCells("line\" with other escape", "line without escape", "\"line with, multiple different\" escape, characters"); + actual = extractBuffer(out); + expected = "\"line\"\" with other escape\",line without escape,\"\"\"line with, multiple different\"\" escape, characters\""; + assertEquals(expected, actual); + } + + @Test + public void printCellWithPipeSeparator() throws IOException { + + StringWriter out = new StringWriter(); + CSVWriter writer = new CSVWriter(out, '|'); + writer.writeCells("line| with escape", "line without escape", "line with| multiple escape| characters"); + String actual = extractBuffer(out); + String expected = "\"line| with escape\"|line without escape|\"line with| multiple escape| characters\""; + assertEquals(expected, actual); + writer.writeCells("line\" with other escape", "line with, escape coma", "\"line with, multiple |different\" escape, characters"); + actual = extractBuffer(out); + expected = "\"line\"\" with other escape\"|line with, escape coma|\"\"\"line with, multiple |different\"\" escape, characters\""; + assertEquals(expected, actual); + } + + @Test + public void printCellWithPipeSeparatorWithRider() throws IOException { + + StringWriter out = new StringWriter(); + CSVWriter writer = new CSVWriter(out, '|'); + writer.writeCells("line| with escape", "line without escape", "line with| multiple escape| characters"); + CSVXReader reader = new CSVXReader(new StringReader(extractBuffer(out)), '|', false, "out"); + reader.nextLine(); + assertEquals(3, reader.getCells().length); + } + + @Test + public void printCellWithAdditionalEscapeCharacters() throws IOException { + + StringWriter out = new StringWriter(); + CSVWriter writer = new CSVWriter(out, '|', '"', '\t'); + writer.writeCells("line| with escape", "line with additional\t\t escape", "line with no\n escape new line", + "line with escape\" quote char"); + String actual = extractBuffer(out); + String expected = "\"line| with escape\"|\"line with additional\t\t escape\"|line with no\n escape new line|\"line with escape\"\" quote char\""; + assertEquals(expected, actual); + } + + @Test + public void printCellWithAnotherQuoteCharacter() throws IOException { + + StringWriter out = new StringWriter(); + CSVWriter writer = new CSVWriter(out, '\t', '\''); + writer.writeCells("line| with ,default\" \r\nnot used escapes", "line with \t\t separator", "line wit' quote char"); + String actual = extractBuffer(out); + String expected = "line| with ,default\" \r\nnot used escapes\t'line with \t\t separator'\t'line wit'' quote char'"; + assertEquals(expected, actual); + } + @Test + public void printCellWithEscapeEOL() throws IOException { + + StringWriter out = new StringWriter(); + CSVWriter writer = new CSVWriter(out, '\t'); + writer.writeCells("line\t with separator", "line with \n new line", "line without escape", "line with \r carriage return"); + String actual = extractBuffer(out); + String expected = "\"line\t with separator\"\t\"line with \n new line\"\tline without escape\t\"line with \r carriage return\""; + assertEquals(expected, actual); + CSVXReader reader = new CSVXReader(new StringReader(actual), '\t', false, "out"); + reader.nextLine(); + assertEquals(4, reader.getCells().length); + } + + private String extractBuffer(StringWriter out) { + String result = out.getBuffer().toString(); + out.getBuffer().setLength(0); + return result; + } +} \ No newline at end of file diff --git a/util/src/test/java/com/epam/deltix/util/net/TCPEchoServer.java b/util/src/test/java/com/epam/deltix/util/net/TCPEchoServer.java new file mode 100644 index 00000000..534e4ffc --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/net/TCPEchoServer.java @@ -0,0 +1,209 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.net; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; +import java.nio.channels.spi.SelectorProvider; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.Set; + + +public class TCPEchoServer { + private Selector selector; + + /** + * Accept a new client and set it up for reading. + */ + private void doAccept(SelectionKey sk) + { + ServerSocketChannel server = (ServerSocketChannel)sk.channel(); + SocketChannel clientChannel; + try { + clientChannel = server.accept(); + clientChannel.configureBlocking(false); + + // Register this channel for reading. + SelectionKey clientKey = + clientChannel.register(selector, SelectionKey.OP_READ); + + // Allocate an EchoClient instance and attach it to this selection key. + EchoClient echoClient = new EchoClient(); + clientKey.attach(echoClient); + + InetAddress clientAddress = clientChannel.socket().getInetAddress(); + System.out.println("Accepted connection from " + + clientAddress.getHostAddress() + "."); + } + catch (Exception e) { + System.out.println("Failed to accept new client."); + e.printStackTrace(); + } + + } + + /** + * Read from a client. Enqueue the data on the clients output + * queue and set the selector to notify on OP_WRITE. + */ + private void doRead(SelectionKey sk) + { + SocketChannel channel = (SocketChannel)sk.channel(); + ByteBuffer bb = ByteBuffer.allocate(8192); + int len; + + try { + len = channel.read(bb); + if (len < 0) { + disconnect(sk); + return; + } + } + catch (Exception e) { + System.out.println("Failed to read from client."); + e.printStackTrace(); + return; + } + + // Flip the buffer. + bb.flip(); + + EchoClient echoClient = (EchoClient)sk.attachment(); + echoClient.enqueue(bb); + + // We've enqueued data to be written to the client, we must + // not set interest in OP_WRITE. + sk.interestOps(SelectionKey.OP_READ | SelectionKey.OP_WRITE); + } + + /** + * Called when a SelectionKey is ready for writing. + */ + private void doWrite(SelectionKey sk) + { + SocketChannel channel = (SocketChannel)sk.channel(); + EchoClient echoClient = (EchoClient)sk.attachment(); + LinkedList outq = echoClient.getOutputQueue(); + + ByteBuffer bb = outq.getLast(); + try { + int len = channel.write(bb); + if (len == -1) { + disconnect(sk); + return; + } + + if (bb.remaining() == 0) { + // The buffer was completely written, remove it. + outq.removeLast(); + } + } + catch (Exception e) { + System.out.println("Failed to write to client."); + e.printStackTrace(); + } + + // If there is no more data to be written, remove interest in + // OP_WRITE. + if (outq.size() == 0) { + sk.interestOps(SelectionKey.OP_READ); + } + } + + private void disconnect(SelectionKey sk) + { + SocketChannel channel = (SocketChannel)sk.channel(); + + InetAddress clientAddress = channel.socket().getInetAddress(); + System.out.println(clientAddress.getHostAddress() + " disconnected."); + + try { + channel.close(); + } + catch (Exception e) { + System.out.println("Failed to close client socket channel."); + e.printStackTrace(); + } + } + + private void startServer(String host, int port) throws Exception + { + selector = SelectorProvider.provider().openSelector(); + + // Create non-blocking server socket. + ServerSocketChannel ssc = ServerSocketChannel.open(); + ssc.configureBlocking(false); + + // Bind the server socket to localhost. + InetSocketAddress isa = new InetSocketAddress(host, port); + ssc.socket().bind(isa); + + // Register the socket for select events. + SelectionKey acceptKey = ssc.register(selector, SelectionKey.OP_ACCEPT); + + // Loop forever. + for (;;) { + selector.select(); + Set readyKeys = selector.selectedKeys(); + Iterator i = readyKeys.iterator(); + + while (i.hasNext()) { + SelectionKey sk = (SelectionKey)i.next(); + i.remove(); + + if (sk.isAcceptable()) { + doAccept(sk); + } + if (sk.isValid() && sk.isReadable()) { + doRead(sk); + } + if (sk.isValid() && sk.isWritable()) { + doWrite(sk); + } + } + } + } + + static class EchoClient { + private LinkedList outq = new LinkedList<>(); + + public LinkedList getOutputQueue() + { + return outq; + } + + // Enqueue a ByteBuffer on the output queue. + public void enqueue(ByteBuffer bb) + { + outq.addFirst(bb); + } + } + + public static void main(String[] args) throws Exception { + String host = args[0]; //InetAddress.getLocalHost(); + int port = Integer.parseInt(args[1]); + new TCPEchoServer().startServer(host, port); + } + + +} \ No newline at end of file diff --git a/util/src/test/java/com/epam/deltix/util/parserts/synthetic/Test_SyntheticInstrumentRuleParser.java b/util/src/test/java/com/epam/deltix/util/parserts/synthetic/Test_SyntheticInstrumentRuleParser.java new file mode 100644 index 00000000..db086678 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/parserts/synthetic/Test_SyntheticInstrumentRuleParser.java @@ -0,0 +1,234 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.parserts.synthetic; + +import com.epam.deltix.util.lang.Util; +import com.epam.deltix.util.parsers.synthetic.SyntheticInstrumentParser; +import com.epam.deltix.util.parsers.synthetic.SyntheticInstrumentRule; +import org.junit.Assert; +import org.junit.Test; + +/** + * + */ +public class Test_SyntheticInstrumentRuleParser { + + @Test + public void testSimple () { + assertRule("A+B", "A+B"); + assertRule("AA+BB", "AA+BB"); + assertRule("AAA+BBB", "AAA+BBB"); + assertRule("GEU3+GEZ3+GEH4+GEM4+GEU4+GEZ4+GEH5+GEM5+GEU5+GEZ5+GEH6+GEM6", "GEU3+GEZ3+GEH4+GEM4+GEU4+GEZ4+GEH5+GEM5+GEU5+GEZ5+GEH6+GEM6"); //CME + assertRule("GEU3-2*GEZ3+GEH4", "GEU3-2*GEZ3+GEH4"); + + assertRule("1*2BBH1-1*2BBM1", "1*2BBH1-1*2BBM1"); // ICE + assertRule("1*2BBH1-1*2BBM1", "1*\"2BBH1\"-1*\"2BBM1\""); // ICE + } + + @Test + public void testLeadingSign () { + assertRule("A+B", "+A+B"); + assertRule("-A+B", "-A+B"); + + assertRule("-A+2*B", "-A+2*B"); + assertBadRule("-A", "-A"); + } + + @Test + public void testLeadingZeros () { + assertRule("A+B", "+000001*A+0001*B"); + assertRule("-A-B", "-000001*A-0001*B"); + } + + @Test + public void testSign () { + assertRule("A-B", "A-B"); + assertRule("A-B-C", "A-B-C"); + assertRule("A-B+C", "A-B+C"); + assertRule("-A-B", "-A-B"); + } + + + @Test + public void testSymbolsStartWithDigit () { + assertRule("1*1A+B", "1A+1*B"); + assertRule("A+1*1B", "1*A+1B"); + + assertRule("1*1A+B", "1*1A+1*B"); + assertRule("A+1*1B", "1*A+1*1B"); + + assertRule("1*1A+B", "1*\"1A\"+1*B"); + assertRule("A+1*1B", "1*A+1*\"1B\""); + } + + @Test + public void testMissingStuff () { + assertBadRule("Rule cannot be empty", null); + assertBadRule("Rule cannot be empty", ""); + assertBadRule("Unexpected end of rule: symbol is missing after sign", "-"); + assertBadRule("Unexpected end of rule: symbol is missing after sign", "+"); + assertBadRule("Rule must specify at least two legs", "A"); + assertBadRule("Unexpected end of rule: symbol is missing", "0"); + assertBadRule("Unexpected end of rule: symbol is missing", "1"); + } + + @Test + public void testMultipleSigns () { + assertRule("A+B", "++A+B"); + assertRule("A+B", "A++B"); + + assertRule("A+B", "--A+B"); + assertRule("A+B", "A--B"); + + assertRule("A+B", "--A+B"); + assertRule("A+B", "A--B"); + assertRule("A-B", "A---B"); + assertRule("A+B", "--A+B"); + assertRule("-A+B", "---A+B"); + assertRule("A-B", " A + -B "); + assertRule("-A+2*B-C", "-A+2*B+-C"); + } + + @Test + public void testTerminationWithSign () { + assertBadRule("Unexpected end of rule: symbol is missing after sign", "A+B+"); + assertBadRule("Unexpected end of rule: symbol is missing after sign", "A+B-"); + } + + @Test + public void testSeparatorAfterSymbol() { + assertBadRule("Unexpected character at position 4 (Leg symbol cannot contain '*')", "2*3*A+1B"); + + assertBadRule("Unexpected character at position 2 (Leg symbol cannot contain '*')", "A*1+B"); + assertBadRule("Unexpected character at position 4 (Leg symbol cannot contain '*')", "A+B*1"); + assertBadRule("Unexpected character at position 4 (Leg symbol cannot contain '*')", "1*A*1+B"); + assertBadRule("Unexpected character at position 6 (Leg symbol cannot contain '*')", "A+1*B*1"); + + //assertRule("A+B", "A*1+B"); duplicate test above + //assertRule("A+B", "A+B*1"); duplicate test above + } + + @Test + public void testTerminationWithRatio () { + assertBadRule("Unexpected end of rule: symbol is missing", "A+1"); + assertBadRule("Unexpected end of rule: symbol is missing", "A-1"); + assertBadRule("Unexpected end of rule: symbol is missing", "A+0"); + assertBadRule("Unexpected end of rule: symbol is missing", "A-0"); + } + + @Test + public void testZeroRatio () { + // Risk Rules rely on the fact that leg ratios must not be zero + assertBadRule("Unexpected character at position 2 (Ratio cannot be zero)", "0*A+B"); + assertBadRule("Unexpected character at position 3 (Ratio cannot be zero)", "+0*A+B"); + assertBadRule("Unexpected character at position 3 (Ratio cannot be zero)", "-0*A+B"); + assertBadRule("Unexpected character at position 4 (Ratio cannot be zero)", "A+0*B"); + assertBadRule("Unexpected character at position 10 (Ratio cannot be zero)", "A+0000000*B"); + assertBadRule("Unexpected character at position 10 (Ratio cannot be zero)", "A-0000000*B"); + } + + @Test + public void testDuplicateSymbol () { + // UHF Risk rules rely on the fact that there are no duplicates among legs + assertBadRule("Duplicate leg symbol: \"A\"", "A-A"); + assertBadRule("Duplicate leg symbol: \"A\"", "A+B-A"); + } + + @Test + public void testOtherQQLSyntax() { + assertBadRule ("NGV3+NGV3", "'NGV3'+'NGV3'"); + assertBadRule ("1NGV+2NGV", "'1NGV'+'2NGV'"); + assertBadRule ("NGV3=NGX3", "NGV3=NGX3"); + assertBadRule ("A+NGV3=NGX3+C", "A + (NGV3 = NGX3) + C"); + assertBadRule ("A+NGV3=NGX3+C", "A + NGV3 = NGX3 + C"); + assertRule ("this+is+built-in", "this + is + built-in"); + assertRule ("this+is+built-in", "this + \"is\" + built-\"in\""); + } + + @Test + public void testArithmeticFunctions() { + assertBadRule ("NGV3/NGX3", "NGV3 / NGX3"); + assertBadRule ("NGV3*NGX3", "NGV3 * NGX3"); + assertBadRule ("NGV3*2*NGX3", "NGV3 * 2 * NGX3"); + assertBadRule ("12*NGV3-NGX3", "2*6*NGV3-NGX3"); + } + + @Test + public void testFractionalCoeffs() { + assertRule ("0.7*A-B", "0.7*A-B"); + assertRule ("A-1.5*B", "A-1.5*B"); + } + + @Test + public void testDoubleQuotes() { + assertRule("2*A+1*1B","2*A+1B"); + assertRule ("1*2A+B", "\"2A\"+B"); + assertRule ("BRNFMH0015!-BRNFMH0014!", "\"BRNFMH0015!\"-\"BRNFMH0014!\""); + } + + @Test + public void testParentheses() { + assertBadRule ("A+B-C-D", "(A+B)- (C+D)"); + assertBadRule ("A+B-2*C-2*D", "A+B-2*(C+D))"); + assertBadRule("A+3*B+3*C", "A + 3 * (B + C)"); + } + + //@Test + //public void testUpperCase() { + // assertRule ("Q1+Q2", "q1+q2"); + // assertRule ("q1+Q2", "\"q1\"+q2"); + //} + + @Test + public void testSpaces () { + SyntheticInstrumentRule actualRule = SyntheticInstrumentParser.createParserAndParse("\"BNX FMM0017\"-\"BNX FMZ0019\""); + Assert.assertEquals (2, actualRule.symbols.length); + Assert.assertEquals ("BNX FMM0017", actualRule.symbols[0]); + Assert.assertEquals ("BNX FMZ0019", actualRule.symbols[1]); + + assertBadRule ("", "'BNX FMM0017'-'BNX FMZ0019'"); + + assertRule ("A+B+2*C", " A + B + 2 * C "); + assertRule ("A+B+2*C", " A + B + 2 * C"); + assertRule ("AB CD+EF GH", "\"AB CD\"+\"EF GH\""); + assertRule("GEU3+GEZ3+GEH4+GEM4+GEU4+GEZ4+GEH5+GEM5+GEU5+GEZ5+GEH6+GEM6", + " GEU3 + GEZ3+GEH4+GEM4+GEU4+GEZ4 + GEH5+GEM5+GEU5+ GEZ5+GEH6+ GEM6 "); + } + + private void assertRule(String expectedResult, String inputTextRule) { + try { + SyntheticInstrumentRule actualRule = SyntheticInstrumentParser.createParserAndParse(inputTextRule); + Assert.assertEquals(expectedResult, actualRule.toString()); + } catch (IllegalArgumentException e) { + Assert.fail(String.format("Parser was supposed to successfully finish the work on rule \"%s but it didn't\n%s", inputTextRule, e.getMessage())); + } + } + + private void assertBadRule (String expectedError, String inputTextRule) { + try { + SyntheticInstrumentRule actualRule = SyntheticInstrumentParser.createParserAndParse(inputTextRule); + Assert.fail(String.format("Parser was supposed to fail on rule \"%s\" but it didn't. Instead it produced: %s", + inputTextRule, + actualRule.toString())); + + } catch (IllegalArgumentException e) { + if (!Util.QUIET) + System.out.println(String.format("Rule '%s' is not valid \n%s", inputTextRule, e.getMessage())); + } + } +} diff --git a/util/src/test/java/com/epam/deltix/util/text/CharSequenceParserTest.java b/util/src/test/java/com/epam/deltix/util/text/CharSequenceParserTest.java new file mode 100644 index 00000000..119876ca --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/text/CharSequenceParserTest.java @@ -0,0 +1,226 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.text; + + +import org.junit.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.nio.CharBuffer; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class CharSequenceParserTest { + + @Test + public void testParseDoubleSpecialCase() { + assertEquals(1233451.3124, CharSequenceParser.parseDouble("123,345,1.3124")); + assertEquals(123.34513124, CharSequenceParser.parseDouble("123,345,1.3124e-4")); + assertEquals(-0.0, CharSequenceParser.parseDouble("-4.9E-324")); + assertEquals(0.0, CharSequenceParser.parseDouble("4.9E-324")); + assertEquals(Double.POSITIVE_INFINITY, CharSequenceParser.parseDouble("Inf")); + assertEquals(Double.NEGATIVE_INFINITY, CharSequenceParser.parseDouble("-Inf")); + + //?? + assertEquals(1234.5, CharSequenceParser.parseDouble("1,2,3,4.5")); + assertEquals(123, CharSequenceParser.parseDouble(",123")); + assertEquals(123, CharSequenceParser.parseDouble("123,")); + assertEquals(1234, CharSequenceParser.parseDouble("1,,234")); + assertEquals(1234, CharSequenceParser.parseDouble("1,,234")); + assertEquals(1234, CharSequenceParser.parseDouble("1,,234")); + + assertEquals(0.0, CharSequenceParser.parseDouble("E10")); + assertEquals(0.0, CharSequenceParser.parseDouble("e-10")); + //?? + assertThrows(NumberFormatException.class, () -> CharSequenceParser.parseDouble("1e-2147483648")); +// "0.0000000000000000000000000000000001234567" +// "0.123456789012345678901234567890123456789012345678901234567890" + } + + @ParameterizedTest(name = "parseDouble({0})") + @MethodSource("source") + void testParseDouble(String input) { + double actual = CharSequenceParser.parseDouble(input); + double expected = Double.parseDouble(input); + if (Double.isInfinite(expected)) { + assertTrue(Double.isInfinite(actual)); + } else if (Double.isNaN(expected)) { + assertTrue(Double.isNaN(actual)); + } else { + assertEquals(expected, actual, Math.abs(expected) * 1E-15); + } + } + + @ParameterizedTest(name = "testParseDoubleSubStr({0})") + @MethodSource("source") + void testParseDoubleSubStr(String input) { + String input2 = "str" + input + "end1"; + double expected = Double.parseDouble(input); + double actual = CharSequenceParser.parseDouble(input2, 3, 3 + input.length()); + if (Double.isInfinite(expected)) { + assertTrue(Double.isInfinite(actual)); + } else if (Double.isNaN(expected)) { + assertTrue(Double.isNaN(actual)); + } else { + assertEquals(expected, actual, Math.abs(expected) * 1E-15); + } + } + + + @ParameterizedTest(name = "parseFloat({0})") + @MethodSource("sourceFloat") + void testParseFloat(String input) { + float expected = Float.parseFloat(input); + float actual = CharSequenceParser.parseFloat(input); + if (Float.isInfinite(expected)) { + assertTrue(Float.isInfinite(actual)); + } else if (Float.isNaN(expected)) { + assertTrue(Float.isNaN(actual)); + } else { + assertEquals(expected, actual, Math.abs(expected) * 1e-6f); + } + } + + static Stream source() { + return Stream.of( + Arguments.of("0.0e10"), + Arguments.of("0e-10"), + Arguments.of("0.0"), + Arguments.of("0.0000"), + Arguments.of("-0.0"), + Arguments.of("-0"), + Arguments.of("1"), + Arguments.of("-1"), + Arguments.of("1.12345678901234567"), + Arguments.of("-1.12345678901234567"), + Arguments.of("1.7976931348623157E308"), + Arguments.of("-1.7976931348623157E308"), + Arguments.of("2E308"), + Arguments.of("-2E308"), + Arguments.of("Infinity"), + Arguments.of("-Infinity"), + Arguments.of("1E-400"), + Arguments.of("-1E-400"), + Arguments.of("NaN"), + Arguments.of("+NaN"), + Arguments.of("-NaN"), + Arguments.of("3.1415E10"), + Arguments.of("3.0e10"), + Arguments.of("3.0E66"), + Arguments.of("3.1415E38"), + Arguments.of("3.1415E-38"), + Arguments.of("1234567890987654321.1234567890987654321"), + Arguments.of("1e309"), + Arguments.of("1e-325"), + Arguments.of("1e2147483647"), + Arguments.of("9007199254740993"), + Arguments.of("1.123456"), + Arguments.of("-1.123456"), + Arguments.of("3.1415E-10"), + Arguments.of("3.4028235e+38"), + Arguments.of("-3.4028235e+38"), + Arguments.of("3.1415E39"), + Arguments.of("-3.1415E39"), + Arguments.of("0.00000000000000001234567"), + Arguments.of("123456700000000000000000"), + Arguments.of("0.00000000000000000000000000000000000000000000000000001"), + Arguments.of(String.valueOf(Float.MAX_VALUE)), + Arguments.of(String.valueOf(Float.MIN_VALUE)), + Arguments.of(".123"), + Arguments.of("123."), + Arguments.of(".123e10"), + Arguments.of("-.123"), + Arguments.of("+.123") + ); + } + + @ParameterizedTest(name = "testParseInvDoubleSunStr({0})") + @MethodSource("invalidSource") + void testParseInvDoubleSunStr(String input) { + assertThrows(NumberFormatException.class, () -> CharSequenceParser.parseDouble(input)); + } + + @ParameterizedTest(name = "testParseFloatFail({0})") + @MethodSource("invalidSource") + void testParseDoubleFail(String input) { + assertThrows(NumberFormatException.class, () -> CharSequenceParser.parseFloat(input)); + } + + static Stream invalidSource() { + return Stream.of( + Arguments.of("1Infinity"), + Arguments.of("Infinity1"), + Arguments.of("123Infinity312"), + Arguments.of("123InfinityE312"), + Arguments.of("-Infinity1"), + Arguments.of("-1Infinity"), + Arguments.of("-1E-400Inf"), + Arguments.of("123NaN"), + Arguments.of("NaN123"), + Arguments.of(""), + Arguments.of(" "), + Arguments.of("1e"), + Arguments.of("1E"), + Arguments.of("1e+"), + Arguments.of("1e-"), + Arguments.of("++1"), + Arguments.of("--1"), + Arguments.of("+-1"), + Arguments.of("-+1"), + Arguments.of("1.2.3"), + Arguments.of("1.2e3.4"), + Arguments.of("1.2e3e4"), + Arguments.of("NaNaNa"), + Arguments.of("Infinit"), + Arguments.of("infinity"), + Arguments.of("nan"), + Arguments.of("1.#INF"), + Arguments.of("1.#IND") + ); + } + + + static Stream sourceFloat() { + List collect = source().collect(Collectors.toList()); + collect.add(Arguments.of("0.0000000000000000000000000000000001234567")); + collect.add(Arguments.of("0.123456789012345678901234567890123456789012345678901234567890")); + return collect.stream(); + } + + + @Test + public void testDifferentCharSequences() { + String str = "123.456"; + StringBuilder sb = new StringBuilder(str); + StringBuffer sbuf = new StringBuffer(str); + CharBuffer cb = CharBuffer.wrap(str); + + double expected = 123.456; + assertEquals(expected, CharSequenceParser.parseDouble(str)); + assertEquals(expected, CharSequenceParser.parseDouble(sb)); + assertEquals(expected, CharSequenceParser.parseDouble(sbuf)); + assertEquals(expected, CharSequenceParser.parseDouble(cb)); + } +} \ No newline at end of file diff --git a/util/src/test/java/com/epam/deltix/util/text/DateFormatDetectorTest.java b/util/src/test/java/com/epam/deltix/util/text/DateFormatDetectorTest.java new file mode 100644 index 00000000..6cc20f22 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/text/DateFormatDetectorTest.java @@ -0,0 +1,93 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.text; + +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.util.Objects; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + + +public class DateFormatDetectorTest { + + + @Test + public void getDateTimeFormatStringFor() throws IOException, URISyntaxException { + BufferedReader formatReader = getSourceReader("com/epam/deltix/util/text/dateTimeFormat.txt"); + String line = formatReader.readLine(); + while (line != null) { + String[] split = line.split(","); + String date = split[0]; + String time = split[1]; + String datetime = date + time; + + String datePattern = split[2]; + String timePattern = split[3]; + String datetimePattern = datePattern + timePattern; + + assertEquals(datePattern, DateFormatDetector.getDateFormatStringFor(date)); + assertEquals(timePattern, DateFormatDetector.getTimeFormatStringFor(time)); + assertEquals(datetimePattern, DateFormatDetector.getDateTimeFormatStringFor(datetime)); + line = formatReader.readLine(); + } + } + + @Test + public void getDateTimeFormatStringForNegativeCases() { + String[] invalidInputs = { + "", + "random text", + "2023-11-14T22:13:20.123456Z", // micro mot supported + "2023-11-14T22:13:20.1Z", // partial millis + "22:13:20.12Z", // partial millis + "22:13:20.12345678Z", // partial nanos + "22:13:20.12345678", // partial nanos + "22:13:20.1234567890", // more than nanos + "2023-11-14 22:13:20.123456789123", // more than nanos +// "2023-99-99", +// "25:61:61", +// "2023-11-14T25:00:00", +// "2023-11-14T22:61:00", +// "2023-11-14T22:13:61", +// "11/31/2023", // 31 nov not exist +// "2023/11/14T22:13:20.123456789Z_extra", +// "2023-11-14T22:13:20.123456789_extra", +// "20231114T221320_extra", +// "2023-11-14T22:13:20AMPM", + }; + for (String invalidInput : invalidInputs) { + assertNull(DateFormatDetector.getDateFormatStringFor(invalidInput)); + assertNull(DateFormatDetector.getTimeFormatStringFor(invalidInput)); + assertNull(DateFormatDetector.getDateTimeFormatStringFor(invalidInput)); + } + } + + private BufferedReader getSourceReader(String res) throws URISyntaxException, IOException { + URL resource = getClass().getClassLoader().getResource(res); + File file = new File(Objects.requireNonNull(resource).toURI()); + return Files.newBufferedReader(file.toPath()); + } +} \ No newline at end of file diff --git a/util/src/test/java/com/epam/deltix/util/time/DefaultTimeSourceProviderTest.java b/util/src/test/java/com/epam/deltix/util/time/DefaultTimeSourceProviderTest.java new file mode 100644 index 00000000..542d5677 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/time/DefaultTimeSourceProviderTest.java @@ -0,0 +1,61 @@ +///* +// * Copyright 2021 EPAM Systems, Inc +// * +// * See the NOTICE file distributed with this work for additional information +// * regarding copyright ownership. Licensed under the Apache License, +// * Version 2.0 (the "License"); you may not use this file except in compliance +// * with the License. You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// * License for the specific language governing permissions and limitations under +// * the License. +// */ +// +//package com.epam.deltix.util.time; +// +//import com.epam.deltix.qsrv.hf.pub.TimeSource; +//import org.junit.After; +//import org.junit.Test; +// +//import static org.junit.Assert.assertTrue; +// +//public class DefaultTimeSourceProviderTest { +// +// @After +// public void after() { +// System.clearProperty(DefaultTimeSourceProvider.TIME_SOURCE_SYS_PROP); +// DefaultTimeSourceProvider.unconfigure(); +// } +// +// @Test +// public void testDefault() { +// TimeSource timeSource = DefaultTimeSourceProvider.getTimeSourceForApp("test"); +// assertTrue(timeSource instanceof KeeperTimeSource); +// } +// +// @Test +// public void testConfiguredByProperty() { +// System.setProperty(DefaultTimeSourceProvider.TIME_SOURCE_SYS_PROP, "MonotonicRealTimeSource"); +// TimeSource timeSource = DefaultTimeSourceProvider.getTimeSourceForApp("test"); +// assertTrue(timeSource instanceof MonotonicRealTimeSource); +// } +// +// @Test +// public void testConfiguredExplicitly() { +// DefaultTimeSourceProvider.configure("testConf", MonotonicRealTimeSource.getInstance()); +// TimeSource timeSource = DefaultTimeSourceProvider.getTimeSourceForApp("test"); +// assertTrue(timeSource instanceof MonotonicRealTimeSource); +// } +// +// @Test +// public void testExplicitConfigOverridesProperty() { +// DefaultTimeSourceProvider.configure("testConf", KeeperTimeSource.getInstance()); +// System.setProperty(DefaultTimeSourceProvider.TIME_SOURCE_SYS_PROP, "MonotonicRealTimeSource"); +// TimeSource timeSource = DefaultTimeSourceProvider.getTimeSourceForApp("test"); +// assertTrue(timeSource instanceof KeeperTimeSource); +// } +//} diff --git a/util/src/test/java/com/epam/deltix/util/time/FrequentTimePollingStressTest.java b/util/src/test/java/com/epam/deltix/util/time/FrequentTimePollingStressTest.java new file mode 100644 index 00000000..cc3c3053 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/time/FrequentTimePollingStressTest.java @@ -0,0 +1,268 @@ +///* +// * Copyright 2021 EPAM Systems, Inc +// * +// * See the NOTICE file distributed with this work for additional information +// * regarding copyright ownership. Licensed under the Apache License, +// * Version 2.0 (the "License"); you may not use this file except in compliance +// * with the License. You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// * License for the specific language governing permissions and limitations under +// * the License. +// */ +// +//package com.epam.deltix.util.time; +// +//import com.google.common.util.concurrent.AtomicDouble; +//import deltix.clock.Clock; +//import deltix.clock.Clocks; +//import com.epam.deltix.qsrv.hf.pub.TimeSource; +//import org.HdrHistogram.Histogram; +// +//import java.util.concurrent.CountDownLatch; +//import java.util.concurrent.atomic.AtomicBoolean; +// +///** +// * Attempts to measure time spent on single {@link TimeSource#currentTimeNanos()} call with semi-realistic +// * background load. +// * +// *

To simulate the load runs a fixed number of threads that call {@link TimeSource#currentTimeNanos()} +// * at fixed rate (approximately). +// * +// *

Main thread performs measurements in loop without any delays. +// * +// *

Fixed rate is achieved by execution of dummy code that is expected to have stable execution time. +// * +// *

What to look at? +// *

    +// *
  • Look at mean and fixed percentile latencies. +// *
  • Look at "Actual rate" to detect significant deviations from target rate. +// *
+// *

What to try? +// *

    +// *
  • Run with different number of background threads (including 0) +// *
  • Run with different targetRate (from 50k to 1M) +// *
+// */ +//public class FrequentTimePollingStressTest { +// +// public static void main(String[] args) throws InterruptedException { +// // Settings +// int backgroundThreads = getLongArg(args, 1, 4L).intValue(); +// long targetRate = getLongArg(args, 2, 100_000L); +// long mainMeasurementIterations = getLongArg(args, 3, 200_000_000L); +// boolean enableTimePollFromBackgroundThread = getLongArg(args, 4, 1L) != 0; +// Long manualDummyIterationCount = getLongArg(args, 5, null); +// +// String clockName = args.length > 0 ? args[0] : "MonotonicReal"; +// +// TimeSource timeSource = getSourcedByName(clockName); +// +// System.out.println("Clock name: " + clockName); +// +// System.out.println("Background threads: " + backgroundThreads); +// System.out.println("Target rate: " + targetRate + " calls/s"); +// System.out.println("Main measurement iterations: " + mainMeasurementIterations); +// System.out.println("Enable time poll from background thread: " + enableTimePollFromBackgroundThread); +// +// float dummyIterationCostNs = measureDummyIterationCost(); +// System.out.println("Dummy iteration cost: " + dummyIterationCostNs + " ns"); +// +// float targetCallPeriodNs = 1_000_000_000f / targetRate; +// System.out.println("Target call period: " + targetCallPeriodNs + " ns"); +// +// float approxCallCost1 = measureApproxCallCost(timeSource); +// System.out.println("Call cost estimate 1: " + approxCallCost1 + " ns"); +// float approxCallCost2 = measureApproxCallCost(timeSource); +// System.out.println("Call cost estimate 2: " + approxCallCost2 + " ns"); +// float approxCallCost3 = measureApproxCallCost(timeSource); +// System.out.println("Call cost estimate 3: " + approxCallCost3 + " ns"); +// +// float extraDelayNeeded = Math.max(0, targetCallPeriodNs - approxCallCost3); +// System.out.println("Extra delay needed: " + extraDelayNeeded + " ns"); +// +// long dummyIterationsPerCallEstimate = (long) (extraDelayNeeded / dummyIterationCostNs); +// System.out.println("Estimated dummy iterations per call: " + dummyIterationsPerCallEstimate); +// +// long dummyIterationsPerCall = manualDummyIterationCount != null ? manualDummyIterationCount : dummyIterationsPerCallEstimate; +// System.out.println("Dummy iterations per call: " + dummyIterationsPerCall); +// +// CountDownLatch backgroundWarmedUp = new CountDownLatch(backgroundThreads); +// CountDownLatch backgroundStopped = new CountDownLatch(backgroundThreads); +// AtomicBoolean stopBackground = new AtomicBoolean(false); +// AtomicDouble rateSum = new AtomicDouble(0); +// for (int i = 0; i < backgroundThreads; i++) { +// new Thread(() -> { +// long t0 = Long.MIN_VALUE; +// long count = 0; +// boolean warmup = true; +// while (!stopBackground.get()) { +// long value = enableTimePollFromBackgroundThread ? timeSource.currentTimeNanos() : 1; +// count++; +// if (value == Long.MIN_VALUE) { +// // Should not happen +// System.out.println("Wrong value"); +// } +// dummyOps(dummyIterationsPerCall); +// if (warmup) { +// if (count >= 100_000) { +// backgroundWarmedUp.countDown(); +// count = 0; +// t0 = System.nanoTime(); +// warmup = false; +// } +// } +// } +// long t1 = System.nanoTime(); +// double rate = count * 1_000_000_000d / (t1 - t0); +// rateSum.addAndGet(rate); +// System.out.println("Actual rate: " + rate + " calls/s"); +// backgroundStopped.countDown(); +// }).start(); +// } +// System.out.println("Warming up background threads..."); +// backgroundWarmedUp.await(); +// System.out.println("Starting main measurement..."); +// +// runMeasurement(mainMeasurementIterations, timeSource); +// +// stopBackground.set(true); +// backgroundStopped.await(); +// System.out.println("========="); +// System.out.println("Total background rate estimate: " + ((long) rateSum.get()) + " calls/s"); +// } +// +// private static TimeSource getSourcedByName(String clockName) { +// switch (clockName) { +// case "MonotonicReal": +// return MonotonicRealTimeSource.getInstance(); +// case "RawReal": +// return new RawRealTimeSource(); +// case "Keeper": // Very imprecise data! +// TimeKeeper.setMode(TimeKeeper.Mode.HIGH_RESOLUTION_SYNC_BACK); +// return KeeperTimeSource.getInstance(); +// default: +// throw new IllegalArgumentException("Unknown clock name: " + clockName); +// } +// } +// +// private static void runMeasurement(long mainMeasurementIterations, TimeSource timeSource) { +// long warmupCount = mainMeasurementIterations / 5; +// System.out.println("========="); +// System.out.println("Warmup count: " + warmupCount); +// System.out.println("========="); +// if (mainMeasurementIterations <= warmupCount) { +// throw new IllegalArgumentException("mainMeasurementIterations < warmupCount"); +// } +// +// long totalCount = warmupCount + mainMeasurementIterations; +// +// Histogram seqMsgHistogram = new Histogram(3); +// long startTime = System.currentTimeMillis(); +// long prevValue = timeSource.currentTimeNanos(); +// long startMeasurementNanos = Long.MIN_VALUE; +// for (long i = 0; i < totalCount; i++) { +// long value = timeSource.currentTimeNanos(); +// +// if (i == warmupCount) { +// seqMsgHistogram.reset(); +// System.out.println("Warmup done. Running measurements..."); +// value = timeSource.currentTimeNanos(); +// startMeasurementNanos = value; +// } +// seqMsgHistogram.recordValue(value - prevValue); +// prevValue = value; +// } +// long endMeasurementNanos = prevValue; +// long endTime = System.currentTimeMillis(); +// synchronized (System.out) { +// System.out.println("========="); +// seqMsgHistogram.outputPercentileDistribution(System.out, 1.0); +// System.out.println("========="); +// System.out.println("Mean : " + seqMsgHistogram.getMean()); +// System.out.println("50% : " + seqMsgHistogram.getValueAtPercentile(50)); +// System.out.println("90% : " + seqMsgHistogram.getValueAtPercentile(90)); +// System.out.println("99% : " + seqMsgHistogram.getValueAtPercentile(99)); +// System.out.println("99.9% : " + seqMsgHistogram.getValueAtPercentile(99.9)); +// System.out.println("99.99% : " + seqMsgHistogram.getValueAtPercentile(99.99)); +// System.out.println("99.999% : " + seqMsgHistogram.getValueAtPercentile(99.999)); +// System.out.println("99.9999% : " + seqMsgHistogram.getValueAtPercentile(99.9999)); +// System.out.println("========="); +// System.out.println("Measurement took: " + (endTime - startTime) + " ms"); +// System.out.println("Main thread rate: " + (mainMeasurementIterations * 1_000_000_000 / (endMeasurementNanos - startMeasurementNanos)) + " calls/s"); +// System.out.println("========="); +// } +// } +// +// /** +// * Nanoseconds per dummy iteration. +// */ +// private static float measureDummyIterationCost() { +// int measurementCount = 10; +// int dummyIterationsPerMeasurement = 1_000_000_000; +// +// float[] measurements = new float[measurementCount]; +// for (int i = 0; i < measurementCount; i++) { +// long t0 = System.nanoTime(); +// dummyOps(dummyIterationsPerMeasurement); +// long t1 = System.nanoTime(); +// float nsPerIteration = ((float)(t1 - t0)) / dummyIterationsPerMeasurement; +// measurements[i] = nsPerIteration; +// System.out.println("Dummy iteration cost: " + nsPerIteration + " ns"); +// } +// // Avg on last 3 measurements +// return (measurements[measurementCount - 3] + measurements[measurementCount - 2] + measurements[measurementCount - 1]) / 3; +// } +// +// // Non-precise estimate of tested cost, so we can calculate right delay between calls to get the right rate. +// private static float measureApproxCallCost(TimeSource timeSource) { +// long val = 0; +// long count = 1_000_000; +// long t0 = System.nanoTime(); +// for (int i = 0; i < count; i++) { +// val = val | timeSource.currentTimeNanos(); +// } +// long t1 = System.nanoTime(); +// return ((float)(t1 - t0)) / count; +// } +// +// private static void dummyOps(long iterations) { +// long val = 0; +// for (long i = 0; i < iterations; i++) { +// val = ((val & ((1L << 60) - 1)) * 31 + i * 29); +// } +// if (val == Long.MAX_VALUE) { +// // Should not happen +// System.out.println("Error in dummy ops"); +// } +// } +// +// private static Long getLongArg(String[] args, int pos, Long defaultValue) { +// if (args.length > pos) { +// return Long.parseLong(args[pos]); +// } else { +// return defaultValue; +// } +// } +// +// /** +// * Non-monotonic real time source. For test use only. Most TimeBase-related use cases require monotonic time source. +// */ +// private static class RawRealTimeSource implements TimeSource { +// private static final Clock clock = Clocks.REALTIME; +// +// @Override +// public long currentTimeMillis() { +// throw new UnsupportedOperationException(); +// } +// +// @Override +// public long currentTimeNanos() { +// return clock.time(); +// } +// } +//} diff --git a/util/src/test/java/com/epam/deltix/util/time/TimerFreqTest.java b/util/src/test/java/com/epam/deltix/util/time/TimerFreqTest.java new file mode 100644 index 00000000..769bddf6 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/time/TimerFreqTest.java @@ -0,0 +1,65 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.time; + +import java.util.concurrent.locks.LockSupport; + +/** + * This test is used to measure effect of JDK-6435126 + * and solution that is described there on Windows. + */ +public class TimerFreqTest { + + private static final boolean ENABLE_WINDOWS_TIMER_RESOLUTION_FIX = true; + + public static void main(String[] args) throws InterruptedException { + // Without the fix each outer loop produces 10000+ ms results on Windows + // With fix each outer loop produces 1000-2000 ms results on Windows + + if (ENABLE_WINDOWS_TIMER_RESOLUTION_FIX) { + magicFix(); + } + + while (true) { + long t0 = System.nanoTime(); + for (int i = 0; i < 1000; i++) { + LockSupport.parkNanos (1_000_000); // 1 ms + //Thread.sleep(1); + } + long t1 = System.nanoTime(); + System.out.println((t1 - t0) / 1_000_000d + " ms"); + } + } + + private static void magicFix() { + Thread magic = + new Thread("Windows System Clock Speeder-Upper") { + @Override + @SuppressWarnings("SleepWhileInLoop") + public void run() { + for (; ; ) { + try { + Thread.sleep(Integer.MAX_VALUE); + } catch (InterruptedException ex) { + } + } + } + }; + magic.setDaemon(true); + magic.start(); + } +} diff --git a/util/src/test/java/com/epam/deltix/util/vsocket/TestServerSocketFactoryNio.java b/util/src/test/java/com/epam/deltix/util/vsocket/TestServerSocketFactoryNio.java new file mode 100644 index 00000000..188e63f0 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/vsocket/TestServerSocketFactoryNio.java @@ -0,0 +1,177 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.vsocket.util.SocketTestUtilities; +import com.epam.deltix.util.io.IOUtil; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketException; + +public class TestServerSocketFactoryNio { + public static abstract class ServerThread extends Thread { + protected ServerSocket ss; + + public int getLocalPort(){ + return ss.getLocalPort(); + } + public String getLocalHost(){ + return ss.getInetAddress().getHostAddress(); + } + + protected void setUtSocket(ServerSocket socket) throws SocketException { + socket.setReuseAddress(true); + socket.setSoTimeout (0); + } + } + + public static class ThroughputServerSocket extends ServerThread { + private final byte [] buffer; + + public ThroughputServerSocket (int port, int bufferCapacity) throws IOException { + ss = new ServerSocket (port); + setUtSocket(ss); + + buffer = new byte [bufferCapacity]; + } + + @Override + public void run () { + Socket s = null; + + try { + System.out.println ("Server: listening on port " + ss.getLocalPort ()); + for(;;) { + s = ss.accept (); // Only one accept is handled + + DataInputStream in = new DataInputStream (s.getInputStream ()); + System.out.println ("Server: connection accepted."); + + while(s.isConnected()) + { + in.read(buffer); + //in.readFully (buffer); + } + } + } catch (Throwable x) { + x.printStackTrace (); + } finally { + IOUtil.close (s); + IOUtil.close (ss); + } + } + } + + public static class LatencyServerSocket extends ServerThread { + public static final int NUM_MESSAGES = 10000; + public static final int NUM_PER_BURST = 100; + + private final byte[] buffer; + + public LatencyServerSocket(int port, int packetSize) throws IOException { + ss = new ServerSocket(port); + setUtSocket(ss); + + buffer = new byte[packetSize]; + } + + @Override + public void run() { + Socket s = null; + System.out.println("Server listening on port " + ss.getLocalPort() + "; packet size: " + buffer.length); + + try { + s = ss.accept(); // Only one accept is handled + System.out.println("Server: connection accepted."); + + DataInputStream in = new DataInputStream(s.getInputStream()); + DataOutputStream out = new DataOutputStream(s.getOutputStream()); + + SocketTestUtilities.proccessLatencyRequests(out, in, buffer, false); + } catch (Throwable x) { + x.printStackTrace(); + } finally { + IOUtil.close(s); + } + + + IOUtil.close(ss); + } + } + + public static class EchoServerSocket extends ServerThread { + + public EchoServerSocket (int port) throws IOException { + ss = new ServerSocket (port); + setUtSocket(ss); + } + + @Override + public void run () { + Socket s = null; + + try { + System.out.println ("Server: listening on port " + ss.getLocalPort ()); + + for(;;) + { + s = ss.accept (); // Only one accept is handled + + DataInputStream in = new DataInputStream (s.getInputStream ()); + DataOutputStream out = new DataOutputStream (s.getOutputStream ()); + + System.out.println ("Server: connection accepted."); + + + String utfString = in.readUTF (); + out.writeUTF (utfString); + out.flush (); + } + } catch (Throwable x) { + x.printStackTrace (); + } finally { + IOUtil.close (s); + IOUtil.close (ss); + } + } + } + + public static ServerThread createThroughputServerSocket(int port, int packetSize) throws IOException { + ThroughputServerSocket serverSocket = new ThroughputServerSocket(port, packetSize); + serverSocket.setDaemon(true); + serverSocket.setPriority(Thread.MAX_PRIORITY); + return serverSocket; + } + + public static ServerThread createLatencyServerSocket(int port, int packetSize) throws IOException { + LatencyServerSocket serverSocket = new LatencyServerSocket(port, packetSize); + serverSocket.setDaemon(true); + serverSocket.setPriority(Thread.MAX_PRIORITY); + return serverSocket; + } + + public static ServerThread createEchoServerSocket(int port) throws IOException { + EchoServerSocket serverSocket = new EchoServerSocket(port); + serverSocket.setDaemon(true); + serverSocket.setPriority(Thread.MAX_PRIORITY); + return serverSocket; + } +} diff --git a/util/src/test/java/com/epam/deltix/util/vsocket/Test_ChannelExecutor.java b/util/src/test/java/com/epam/deltix/util/vsocket/Test_ChannelExecutor.java new file mode 100644 index 00000000..273b7813 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/vsocket/Test_ChannelExecutor.java @@ -0,0 +1,212 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.lang.DisposableListener; +import com.epam.deltix.util.vsocket.ChannelExecutor; +import org.jetbrains.annotations.Nullable; +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class Test_ChannelExecutor { + + @Test(timeout = 10_000) + public void testFlush() throws InterruptedException { + ChannelExecutor channelExecutor = ChannelExecutor.createNonSharedTestInstance(null); + + StubVSChannel channel1 = new StubVSChannel(false); + StubVSChannel channel2 = new StubVSChannel(true); + channelExecutor.addChannel(channel1); + channelExecutor.addChannel(channel2); + + while (!channel2.flushed) { + Thread.sleep(1); + } + + assertFalse(channel1.flushed); + assertTrue(channel2.flushed); + + channelExecutor.shutdown(); + } + + private static class StubVSChannel implements VSChannel { + private final boolean noDelay; + private final StubVSOutputStream stubVSOutputStream = new StubVSOutputStream(); + private volatile boolean flushed = false; + + public StubVSChannel(boolean noDelay) { + this.noDelay = noDelay; + } + + @Override + public int getLocalId() { + return 0; + } + + @Override + public int getRemoteId() { + return 0; + } + + @Override + public String getRemoteAddress() { + return ""; + } + + @Override + public String getClientAddress() { + return ""; + } + + @Override + public String getRemoteApplication() { + return ""; + } + + @Override + public String getClientId() { + return ""; + } + + @Override + public VSOutputStream getOutputStream() { + return stubVSOutputStream; + } + + @Override + public DataOutputStream getDataOutputStream() { + return null; + } + + @Override + public InputStream getInputStream() { + return null; + } + + @Override + public DataInputStream getDataInputStream() { + return null; + } + + @Override + public VSChannelState getState() { + return VSChannelState.Connected; + } + + @Override + public boolean setAutoflush(boolean value) { + return false; + } + + @Override + public boolean isAutoflush() { + return false; + } + + @Override + public void close(boolean terminate) { + + } + + @Override + public void setAvailabilityListener(Runnable lnr) { + + } + + @Override + public Runnable getAvailabilityListener() { + return null; + } + + @Override + public boolean getNoDelay() { + return noDelay; + } + + @Override + public void setNoDelay(boolean value) { + + } + + @Override + public String encode(String value) { + return ""; + } + + @Override + public String decode(String value) { + return ""; + } + + @Override + public void addDisposableListener(DisposableListener listener) { + + } + + @Override + public void removeDisposableListener(DisposableListener listener) { + + } + + @Nullable + @Override + public String getTag() { + return ""; + } + + @Override + public void setTag(@Nullable String tag) { + + } + + @Override + public void close() { + + } + + private class StubVSOutputStream extends VSOutputStream { + @Override + public void enableFlushing() throws IOException { + + } + + @Override + public void disableFlushing() { + + } + + @Override + public int flushAvailable(boolean flushAll) throws IOException { + StubVSChannel.this.flushed = true; + return 0; + } + + @Override + public void write(int b) throws IOException { + + } + } + } +} diff --git a/util/src/test/java/com/epam/deltix/util/vsocket/Test_ChannelOutputStream.java b/util/src/test/java/com/epam/deltix/util/vsocket/Test_ChannelOutputStream.java new file mode 100644 index 00000000..0afd8807 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/vsocket/Test_ChannelOutputStream.java @@ -0,0 +1,102 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.vsocket.VSChannelImpl; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; + +public class Test_ChannelOutputStream { + + /** + * Ensures that we do not block if we can't flush data and buffer is not yet full (<75%) + * on {@link ChannelOutputStream#enableFlushing()}. + */ + @Test + public void testNoFlushOnPartiallyFull() throws IOException, InterruptedException { + VSChannelImpl vsChannel = Mockito.mock(VSChannelImpl.class); + ChannelOutputStream cos = new ChannelOutputStream(vsChannel, 8 * 1024); + cos.disableFlushing(); + byte[] buf = new byte[1024]; + // 5kb is less than 75% of 8kb + for (int i = 1; i <= 5; i++) { + cos.write(buf); + } + // No flush, because flushing is disabled + Mockito.verifyNoInteractions(vsChannel); + + cos.enableFlushing(); + // No flush, because remote capacity is 0 + Mockito.verifyNoInteractions(vsChannel); + + cos.addAvailableCapacity(8*1024); + cos.disableFlushing(); + cos.enableFlushing(); + // Data flushed + verify(vsChannel).send(Mockito.any(), eq(0), eq(5 * 1024)); + } + + + /** + * Ensures that we block if we can't flush data and buffer is almost full (>75%) + * on {@link ChannelOutputStream#enableFlushing()}. + */ + @Test + public void testBlockingFlushOnAlmostFull() throws IOException, InterruptedException, ExecutionException, TimeoutException { + VSChannelImpl vsChannel = Mockito.mock(VSChannelImpl.class); + ChannelOutputStream cos = new ChannelOutputStream(vsChannel, 8 * 1024); + cos.disableFlushing(); + byte[] buf = new byte[1024]; + // 7kb is more than 75% of 8kb + for (int i = 1; i <= 7; i++) { + cos.write(buf); + } + // No flush, because flushing is disabled + Mockito.verifyNoInteractions(vsChannel); + + // We are expected to block here + CompletableFuture future = CompletableFuture.runAsync(() -> { + try { + cos.enableFlushing(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + Thread.sleep(100); + assertFalse("We expect enableFlushing() to be blocked yet", future.isDone()); + + // Add capacity + cos.addAvailableCapacity(8*1024); + future.get(100, TimeUnit.MILLISECONDS); + assertTrue("enableFlushing() is expected to finish", future.isDone()); + + // Data flushed + verify(vsChannel).send(Mockito.any(), eq(0), eq(7 * 1024)); + } +} diff --git a/util/src/test/java/com/epam/deltix/util/vsocket/Test_ClientReconnect.java b/util/src/test/java/com/epam/deltix/util/vsocket/Test_ClientReconnect.java new file mode 100644 index 00000000..b2e365d7 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/vsocket/Test_ClientReconnect.java @@ -0,0 +1,431 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.vsocket; + +import com.epam.deltix.gflog.api.Log; +import com.epam.deltix.gflog.api.LogFactory; +import com.epam.deltix.gflog.jul.JulBridge; +import com.epam.deltix.qsrv.hf.spi.conn.DisconnectEventListener; +import com.epam.deltix.util.annotations.Duration; +import com.epam.deltix.util.lang.Util; +import com.epam.deltix.util.vsocket.util.TestVServerSocketFactory; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.RepeatedTest; +import org.junit.jupiter.api.Timeout; +import org.netcrusher.NetFreezer; +import org.netcrusher.core.reactor.NioReactor; +import org.netcrusher.tcp.TcpCrusher; +import org.netcrusher.tcp.TcpCrusherBuilder; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BooleanSupplier; +import java.util.function.Consumer; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Contains tests that use an intermediate proxy to simulate network issues between {@link VSClient} and server. + */ +public class Test_ClientReconnect { + + + // Warning: having more than 120 transports may lead to Gradle test instability because + // of insufficient off-heap buffer capacity for all connections. + private static final int RECOVERY_TEST_TRANSPORTS = Integer.parseInt(System.getProperty("Test_ClientReconnect.transports", "100")); + + // Linger interval is 10 seconds + 5 seconds for notification delays (especially on CI) + @Duration(timeUnit = TimeUnit.SECONDS) + private static final int DISCONNECT_WAIT_TIMEOUT = 15; + + static { + JulBridge.install(); + } + + private static final Log LOG = LogFactory.getLog(Test_ClientReconnect.class); + + public static final int PROXY_PORT = 34_781; + public static final int EMBEDDED_SERVER_PORT = 35_782; + + + private VSServer server; + private NioReactor nioReactor; + private TcpCrusher tcpCrusher; + private NetFreezer acceptorFreezer; + private final AtomicInteger connectCounter = new AtomicInteger(0); + + private volatile Consumer proxyConnectListener = null; + + + @BeforeEach + public void start() throws Throwable { + this.server = TestVServerSocketFactory.createBinaryEchoVServer(EMBEDDED_SERVER_PORT); + this.server.setTransportsLimit((short) 1000); // Allow many transports for reconnect tests + this.server.start(); + + int serverPort = server.getLocalPort(); + String serverHost = "localhost"; + + this.nioReactor = new NioReactor(); + + this.tcpCrusher = TcpCrusherBuilder.builder() + .withReactor(nioReactor) + .withBindAddress("localhost", PROXY_PORT) + .withConnectAddress(serverHost, serverPort) + //.withBacklog(1) + .withCreationListener(clientAddress -> { + int connectionNumber = connectCounter.incrementAndGet(); + LOG.info("Proxy: Client %s connected: %s").with(connectionNumber).with(clientAddress); + if (proxyConnectListener != null) { + proxyConnectListener.accept(clientAddress); + } + }) + .buildAndOpen(); + this.acceptorFreezer = tcpCrusher.getAcceptorFreezer(); + } + + @AfterEach + public void stop() { + proxyConnectListener = null; +// if (acceptorFreezer != null && acceptorFreezer.isFrozen()) { +// // Prevent incorrect state on close +// try { +// acceptorFreezer.unfreeze(); +// } catch (RuntimeException e) { +// LOG.warn("Failed to unfreeze acceptor during cleanup: %s").with(e); +// } +// } + + if (tcpCrusher != null) { + tcpCrusher.close(); + } + if (nioReactor != null) { + nioReactor.close(); + } + if (server != null) { + server.close(); + } + } + + @NotNull + private static VSClient connectClient() throws IOException { + return new VSClient("localhost", PROXY_PORT); + } + + private int getConnectedClientCount() { + return tcpCrusher.getClientAddresses().size(); + } + + /** + * Client should not get blocked on connection loss. + */ + @RepeatedTest(1) + @Timeout(20) + public void testConnectionLoss() throws Exception { + VSClient client = connectClient(); + try { + assertEquals(0, getConnectedClientCount()); + client.connect(); + LOG.info("Client connected"); + assertTrue(client.isConnected()); + assertEquals(3, getConnectedClientCount()); + Test_VSocket_Correctness.assertEchoClientCorrectness(client, 1_000); + + CountDownLatch disconnectedLatch = new CountDownLatch(1); + AtomicInteger disconnectCount = new AtomicInteger(0); + long listenerInstallTime = System.currentTimeMillis(); + client.setDisconnectedListener(new DisconnectEventListener() { + @Override + public void onDisconnected() { + LOG.info("Client onDisconnected listener triggered after %s ms") + .with(System.currentTimeMillis() - listenerInstallTime); + disconnectCount.incrementAndGet(); + disconnectedLatch.countDown(); + } + + @Override + public void onReconnected() { + } + }); + + long disconnectStart = System.currentTimeMillis(); + + // Close all connections and disable proxy + tcpCrusher.close(); + + boolean success = disconnectedLatch.await(DISCONNECT_WAIT_TIMEOUT, TimeUnit.SECONDS); + long disconnectEnd = System.currentTimeMillis(); + LOG.info("Stopped to wait for disconnected after %s ms").with(disconnectEnd - disconnectStart); + LOG.info("Disconnect event count: %s").with(disconnectCount.get()); + + assertTrue(success, "Client did not receive disconnect event in time"); + // Client is still disconnected + assertFalse(client.isConnected()); + + // Wait for any redundant events + Thread.sleep(10); + assertEquals(1, disconnectCount.get(), "Disconnect event should be fired exactly once"); + } finally { + // TODO: Probably we may want to reconsider this in future and allow graceful client close + // For this test we do not care if client throws exception + Util.close(client); + } + } + + /** + * Client should be able to reconnect after single recoverable connection loss. + */ + @RepeatedTest(1) + @Timeout(200) + public void testReconnectAfterSingleDisconnected() throws Exception { + try (VSClient client = connectClient()) { + client.connect(); + var eventListener = installListener(client); + + // Multiple iterations to ensure that we return to stable state + for (int i = 0; i < 10; i++) { + assertTrue(client.isConnected()); + assertEquals(3, getConnectedClientCount()); + Test_VSocket_Correctness.assertEchoClientCorrectness(client, 1_000); + + // Simulates temporary connection loss for single connection + InetSocketAddress clientSocketAddress = tcpCrusher.getClientAddresses().iterator().next(); + Assertions.assertNotNull(clientSocketAddress); + + boolean closed = tcpCrusher.closeClient(clientSocketAddress); + assertTrue(closed, "Failed to close client connection"); + + Thread.sleep(1000); // Wait for the connection to be closed + + boolean gotDisconnectEvent = eventListener.disconnectedLatch.await(DISCONNECT_WAIT_TIMEOUT, TimeUnit.SECONDS); + assertFalse(gotDisconnectEvent, "Client is not supposed to generate disconnect if it was able to reconnect"); + assertEquals(0, eventListener.disconnectCount.get()); + //Assert.assertEquals(0, eventListener.reconnectCount.get()); + + LOG.info("State: %s").with(client.getDispatcher().getInternalState()); + assertTrue(client.tryGetConnectionStatus()); + assertEquals(3, getConnectedClientCount()); + } + } + } + + /** + * Client should be able to reconnect after connection loss if network is restored. + */ + @RepeatedTest(1) + @Timeout(200) + public void testReconnectAfterAllDisconnected() throws Exception { + try (VSClient client = connectClient()) { + client.connect(); + var eventListener = installListener(client); + + // Multiple iterations to ensure that we return to stable state + for (int i = 0; i < 10; i++) { + assertTrue(client.isConnected()); + assertEquals(3, getConnectedClientCount()); + Test_VSocket_Correctness.assertEchoClientCorrectness(client, 1_000); + + // Simulates temporary connection loss for all transports + tcpCrusher.close(); + tcpCrusher.open(); + + Thread.sleep(1000); // Wait for the connection to be closed + + + boolean gotDisconnectEvent = eventListener.disconnectedLatch.await(DISCONNECT_WAIT_TIMEOUT, TimeUnit.SECONDS); + assertFalse(gotDisconnectEvent, "Client is not supposed to generate disconnect if it was able to reconnect"); + assertEquals(0, eventListener.disconnectCount.get()); + // TODO: Uncomment - currently client fires redundant reconnect event + //Assert.assertEquals(0, eventListener.reconnectCount.get()); + + LOG.info("State: %s").with(client.getDispatcher().getInternalState()); + assertTrue(client.tryGetConnectionStatus()); + assertEquals(3, getConnectedClientCount()); + } + } + // Should be closed gracefully + } + + /** + * Ensure that if client is closed during disconnect event, it does not get stuck. + */ + @RepeatedTest(1) + @Timeout(20) + public void testCloseOnDisconnect() throws Exception { + try (VSClient client = connectClient()) { + client.connect(); + + assertTrue(client.isConnected()); + assertEquals(3, getConnectedClientCount()); + Test_VSocket_Correctness.assertEchoClientCorrectness(client, 1_000); + + CountDownLatch disconnectedLatch = new CountDownLatch(1); + AtomicInteger disconnectCount = new AtomicInteger(0); + client.setDisconnectedListener(new DisconnectEventListener() { + @Override + public void onDisconnected() { + LOG.info("Client disconnected"); + disconnectCount.incrementAndGet(); + client.close(); + disconnectedLatch.countDown(); + } + + @Override + public void onReconnected() { + } + }); + + // Simulates connection loss for all transports + tcpCrusher.close(); + + disconnectedLatch.await(); + + // Wait for any redundant events + Thread.sleep(10); + assertEquals(1, disconnectCount.get(), "Disconnect event should be fired exactly once"); + } + } + + /** + * Starts with 1000 connections, kills them, recovers only some of them. + * Expected to end up in DISCONNECTED state. + */ + @RepeatedTest(1) + //@Test + @Timeout(60) + public void testPartialRecovery() throws Exception { + int transports = RECOVERY_TEST_TRANSPORTS; + + int halfTransports = transports / 2; + if (halfTransports == 0) { + throw new IllegalStateException("Number of transports is too low for this test"); + } + + try (VSClient client = connectClient()) { + client.setNumTransportChannels(transports); + client.connect(); + + waitUntil(10_000, "Client is not connected in time", client::isConnected); + + assertEquals(transports, getConnectedClientCount()); + LOG.info("Client connected with %s transports").with(transports); + Test_VSocket_Correctness.assertEchoClientCorrectness(client, 1_000); + + List initialConnections = new ArrayList<>(tcpCrusher.getClientAddresses()); + + + // Allow only one reconnection + AtomicInteger reconnectedCount = new AtomicInteger(0); + proxyConnectListener = (inetSocketAddress) -> { + // Warning: this listener is asynchronous, so it's not guaranteed that exactly 500 connections will fail + + int newCount = reconnectedCount.incrementAndGet(); + + if (newCount == halfTransports) { + // Disable new connections + acceptorFreezer.freeze(); + } + }; + + LOG.info("Killing initial connections"); + for (InetSocketAddress address : initialConnections) { + tcpCrusher.closeClient(address); + } + + VSDispatcher[] dispatchers = server.getDispatchers(); + assertEquals(1, dispatchers.length); + VSDispatcher dispatcher = dispatchers[0]; + + // Wait until dispatcher detects broken transport + LOG.info("Waiting for dispatcher to detect disconnection"); + long now = System.currentTimeMillis(); + long start = now; + long deadline = now + 20_000; + while ((now = System.currentTimeMillis()) < deadline) { + VSDispatcherState state = dispatcher.getInternalState(); + if (state != VSDispatcherState.DISCONNECTED) { + Thread.sleep(100); + } else { + break; + } + } + LOG.info("Waited %s ms for dispatcher to detect disconnection").with(now - start); + + LOG.info("Dispatcher state: %s").with(dispatcher.getInternalState()); + + // Loss of any transport must cause dispatcher to go to DISCONNECTED state + assertEquals(VSDispatcherState.DISCONNECTED, dispatcher.getInternalState()); + } + } + + static void waitUntil(int timeoutMs, String errorMessage, BooleanSupplier condition) { + long startTime = System.currentTimeMillis(); + long endTime = startTime + timeoutMs; + while (System.currentTimeMillis() < endTime) { + if (condition.getAsBoolean()) { + // Success + return; + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted while waiting", e); + } + } + fail(errorMessage); + } + + static TestEventListener installListener(VSClient client) { + TestEventListener eventListener = new TestEventListener(); + client.setDisconnectedListener(eventListener); + return eventListener; + } + + static class TestEventListener implements DisconnectEventListener { + CountDownLatch disconnectedLatch = new CountDownLatch(1); + CountDownLatch reconnectedLatch = new CountDownLatch(1); + AtomicInteger disconnectCount = new AtomicInteger(0); + AtomicInteger reconnectCount = new AtomicInteger(0); + + public TestEventListener() { + } + + @Override + public void onDisconnected() { + LOG.info("Client disconnected"); + disconnectCount.incrementAndGet(); + disconnectedLatch.countDown(); + } + + @Override + public void onReconnected() { + LOG.info("Client reconnected"); + reconnectCount.incrementAndGet(); + reconnectedLatch.countDown(); + } + } +} diff --git a/util/src/test/java/com/epam/deltix/util/vsocket/Test_SocketEcho.java b/util/src/test/java/com/epam/deltix/util/vsocket/Test_SocketEcho.java new file mode 100644 index 00000000..2cc7a6e6 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/vsocket/Test_SocketEcho.java @@ -0,0 +1,71 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.vsocket.util.SocketTestUtilities; +import com.epam.deltix.util.vsocket.util.TestServerSocketFactory; +import com.epam.deltix.util.io.IOUtil; +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.Socket; + +/** + * + */ +public class Test_SocketEcho { + public static void main (String [] args) throws Exception { + int port = SocketTestUtilities.parsePort(args); + + TestServerSocketFactory.ServerThread server = TestServerSocketFactory.createEchoServerSocket(port); + server.start (); + + client ("localhost", server.getLocalPort()); + } + + public static void client (String host, int port) + throws IOException { + String utfString = "Hello world"; + + Socket s = null; + try { + s = new Socket(host, port); + DataOutputStream os = new DataOutputStream(s.getOutputStream()); + DataInputStream is = new DataInputStream(s.getInputStream()); + + os.writeUTF(utfString); + os.flush(); + + String readStr = is.readUTF(); + if (!readStr.equals(utfString)) + throw new AssertionError(readStr + " != " + utfString); + + } catch (Throwable x) { + x.printStackTrace(); + throw x; + } finally { + IOUtil.close(s); + } + } + + @Test + public void TestSocket() throws Exception { + Test_SocketEcho.main(new String[0]); + } +} diff --git a/util/src/test/java/com/epam/deltix/util/vsocket/Test_SocketLatency.java b/util/src/test/java/com/epam/deltix/util/vsocket/Test_SocketLatency.java new file mode 100644 index 00000000..24cdc344 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/vsocket/Test_SocketLatency.java @@ -0,0 +1,80 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.vsocket.util.SocketTestUtilities; +import com.epam.deltix.util.vsocket.util.TestServerSocketFactory; +import com.epam.deltix.util.io.IOUtil; +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.Socket; + +/** + * Date: Mar 1, 2010 + */ +public class Test_SocketLatency { + public static void main (String [] args) throws Exception { + int packetSize = SocketTestUtilities.parsePacketSize(args, 1024); + int port = SocketTestUtilities.parsePort(args); + + TestServerSocketFactory.ServerThread server = TestServerSocketFactory.createLatencyServerSocket(port, packetSize); + server.start (); + + client ("localhost", server.getLocalPort(), packetSize, 10, true); + } + + public static void client (String host, int port, int packetSize, int cycles, boolean measure) + throws IOException, InterruptedException { + byte[] buffer = new byte[packetSize]; + + for (int ii = 0; ii < packetSize; ii++) + buffer[ii] = (byte) ii; + + Socket socket = null; + try { + socket = new Socket(host, port); + socket.setTcpNoDelay(true); + socket.setSoTimeout(0); + socket.setKeepAlive(true); + + socket.setReuseAddress(true); + + OutputStream out = socket.getOutputStream(); + DataInputStream in = new DataInputStream(socket.getInputStream()); + + SocketTestUtilities.measureLatency(out, in, buffer, cycles, measure); + + } catch (Throwable x) { + x.printStackTrace(); + throw x; + } finally { + IOUtil.close(socket); + } + + Thread.sleep(100); + } + + @Test + public void TestSocket() throws Exception { + Test_SocketLatency.main(new String[0]); + } +} + + diff --git a/util/src/test/java/com/epam/deltix/util/vsocket/Test_SocketThroughput.java b/util/src/test/java/com/epam/deltix/util/vsocket/Test_SocketThroughput.java new file mode 100644 index 00000000..1d13d7cb --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/vsocket/Test_SocketThroughput.java @@ -0,0 +1,76 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.vsocket.util.SocketTestUtilities; +import com.epam.deltix.util.vsocket.util.TestServerSocketFactory; +import com.epam.deltix.util.io.IOUtil; +import org.junit.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.Socket; + +/** + * + */ +public class Test_SocketThroughput { + + public static void main (String [] args) throws IOException { + int packetSize = SocketTestUtilities.parsePacketSize(args); + int port = SocketTestUtilities.parsePort(args); + + TestServerSocketFactory.ServerThread server = TestServerSocketFactory.createThroughputServerSocket(port, packetSize); + server.start (); + + client ("localhost", server.getLocalPort(), packetSize, 15); + } + + public static void client (String host, int port, int packetSize, int cycle) + throws IOException { + byte[] buffer = new byte[packetSize]; + + for (int ii = 0; ii < packetSize; ii++) + buffer[ii] = (byte) ii; + + Socket s = null; + try { + s = new Socket(); + s.connect(new InetSocketAddress(host, port), 5000); + s.setTcpNoDelay (true); + s.setSoTimeout (0); + s.setKeepAlive (true); + s.setReceiveBufferSize(1 << 16); + s.setSendBufferSize(1 << 16); + + OutputStream os = s.getOutputStream(); + + SocketTestUtilities.measureOutputThroughput(os, buffer, cycle); + } catch (Throwable x) { + x.printStackTrace(); + throw x; + } finally { + IOUtil.close(s); + } + } + + @Test + public void TestSocket() throws IOException{ + Test_SocketThroughput.main(new String[0]); + } +} \ No newline at end of file diff --git a/util/src/test/java/com/epam/deltix/util/vsocket/Test_SocketThroughput_Nio.java b/util/src/test/java/com/epam/deltix/util/vsocket/Test_SocketThroughput_Nio.java new file mode 100644 index 00000000..b8248c5e --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/vsocket/Test_SocketThroughput_Nio.java @@ -0,0 +1,76 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.vsocket.util.SocketTestUtilities; +import com.epam.deltix.util.vsocket.util.TestServerSocketFactory; +import com.epam.deltix.util.io.IOUtil; +import org.junit.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.Socket; + +/** + * + */ +public class Test_SocketThroughput_Nio { + + public static void main (String [] args) throws IOException { + int packetSize = SocketTestUtilities.parsePacketSize(args); + int port = SocketTestUtilities.parsePort(args); + + TestServerSocketFactory.ServerThread server = TestServerSocketFactory.createThroughputServerSocket(port, packetSize); + server.start (); + + client ("localhost", server.getLocalPort(), packetSize, 15); + } + + public static void client (String host, int port, int packetSize, int cycle) + throws IOException { + byte[] buffer = new byte[packetSize]; + + for (int ii = 0; ii < packetSize; ii++) + buffer[ii] = (byte) ii; + + Socket s = null; + try { + s = new Socket(); + s.connect(new InetSocketAddress(host, port), 5000); + s.setTcpNoDelay (true); + s.setSoTimeout (0); + s.setKeepAlive (true); + s.setReceiveBufferSize(1 << 16); + s.setSendBufferSize(1 << 16); + + OutputStream os = s.getOutputStream(); + + SocketTestUtilities.measureOutputThroughput(os, buffer, cycle); + } catch (Throwable x) { + x.printStackTrace(); + throw x; + } finally { + IOUtil.close(s); + } + } + + @Test + public void TestSocket() throws IOException{ + Test_SocketThroughput_Nio.main(new String[0]); + } +} \ No newline at end of file diff --git a/util/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketChannelLeak.java b/util/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketChannelLeak.java new file mode 100644 index 00000000..131dde71 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketChannelLeak.java @@ -0,0 +1,157 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.concurrent.QuickExecutor; +import com.epam.deltix.util.lang.DisposableListener; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Tests if there is a memory leak in VSChannel when channel gets closed on the client side. + * + *

Run the test with -Xmx500m to see the problem. + */ +@SuppressWarnings("NewClassNamingConvention") +public class Test_VSocketChannelLeak { + private static final boolean enableCloseFix = true; + private static final int ITERATIONS = 1000; + private static final int PAYLOAD_SIZE = 1_024 * 1_024; // 1 MB + private static final boolean waitForFreeMem = false; // Needed for CI env - it's slow + + public static void main (String [] args) throws Exception { + testImpl(); + //Thread.sleep(Long.MAX_VALUE); + } + + @Ignore // Fails on CI + @Test(timeout = 60_000) + public void test() throws IOException, InterruptedException { + testImpl(); + } + + @SuppressWarnings("Convert2Lambda") + private static void testImpl() throws IOException, InterruptedException { + VSServer server = new VSServer(0); + + AtomicLong openChannels = new AtomicLong(); + AtomicLong closedChannels = new AtomicLong(); + + ExecutorService executorService = Executors.newCachedThreadPool(); + + server.setConnectionListener(new VSConnectionListener() { + @Override + public void connectionAccepted(QuickExecutor executor, VSChannel serverChannel) { + openChannels.incrementAndGet(); + + // This payload will be kept in memory until the channel is closed + byte[] payload = new byte[PAYLOAD_SIZE]; + payload[0] = 1; + + + + serverChannel.addDisposableListener(new DisposableListener<>() { + @Override + public void disposed(VSChannel resource) { + byte val = payload[11]; + if (val != 0) { + System.out.println("Should never happen"); + } + // Intentionally do not remove the listener from the channel to release the memory only if channel is released + } + }); + + if (enableCloseFix) { + executorService.submit(() -> { + DataInputStream dis = serverChannel.getDataInputStream(); + //noinspection TryFinallyCanBeTryWithResources + try { + while (true) { + try { + dis.readByte(); + } catch (EOFException e) { + // Graceful close + break; + } catch (IOException e) { + break; + } + } + } finally { + serverChannel.close(); // This will release the memory + closedChannels.incrementAndGet(); + } + }); + } + } + }); + server.setDaemon(true); + server.start(); + System.out.println("Server started on " + server.getLocalPort()); + + try { + createConnections("localhost", server.getLocalPort()); + } finally { + System.out.println("Open channels: " + openChannels.get()); + System.out.println("Closed channels: " + closedChannels.get()); + } + + server.close(); + executorService.shutdown(); + } + + public static void createConnections(String host, int port) + throws IOException, InterruptedException { + + VSClient client = new VSClient(host, port); + client.connect(); + + // This loop fails with OutOfMemoryError + for (int i = 0; i < ITERATIONS; i++) { + VSChannel s = client.openChannel(); + s.close(false); + + if (waitForFreeMem && (Runtime.getRuntime().freeMemory() < Runtime.getRuntime().totalMemory() / 5)) { + // Wait if we below 20% of free memory + System.gc(); + Thread.sleep(200); + } else { + Thread.yield(); + } + } + long usedMemory1 = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); + + System.gc(); + Thread.sleep(3000); + + long usedMemory2 = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); + System.out.println("Used memory before gc: " + usedMemory1); + System.out.println("Used memory after gc: " + usedMemory2); + + client.close(); + + if (usedMemory2 > ITERATIONS * PAYLOAD_SIZE) { + throw new RuntimeException("Memory leak detected"); + } + } +} diff --git a/util/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketEcho.java b/util/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketEcho.java new file mode 100644 index 00000000..982d27d1 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketEcho.java @@ -0,0 +1,93 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.vsocket.util.SocketTestUtilities; +import com.epam.deltix.util.vsocket.util.TestVServerSocketFactory; +import com.epam.deltix.util.time.TimeKeeper; + +import java.io.DataOutputStream; +import java.io.IOException; + +public class Test_VSocketEcho { + public static void main (String args []) throws Throwable { + int port = SocketTestUtilities.parsePort(args); + + VSServer server = TestVServerSocketFactory.createEchoVServer(port); + server.setDaemon(true); + server.start(); + System.out.println("Server started on " + server.getLocalPort()); + + client("localhost", server.getLocalPort()); + } + + private static void client(String host, int port) throws IOException { + VSClient client = new VSClient (host, port); + VSChannel channel = null; + String s = "Hello world"; + int counter = 0; + long lastReportTime = TimeKeeper.currentTime; + long nextReportTime = lastReportTime + 1000; + long lastReportedCount = 0; + + try { + client.connect (); + + for (;;) { + channel = client.openChannel (); + + DataOutputStream os = channel.getDataOutputStream (); + os.writeUTF (s); + os.flush (); + + String check = channel.getDataInputStream ().readUTF (); + + if (!s.equals (check)) + throw new AssertionError (check + " != " + s); + + channel.close (); + + counter++; + + long now = TimeKeeper.currentTime; + + if (now > nextReportTime) { + long num = counter - lastReportedCount; + double sec = (now - lastReportTime) * 0.001; + double rate = num / sec; + + System.out.println ((int) rate + " packets/sec"); + + lastReportedCount = counter; + lastReportTime = now; + nextReportTime = now + 1000; + } + } + } catch (IOException x) { + x.printStackTrace (); + } finally { + if (channel != null) + channel.close (); + + client.close (); + } + } + + public void TestSocket() throws Throwable { + Test_VSocketEcho.main(new String[0]); + } +} diff --git a/util/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketInputThroughput.java b/util/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketInputThroughput.java new file mode 100644 index 00000000..837d9e40 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketInputThroughput.java @@ -0,0 +1,64 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.vsocket.util.SocketTestUtilities; +import com.epam.deltix.util.vsocket.util.TestVServerSocketFactory; +import org.junit.Test; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Created by PosudevskiyK on 1/24/2017. + */ +public class Test_VSocketInputThroughput { + public static void main (String [] args) throws Exception { + int port = SocketTestUtilities.parsePort(args); + int packetSize = SocketTestUtilities.parsePacketSize(args); + + VSServer server = TestVServerSocketFactory.createInputThroughputVServer(port, packetSize); + server.setDaemon(true); + server.start(); + System.out.println("Server started on " + server.getLocalPort()); + + client ("localhost", server.getLocalPort(), packetSize, 15); + server.close(); + } + + public static void client (String host, int port, int packetSize, int cycles) + throws IOException { + byte[] buffer = new byte[packetSize]; + + for (int ii = 0; ii < packetSize; ii++) + buffer[ii] = (byte) ii; + + VSClient c = new VSClient(host, port); + c.connect(); + + VSChannel s = c.openChannel(); + InputStream os = s.getInputStream(); + + SocketTestUtilities.measureInputThroughput(os, buffer, cycles); + c.close(); + } + + @Test + public void TestOutputThroughput() throws Throwable { + Test_VSocketInputThroughput.main(new String[0]); + } +} diff --git a/util/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketLatency.java b/util/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketLatency.java new file mode 100644 index 00000000..7d616700 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketLatency.java @@ -0,0 +1,59 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.vsocket.util.SocketTestUtilities; +import com.epam.deltix.util.vsocket.util.TestVServerSocketFactory; +import org.junit.Test; + +import java.io.IOException; + +public class Test_VSocketLatency { + public static void main (String [] args) throws Exception { + int port = SocketTestUtilities.parsePort(args); + int packetSize = SocketTestUtilities.parsePacketSize(args, 1024); + + VSServer server = TestVServerSocketFactory.createLatencyVServer(port, packetSize); + server.setDaemon(true); + server.start(); + System.out.println("Server started on " + server.getLocalPort()); + + client ("localhost", server.getLocalPort(), packetSize, 10); + } + + public static void client (String host, int port, int packetSize, int cycles) + throws IOException, InterruptedException { + byte[] buffer = new byte[packetSize]; + + for (int ii = 0; ii < packetSize; ii++) + buffer[ii] = (byte) ii; + + VSClient c = new VSClient(host, port); + c.connect(); + + VSChannel s = c.openChannel(); + s.setAutoflush(true); + + boolean measure = true; + SocketTestUtilities.measureLatency(s.getDataOutputStream(), s.getDataInputStream(), buffer, cycles, measure); + } + + @Test + public void TestSocket() throws Throwable { + Test_VSocketLatency.main(new String[0]); + } +} diff --git a/util/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketOutputThroughput.java b/util/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketOutputThroughput.java new file mode 100644 index 00000000..ed77a53a --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketOutputThroughput.java @@ -0,0 +1,66 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.vsocket.util.SocketTestUtilities; +import com.epam.deltix.util.vsocket.util.TestVServerSocketFactory; +import org.junit.Test; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * + */ +public class Test_VSocketOutputThroughput { + public static void main (String [] args) throws Exception { + int port = SocketTestUtilities.parsePort(args); + int packetSize = SocketTestUtilities.parsePacketSize(args); + + VSServer server = TestVServerSocketFactory.createOutputThroughputVServer(port, packetSize); + server.setDaemon(true); + server.start(); + System.out.println("Server started on " + server.getLocalPort()); + + client ("localhost", server.getLocalPort(), packetSize, 15); + server.close(); + } + + public static void client (String host, int port, int packetSize, int cycles) + throws IOException { + byte[] buffer = new byte[packetSize]; + + for (int ii = 0; ii < packetSize; ii++) + buffer[ii] = (byte) ii; + + VSClient c = new VSClient(host, port); + c.connect(); + + VSChannel s = c.openChannel(); + s.setAutoflush(true); + + OutputStream os = s.getOutputStream(); + + SocketTestUtilities.measureOutputThroughput(os, buffer, cycles); + c.close(); + } + + @Test + public void TestOutputThroughput() throws Throwable { + Test_VSocketOutputThroughput.main(new String[0]); + } +} diff --git a/util/src/test/java/com/epam/deltix/util/vsocket/Test_VSocket_Correctness.java b/util/src/test/java/com/epam/deltix/util/vsocket/Test_VSocket_Correctness.java new file mode 100644 index 00000000..0242c898 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/vsocket/Test_VSocket_Correctness.java @@ -0,0 +1,155 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.vsocket.util.SocketTestUtilities; +import com.epam.deltix.util.vsocket.util.TestVServerSocketFactory; +import com.epam.deltix.util.io.BasicIOUtil; +import com.epam.deltix.util.lang.Util; +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Arrays; +import java.util.Random; + +/** + * Sends random data to an echo server and checks that correct data is sent back. + */ +@SuppressWarnings("SameParameterValue") +public class Test_VSocket_Correctness { + + public static final int WRITE_COUNT = 1_000_000; + + @Test + public void testSocket() throws Throwable { + Test_VSocket_Correctness.main(new String[0]); + } + + public static void main(String[] args) throws Throwable { + int port = SocketTestUtilities.parsePort(args); + + VSServer server = TestVServerSocketFactory.createBinaryEchoVServer(port); + server.setDaemon(true); + server.start(); + System.out.println("Server started on " + server.getLocalPort()); + + try { + client("localhost", server.getLocalPort()); + } finally { + server.close(); + } + } + + private static void client(String host, int port) { + try (VSClient client = new VSClient(host, port)) { + client.connect(); + + assertEchoClientCorrectness(client, WRITE_COUNT); + } catch (IOException x) { + throw new UncheckedIOException(x); + } + } + + static void assertEchoClientCorrectness(VSClient client, int writeCount) { + Random rngSrc = new Random(0); + Random rngDst = new Random(0); + + // Fills data with values 1..127 + byte[] data = makeTestData(); + + try (VSChannel channel = client.openChannel()) { + DataOutputStream os = channel.getDataOutputStream(); + + // Data writer thread + new Thread(() -> { + Thread.currentThread().setName("PRODUCER"); + sendData(rngSrc, os, data, writeCount); + }).start(); + + // Reader + readData(channel, data, rngDst, writeCount); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static void sendData(Random rngSrc, DataOutputStream os, byte[] data, int writeCount) { + long totalSent = 0; + try { + for (int i = 0; i < writeCount; i++) { + int size = generateNextSize(rngSrc); + //rngSrc.nextBytes(data); + os.write(data, 0, size); + totalSent += size; + os.write(Byte.MIN_VALUE); + totalSent += 1; + } + os.flush(); + System.out.println("Producer finished to send data. Sent: " + totalSent + " bytes"); + } catch (IOException x) { + throw new UncheckedIOException(x); + } + } + + private static void readData(VSChannel channel, byte[] expected, Random rngDst, int readCount) throws IOException { + long totalRead = 0; + DataInputStream is = channel.getDataInputStream(); + byte[] actual = new byte[8 * 1024]; + for (int i = 0; i < readCount; i++) { + int size = generateNextSize(rngDst); + //rngDst.nextBytes(expected); + BasicIOUtil.readFully(is, actual, 0, size); + + int mismatch = Arrays.mismatch(expected, 0, size, actual, 0, size); + if (mismatch >= 0) { + throw new AssertionError("Data mismatch at position " + (totalRead + mismatch)); + } + totalRead += size; + + int single = is.read(); + if (single < 0) { + throw new AssertionError("Unexpected end of stream"); + } + if (single - 256 != Byte.MIN_VALUE) { + throw new AssertionError("Unexpected value " + single + " at position " + totalRead); + } + totalRead += 1; + } + System.out.println("Consumer finished to read data. Read: " + totalRead + " bytes"); + } + + private static int generateNextSize(Random rngSrc) { + return 10 + rngSrc.nextInt(1000); + } + + private static byte[] makeTestData() { + byte[] data = new byte[8 * 1024]; + byte val = 0; + for (int ii = 0; ii < data.length; ii++) { + val++; + data[ii] = val; + if (val == Byte.MAX_VALUE) { + val = 0; + } + } + return data; + } +} diff --git a/util/src/test/java/com/epam/deltix/util/vsocket/VSDispatcherTest.java b/util/src/test/java/com/epam/deltix/util/vsocket/VSDispatcherTest.java new file mode 100644 index 00000000..51e4be31 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/vsocket/VSDispatcherTest.java @@ -0,0 +1,155 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.concurrent.ContextContainer; +import com.epam.deltix.util.vsocket.ConnectionStateListener; +import com.epam.deltix.util.vsocket.VSChannelImpl; +import com.epam.deltix.util.vsocket.VSTransportChannel; +import com.epam.deltix.util.vsocket.VSocketRecoveryInfo; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author Alexei Osipov + */ +public class VSDispatcherTest { + + /** + * Test for https://gitlab.deltixhub.com/Deltix/QuantServer/QuantServer/issues/43 + */ + @Test (timeout = 10_000) // Note: test timeout must be greater than reconnectInterval + 2000ms + public void testNoHangsOnConcurrentDisconnects() throws IOException, InterruptedException { + int reconnectInterval = 2000; + + VSDispatcher dispatcher = new VSDispatcher("TEST_SERVER", false, new ContextContainer()); + dispatcher.setLingerInterval(reconnectInterval); + dispatcher.setStateListener(new ConnectionStateListener() { + @Override + void onDisconnected() { + } + + @Override + void onConnected() { + } + + @Override + boolean onTransportRecoveryStart(VSocketRecoveryInfo recoveryInfo) { + return false; + } + + @Override + boolean onTransportRecoveryStop(VSocketRecoveryInfo recoveryInfo) { + return true; + } + }); + + int transportCount = 2; + int channelCount = 2; + + for (int i = 0; i < transportCount; i++) { + MemorySocket receiverSocket = new MemorySocket(i + 1); + MemorySocket senderSocket = new MemorySocket(receiverSocket, i + 1); + dispatcher.addTransportChannel(senderSocket); + } + + ArrayList transports = new ArrayList<>(); + + for (int i = 0; i < transportCount; i++) { + transports.add(dispatcher.checkOut()); + } + + for (VSTransportChannel transport : transports) { + dispatcher.checkIn(transport); + } + + CountDownLatch startBarrier = new CountDownLatch(1); + List errorThreads = new ArrayList<>(); + for (int i = 0; i < transportCount; i++) { + VSTransportChannel transport = transports.get(i); + int index = i + 1; + Thread thread = new Thread(() -> { + Thread.currentThread().setName("Error thread " + index); + + try { + startBarrier.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + return; + } + + try { + dispatcher.transportStopped(transport, new Exception("EX" + index)); + } finally { + System.out.println("Error thread finished: " + index); + } + + }); + thread.start(); + errorThreads.add(thread); + } + + // Put some data into VSOutputStream + List channels = new ArrayList<>(); + for (int i = 0; i < channelCount; i++) { + VSChannelImpl channel = dispatcher.newChannel(1024, 1024, true); + channel.onRemoteConnected(i, 64*1024, i); + channels.add(channel); + byte[] buffer = new byte[1024]; + VSOutputStream out = channel.getOutputStream(); + out.write(buffer, 0, buffer.length); + } + + assertTrue(dispatcher.isConnectedOrReconnecting()); + + System.out.println("Emulating broken transports..."); + startBarrier.countDown(); + Thread.sleep(100); // Let threads get into blocked state + + // Now no transports should be available + assertFalse(dispatcher.isConnectedAndNotReconnecting()); + + // Emulate Flusher thread + VSChannelImpl vsChannel = channels.get(0); + boolean gotChanelClosedException = false; + try { + // Note: test may hang here if Dispatcher bug still present + vsChannel.getOutputStream().flushAvailable(true); + } catch (ConnectionAbortedException e) { + gotChanelClosedException = true; + System.out.println("Got ConnectionAbortedException as expected"); + } + Assert.assertTrue(gotChanelClosedException); + + // Waiting for error threads to finish + for (Thread thread : errorThreads) { + thread.join(); + } + + Test_ClientReconnect.waitUntil(5000, "Dispatcher did not reach DISCONNECTED state in time", + () -> dispatcher.getInternalState() == VSDispatcherState.DISCONNECTED); + } +} diff --git a/util/src/test/java/com/epam/deltix/util/vsocket/util/SocketTestUtilities.java b/util/src/test/java/com/epam/deltix/util/vsocket/util/SocketTestUtilities.java new file mode 100644 index 00000000..ed6c20eb --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/vsocket/util/SocketTestUtilities.java @@ -0,0 +1,295 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket.util; + + +import com.epam.deltix.util.memory.DataExchangeUtils; +import com.epam.deltix.util.time.TimeKeeper; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; + + +public class SocketTestUtilities { + public static int DEFAULT_PACKET_SIZE = 4096; + public static int DEFAULT_PORT = 0; + + public static int parsePacketSize(String[] args) { + return parsePacketSize(args, DEFAULT_PACKET_SIZE); + } + + public static int parsePacketSize(String[] args, int defaultPacketSize) { + int packetSize = defaultPacketSize; + if (args.length >= 2) + packetSize = Integer.parseInt(args[1]); + + return packetSize; + } + + public static int parsePort(String[] args) { + int port = DEFAULT_PORT; + if (args.length >= 1) + port = Integer.parseInt(args[0]); + + return port; + } + + public static void measureInputThroughput(InputStream os, byte[] buffer, int cycles) throws IOException { + double avgPackagesPerSec = 0; + double maxPackagesPerSec = 0; + int reportedCounter = 0; + int packages = 0; + int packetSize = buffer.length; + int lastReportedPackages = 0; + + System.out.printf("Measuring throughput by reading packages of size %,d\n", packetSize); + System.out.println("Measurements will be taken approximately every second. They will be printed."); + System.out.println("At the end of test, average throughput will be printed."); + System.out.printf("Test will stop in %,d seconds\n", cycles); + + long lastReportTime = TimeKeeper.currentTime; + long nextReportTime = lastReportTime + 1000; + long stopTime = lastReportTime + cycles * 1000; + + for (;;) { + int offset = 0; + int bytesRead = 0; + int size = packetSize; + while (bytesRead != packetSize) + { + int read = os.read(buffer, offset, size); + bytesRead += read; + offset += read; + size -= read; + } + + packages++; + + long now = TimeKeeper.currentTime; + if (now > nextReportTime) { + long packagesSent = packages - lastReportedPackages; + double secondsExpired = (now - lastReportTime) * 0.001; + double packagesPerSec = packagesSent / secondsExpired; + + System.out.printf("%,d packages/s; %.2f Bytes/s; %.3f MB/s\n", (int) packagesPerSec, packagesPerSec * packetSize, (packagesPerSec * packetSize / (1024*1024))); + avgPackagesPerSec = (packagesPerSec + avgPackagesPerSec * reportedCounter) / (++reportedCounter); + maxPackagesPerSec = Math.max(packagesPerSec, maxPackagesPerSec); + + lastReportedPackages = packages; + lastReportTime = TimeKeeper.currentTime; + nextReportTime = now + 1000; + + if (lastReportTime > stopTime) + { + break; + } + } + } + System.out.println("------------ Average ---------"); + System.out.printf("%,d packages/s; %.2f Bytes/s; %.3f MB/s\n", (int) avgPackagesPerSec, avgPackagesPerSec * packetSize, (avgPackagesPerSec * packetSize / (1024*1024))); + + System.out.println("------------ Max ---------"); + System.out.printf("%,d packages/s; %.2f Bytes/s; %.3f MB/s\n", (int) maxPackagesPerSec, maxPackagesPerSec * packetSize, (maxPackagesPerSec * packetSize / (1024*1024))); + } + + public static void measureOutputThroughput(OutputStream os, byte[] buffer, int cycles) throws IOException { + double avgPackagesPerSec = 0; + double maxPackagesPerSec = 0; + int reportedCounter = 0; + int packages = 0; + int packetSize = buffer.length; + int lastReportedPackages = 0; + + System.out.printf("Measuring throughput by writing packages of size %,d\n", packetSize); + System.out.println("Measurements will be taken approximately every second. They will be printed."); + System.out.println("At the end of test, average throughput will be printed."); + System.out.printf("Test will stop in %,d seconds\n", cycles); + + long lastReportTime = TimeKeeper.currentTime; + long nextReportTime = lastReportTime + 1000; + long stopTime = lastReportTime + cycles * 1000; + + for (;;) { + os.write(buffer); + packages++; + + long now = TimeKeeper.currentTime; + if (now > nextReportTime) { + long packagesSent = packages - lastReportedPackages; + double secondsExpired = (now - lastReportTime) * 0.001; + double packagesPerSec = packagesSent / secondsExpired; + + System.out.printf("%,d packages/s; %.2f Bytes/s; %.3f MB/s\n", (int) packagesPerSec, packagesPerSec * packetSize, (packagesPerSec * packetSize / (1024*1024))); + avgPackagesPerSec = (packagesPerSec + avgPackagesPerSec * reportedCounter) / (++reportedCounter); + maxPackagesPerSec = Math.max(packagesPerSec, maxPackagesPerSec); + + lastReportedPackages = packages; + lastReportTime = TimeKeeper.currentTime; + nextReportTime = now + 1000; + + if (lastReportTime > stopTime) + { + break; + } + } + } + System.out.println("------------ Average ---------"); + System.out.printf("%,d packages/s; %.2f Bytes/s; %.3f MB/s\n", (int) avgPackagesPerSec, avgPackagesPerSec * packetSize, (avgPackagesPerSec * packetSize / (1024*1024))); + + System.out.println("------------ Max ---------"); + System.out.printf("%,d packages/s; %.2f Bytes/s; %.3f MB/s\n", (int) maxPackagesPerSec, maxPackagesPerSec * packetSize, (maxPackagesPerSec * packetSize / (1024*1024))); + } + + public static void measureLatency(OutputStream os, DataInputStream is, byte[] buffer, int cycles, boolean measure) throws IOException, InterruptedException { + int counter = 0; + + double minLatency = Long.MAX_VALUE; + double maxLatency = 0; + double avgLatency = 0; + + System.out.printf("Measuring latency by writing package of size %,d with time and waiting for the responce.", buffer.length); + System.out.println("Measurements will be taken every time, after the responce is received."); + System.out.println("Measurement resolution is defined by TimeKeeper resolution. "); + System.out.println("This test can take some time.\n"); + + for (int cycle = 0; cycle < cycles;) { + double minCycleLatency = Long.MAX_VALUE; + double maxCycleLatency = 0; + double avgCycleLatency = 0; + + while (counter++ < TestServerSocketFactory.LatencyServerSocket.NUM_MESSAGES) { + for (int i = 0; i < TestServerSocketFactory.LatencyServerSocket.NUM_PER_BURST; i++) { + DataExchangeUtils.writeLong(buffer, 0, System.nanoTime()); + + long start = measure ? System.nanoTime() : 0; + + os.write(buffer); + os.flush(); + long count = is.readLong(); + + if (measure) { + long latency = (System.nanoTime() - start) / 1000; + + if (latency > 0) { + minCycleLatency = Math.min(minCycleLatency, latency); + maxCycleLatency = Math.max(maxCycleLatency, latency); + + avgCycleLatency = (avgCycleLatency * (count) + latency) / (++count); + } + } + + counter++; + } + + os.flush(); + + Thread.sleep(10); + } + + counter = 0; + + if (measure) { + System.out.printf("Intermediate latency: Max %,.1f ns; Min %,.1f ns; AVG %,.1f ns\n", maxCycleLatency * 1000, minCycleLatency * 1000, avgCycleLatency * 1000); + } + + minLatency = Math.min(minCycleLatency, minLatency); + maxLatency = Math.max(maxCycleLatency, maxLatency); + + avgLatency = (avgLatency * (cycle) + avgCycleLatency) / (++cycle); + } + + System.out.println("Test final result latency:"); + System.out.printf("Latency: Max %,.1f ns; Min %,.1f ns; AVG %,.1f ns\n", maxLatency * 1000, minLatency * 1000, avgLatency * 1000); + } + + public static void proccessLatencyRequests(DataOutputStream os, DataInputStream is, byte[] buffer, boolean measure) { + int count = 0; + int TOTAL = TestServerSocketFactory.LatencyServerSocket.NUM_MESSAGES * TestServerSocketFactory.LatencyServerSocket.NUM_PER_BURST; + + long[] results = new long[TOTAL]; + + StringBuilder sb = new StringBuilder(); + + try { + System.out.println("Server: connection accepted."); + + for (; ; ) { + + is.readFully(buffer); + os.writeLong(count); + os.flush(); + + long time = DataExchangeUtils.readLong(buffer, 0); + results[count] = (System.nanoTime() - time) / 1000; + + //results[count] = latency = (System.nanoTime() - time) / 1000; + +// if (latency > 0 && count > 0) { +// minLatency = Math.min(minLatency, latency); +// maxLatency = Math.max(maxLatency, latency); +// avgLatency = (avgLatency * (count - 1) + latency) / count; +// } + //result[(int)count] = latency; + + count++; + + if (count % TestServerSocketFactory.LatencyServerSocket.NUM_MESSAGES * TestServerSocketFactory.LatencyServerSocket.NUM_PER_BURST == 0) { + + results[0] = 0; // clear first result + + Arrays.sort(results); + + double avg = 0; + for (int i = 0; i < results.length; i++) + avg = (avg * (count - 1) + results[i]) / count; + + long threshold = results[TOTAL / 100000 * 99999]; + + int index = results.length - 1; + while (index > 0 && results[index] > threshold) + index--; + + if (measure) { + sb.append("Server latency report:-----------------\n"); + sb.append("MIN: ").append(results[0]).append(" mks\n"); + sb.append("MAX: ").append(results[TOTAL - 1]).append(" mks\n"); + sb.append("AVG: ").append((long) avg).append(" mks\n"); + sb.append("> ").append(threshold).append(" mks = ").append(TOTAL - index).append(" results ").append("(99.999%)\n"); + sb.append("--------------------------------------\n"); + + synchronized (System.out) { + for (int i = 0; i < sb.length(); i++) + System.out.write(sb.charAt(i)); + } + } + + Arrays.fill(results, 0); + count = 0; + } + } + } catch (EOFException eof) { + // disconnect + } catch (Throwable x) { + x.printStackTrace(); + } + } +} diff --git a/util/src/test/java/com/epam/deltix/util/vsocket/util/TestServerSocketFactory.java b/util/src/test/java/com/epam/deltix/util/vsocket/util/TestServerSocketFactory.java new file mode 100644 index 00000000..d2ebdd01 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/vsocket/util/TestServerSocketFactory.java @@ -0,0 +1,176 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket.util; + +import com.epam.deltix.util.io.IOUtil; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketException; + +public class TestServerSocketFactory { + public static abstract class ServerThread extends Thread { + protected ServerSocket ss; + + public int getLocalPort(){ + return ss.getLocalPort(); + } + public String getLocalHost(){ + return ss.getInetAddress().getHostAddress(); + } + + protected void setUtSocket(ServerSocket socket) throws SocketException { + socket.setReuseAddress(true); + socket.setSoTimeout (0); + } + } + + public static class ThroughputServerSocket extends ServerThread { + private final byte [] buffer; + + public ThroughputServerSocket (int port, int bufferCapacity) throws IOException { + ss = new ServerSocket (port); + setUtSocket(ss); + + buffer = new byte [bufferCapacity]; + } + + @Override + public void run () { + Socket s = null; + + try { + System.out.println ("Server: listening on port " + ss.getLocalPort ()); + for(;;) { + s = ss.accept (); // Only one accept is handled + + DataInputStream in = new DataInputStream (s.getInputStream ()); + System.out.println ("Server: connection accepted."); + + while(s.isConnected()) + { + in.read(buffer); + //in.readFully (buffer); + } + } + } catch (Throwable x) { + x.printStackTrace (); + } finally { + IOUtil.close (s); + IOUtil.close (ss); + } + } + } + + public static class LatencyServerSocket extends ServerThread { + public static final int NUM_MESSAGES = 10000; + public static final int NUM_PER_BURST = 100; + + private final byte[] buffer; + + public LatencyServerSocket(int port, int packetSize) throws IOException { + ss = new ServerSocket(port); + setUtSocket(ss); + + buffer = new byte[packetSize]; + } + + @Override + public void run() { + Socket s = null; + System.out.println("Server listening on port " + ss.getLocalPort() + "; packet size: " + buffer.length); + + try { + s = ss.accept(); // Only one accept is handled + System.out.println("Server: connection accepted."); + + DataInputStream in = new DataInputStream(s.getInputStream()); + DataOutputStream out = new DataOutputStream(s.getOutputStream()); + + SocketTestUtilities.proccessLatencyRequests(out, in, buffer, false); + } catch (Throwable x) { + x.printStackTrace(); + } finally { + IOUtil.close(s); + } + + + IOUtil.close(ss); + } + } + + public static class EchoServerSocket extends ServerThread { + + public EchoServerSocket (int port) throws IOException { + ss = new ServerSocket (port); + setUtSocket(ss); + } + + @Override + public void run () { + Socket s = null; + + try { + System.out.println ("Server: listening on port " + ss.getLocalPort ()); + + for(;;) + { + s = ss.accept (); // Only one accept is handled + + DataInputStream in = new DataInputStream (s.getInputStream ()); + DataOutputStream out = new DataOutputStream (s.getOutputStream ()); + + System.out.println ("Server: connection accepted."); + + + String utfString = in.readUTF (); + out.writeUTF (utfString); + out.flush (); + } + } catch (Throwable x) { + x.printStackTrace (); + } finally { + IOUtil.close (s); + IOUtil.close (ss); + } + } + } + + public static ServerThread createThroughputServerSocket(int port, int packetSize) throws IOException { + ThroughputServerSocket serverSocket = new ThroughputServerSocket(port, packetSize); + serverSocket.setDaemon(true); + serverSocket.setPriority(Thread.MAX_PRIORITY); + return serverSocket; + } + + public static ServerThread createLatencyServerSocket(int port, int packetSize) throws IOException { + LatencyServerSocket serverSocket = new LatencyServerSocket(port, packetSize); + serverSocket.setDaemon(true); + serverSocket.setPriority(Thread.MAX_PRIORITY); + return serverSocket; + } + + public static ServerThread createEchoServerSocket(int port) throws IOException { + EchoServerSocket serverSocket = new EchoServerSocket(port); + serverSocket.setDaemon(true); + serverSocket.setPriority(Thread.MAX_PRIORITY); + return serverSocket; + } +} diff --git a/util/src/test/java/com/epam/deltix/util/vsocket/util/TestVServerSocketFactory.java b/util/src/test/java/com/epam/deltix/util/vsocket/util/TestVServerSocketFactory.java new file mode 100644 index 00000000..1c3d4ecd --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/vsocket/util/TestVServerSocketFactory.java @@ -0,0 +1,279 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket.util; + +import com.epam.deltix.util.concurrent.QuickExecutor; +import com.epam.deltix.util.io.EOQException; +import com.epam.deltix.util.memory.DataExchangeUtils; +import com.epam.deltix.util.vsocket.ChannelClosedException; +import com.epam.deltix.util.vsocket.VSChannel; +import com.epam.deltix.util.vsocket.VSConnectionListener; +import com.epam.deltix.util.vsocket.VSServer; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; + + +public class TestVServerSocketFactory { + public static VSServer createInputThroughputVServer(int port, int packetSize) throws IOException { + return createVServerSocket(port, ((executor, serverChannel) -> new InputThroughputServer(executor, serverChannel, packetSize).submit())); + } + + public static VSServer createOutputThroughputVServer(int port, int packetSize) throws IOException { + return createVServerSocket(port, ((executor, serverChannel) -> new OutputThroughputServer(executor, serverChannel, packetSize).submit())); + } + + public static VSServer createLatencyVServer(int port, int packetSize) throws IOException { + return createVServerSocket(port, ((executor, serverChannel) -> new LatencyServer(executor, serverChannel, packetSize).submit())); + } + + public static VSServer createEchoVServer(int port) throws IOException { + return createVServerSocket(port, ((executor, serverChannel) -> new EchoServer(executor, serverChannel).submit())); + } + + public static VSServer createBinaryEchoVServer(int port) throws IOException { + return createVServerSocket(port, ((executor, serverChannel) -> { + BinaryEchoServer binaryEchoServer = new BinaryEchoServer(executor, serverChannel); + serverChannel.setAvailabilityListener(binaryEchoServer::submit); + binaryEchoServer.submit(); + })); + } + + public static VSServer createReadingVServer(int port, int packetSize) throws IOException { + return createVServerSocket(port, ((executor, serverChannel) -> new ReadingServer(executor, serverChannel, packetSize).submit())); + } + + public static VSServer createEmptyVServer(int port) throws IOException { + return createVServerSocket(port, ((executor, serverChannel) -> new EmptyServer(executor, serverChannel).submit())); + } + + private static VSServer createVServerSocket(int port, VSConnectionListener listener) throws IOException { + VSServer server = new VSServer(port); + server.setConnectionListener(listener); + return server; + } + + static class ReadingServer extends QuickExecutor.QuickTask { + private VSChannel channel; + private final byte [] buffer; + + public ReadingServer (QuickExecutor executor, VSChannel channel, int bufferCapacity) throws IOException { + super (executor); + this.channel = channel; + this.buffer = new byte [bufferCapacity]; + } + + @Override + public void run () { + try { + DataInputStream in = new DataInputStream (channel.getInputStream ()); + + long index = 0; + for (;;) { + + in.readFully (buffer); + + long first = DataExchangeUtils.readLong(buffer, 0); + long second = DataExchangeUtils.readLong(buffer, 8); + if (second - first != 1) + throw new IllegalStateException("mismatch: first = " + first + "; second = " + second); + + if (first != index) + throw new IllegalStateException("mismatch: first(" + first + ") != index (" + index + ")"); + + index += 2; + + } + } catch (Throwable x) { + x.printStackTrace (); + } finally { + channel.close (); + } + } + } + + static class EchoServer extends QuickExecutor.QuickTask { + private VSChannel channel; + + public EchoServer(QuickExecutor executor, VSChannel channel) throws IOException { + super(executor); + this.channel = channel; + } + + @Override + public void run() throws InterruptedException { + try { + String s = channel.getDataInputStream().readUTF(); + + DataOutputStream out = channel.getDataOutputStream(); + out.writeUTF(s); + out.flush(); + } catch (Throwable x) { + x.printStackTrace(); + } finally { + channel.close(); + } + } + } + + /** + * Unlike EchoServer, this server reads and writes any binary data, not just text strings. + */ + static class BinaryEchoServer extends QuickExecutor.QuickTask { + private final VSChannel channel; + private final byte[] buffer = new byte[8 * 1024]; + private long total = 0; + volatile boolean closed = false; // Protects from extra-execution immediately after the channel closure + + public BinaryEchoServer(QuickExecutor executor, VSChannel channel) { + super(executor); + this.channel = channel; + } + + @Override + public void run() throws InterruptedException { + // This task will be re-armed when more data will be available + if (closed) { + return; + } + String oldName = Thread.currentThread().getName(); + Thread.currentThread().setName("BinaryEchoServer"); + + DataInputStream is = channel.getDataInputStream(); + DataOutputStream out = channel.getDataOutputStream(); + + try { + int available; + while ((available = is.available()) > 0) { + // This read should not block because we read only up to "available" bytes + int read = is.read(buffer, 0, Math.min(available, buffer.length)); + out.write(buffer, 0, read); + total += read; + } + out.flush(); + } catch (EOQException e) { + finish(); + } catch (IOException e) { + throw new RuntimeException(e); + } finally { + Thread.currentThread().setName(oldName); + } + } + + private void finish() { + closed = true; + channel.setAvailabilityListener(null); + channel.close(); + System.out.println("BinaryEchoServer: total bytes echoed: " + total); + } + } + + static class InputThroughputServer extends QuickExecutor.QuickTask { + private VSChannel channel; + private byte[] buffer; + + public InputThroughputServer(QuickExecutor executor, VSChannel channel, int packetSize) throws IOException { + super(executor); + this.channel = channel; + buffer = new byte[packetSize]; + for (int index = 0; index < packetSize; index++) + { + buffer[index] = (byte) index; + } + } + + @Override + public void run() throws InterruptedException { + try { + DataOutputStream out = channel.getDataOutputStream(); + + for (;;) { + out.write(buffer); + } + } catch(ChannelClosedException x) { + // do nothing + } catch (Throwable x) { + x.printStackTrace(); + } finally { + channel.close(); + } + } + } + + static class OutputThroughputServer extends QuickExecutor.QuickTask { + private VSChannel channel; + private byte[] buffer; + + public OutputThroughputServer(QuickExecutor executor, VSChannel channel, int packetSize) throws IOException { + super(executor); + this.channel = channel; + buffer = new byte[packetSize]; + } + + @Override + public void run() throws InterruptedException { + try { + DataInputStream in = new DataInputStream(channel.getInputStream()); + + for (;;) { + in.readFully(buffer); + } + } catch (Throwable x) { + x.printStackTrace(); + } finally { + channel.close(); + } + } + } + + static class LatencyServer extends QuickExecutor.QuickTask { + private VSChannel channel; + private byte[] buffer; + + public LatencyServer(QuickExecutor executor, VSChannel channel, int packetSize) throws IOException { + super(executor); + this.channel = channel; + buffer = new byte[packetSize]; + } + + @Override + public void run() throws InterruptedException { + try { + DataInputStream in = channel.getDataInputStream(); + DataOutputStream out = channel.getDataOutputStream(); + SocketTestUtilities.proccessLatencyRequests(out, in, buffer, false); + } + catch (Throwable x) { + x.printStackTrace(); + } + } + } + + static class EmptyServer extends QuickExecutor.QuickTask { + private VSChannel channel; + + public EmptyServer(QuickExecutor executor, VSChannel channel) throws IOException { + super(executor); + this.channel = channel; + } + + @Override + public void run() throws InterruptedException { + } + } +} diff --git a/util/src/test/java/com/epam/deltix/util/vsocket/util/VServerTestLauncher.java b/util/src/test/java/com/epam/deltix/util/vsocket/util/VServerTestLauncher.java new file mode 100644 index 00000000..fe248ad2 --- /dev/null +++ b/util/src/test/java/com/epam/deltix/util/vsocket/util/VServerTestLauncher.java @@ -0,0 +1,181 @@ +/* + * Copyright 2021 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.epam.deltix.util.vsocket.util; + +import com.epam.deltix.util.vsocket.VSServer; + +import java.io.IOException; + + +public class VServerTestLauncher { + private VSServer startedServer; + private int port = 8025; + private int packetSize = 4096; + + public static void main(String[] args) { + if(args.length == 0 || (args.length == 1 && args[0] == "h")) + { + printHelp(); + } + + StartServer(args); + } + + private static void StartServer(String[] args) + { + VServerTestLauncher vserverInstance = new VServerTestLauncher(); + if (args.length > 1) + vserverInstance.packetSize = Integer.parseInt(args[1]); + + if (args.length > 2) + vserverInstance.port = Integer.parseInt(args[2]); + + System.out.println("Going to start server at port " + vserverInstance.port); + try{ + switch (args[0]) + { + case "lv": + vserverInstance.startLatencyServer(); + break; + + case "ls": + vserverInstance.startLatencySocket(); + break; + + case "ev": + vserverInstance.startEchoServer(); + break; + + case "es": + vserverInstance.startEchoSocket(); + break; + + case "iv": + vserverInstance.startInputThroughputServer(); + break; + + case "is": + vserverInstance.startInputThroughputSocket(); + break; + + case "ov": + vserverInstance.startOutputThroughputServer(); + break; + + case "os": + vserverInstance.startOutputThroughputSocket(); + break; + + case "nv" : + vserverInstance.startEmptyServer(); + break; + + case "rv" : + vserverInstance.startReadingServer(); + break; + + default: + System.out.println("Unknown argument: " + args[0]); + } + } + catch (IOException e) { + System.out.println("Cannot start VServer."); + System.out.print(e.toString()); + } + } + + private static void printHelp() + { + System.out.println("Help guide:"); + System.out.println("Arguments: [e]"); + System.out.println("h - Help."); + System.out.println("nv - Runs VSocket server, that only listens to clients."); + System.out.println("lv - Runs VSocket server, configured for latency test."); + System.out.println("ls - Runs Socket server, configured for latency test."); + System.out.println("ev - Runs VSocket server, configured for echo test."); + System.out.println("es - Runs Socket server, configured for echo test."); + System.out.println("iv - Runs VSocket server, configured for input throughput test."); + System.out.println("is - Runs Socket server, configured for input throughput test."); + System.out.println("ov - Runs VSocket server, configured for output throughput test."); + System.out.println("os - Runs Socket server, configured for output throughput test."); + System.out.println("rv - Runs VSocket Reading server, configured for reconnecting tests."); + } + + + private void startLatencyServer() throws IOException { + Thread server = TestVServerSocketFactory.createLatencyVServer(port, packetSize); + server.setDaemon(false); + server.start(); + } + + private void startLatencySocket() throws IOException { + Thread server = TestServerSocketFactory.createLatencyServerSocket(port, packetSize); + server.setDaemon(false); + server.start(); + } + + private void startEchoServer() throws IOException { + Thread server = TestVServerSocketFactory.createEchoVServer(port); + server.setDaemon(false); + server.start(); + } + + private void startEchoSocket() throws IOException { + Thread server = TestServerSocketFactory.createEchoServerSocket(port); + server.setDaemon(false); + server.start(); + } + + private void startInputThroughputServer() throws IOException { + Thread server = TestVServerSocketFactory.createInputThroughputVServer(port, packetSize); + server.setDaemon(false); + server.start(); + } + + private void startInputThroughputSocket() throws IOException { +// Thread server = TestServerSocketFactory.createThroughputServerSocket(port, packetSize); +// server.setDaemon(false); +// server.start(); + } + + private void startOutputThroughputServer() throws IOException { + Thread server = TestVServerSocketFactory.createOutputThroughputVServer(port, packetSize); + server.setDaemon(false); + server.start(); + } + + private void startOutputThroughputSocket() throws IOException { + Thread server = TestServerSocketFactory.createThroughputServerSocket(port, packetSize); + server.setDaemon(false); + server.start(); + } + + private void startReadingServer() throws IOException { + Thread server = TestVServerSocketFactory.createReadingVServer(port, packetSize); + server.setDaemon(false); + server.start(); + } + + private void startEmptyServer() throws IOException { + Thread server = TestVServerSocketFactory.createEmptyVServer(port); + server.setDaemon(false); + server.start(); + } + + + +} diff --git a/util/src/test/resources/com/epam/deltix/util/text/dateTimeFormat.txt b/util/src/test/resources/com/epam/deltix/util/text/dateTimeFormat.txt new file mode 100644 index 00000000..793654cf --- /dev/null +++ b/util/src/test/resources/com/epam/deltix/util/text/dateTimeFormat.txt @@ -0,0 +1,68 @@ +2023-11-14T,22:13:20Z,yyyy-MM-dd'T',HH:mm:ss'Z' +11/14/2023T,22:13:20Z,MM/dd/yyyy'T',HH:mm:ss'Z' +2023/11/14T,22:13:20Z,yyyy/MM/dd'T',HH:mm:ss'Z' +2023-11-14 ,22:13:20,yyyy-MM-dd ,HH:mm:ss +11/14/2023 ,22:13:20,MM/dd/yyyy ,HH:mm:ss +2023/11/14 ,22:13:20,yyyy/MM/dd ,HH:mm:ss +2023-11-14T,22:13:20.000Z,yyyy-MM-dd'T',HH:mm:ss.SSS'Z' +11/14/2023T,22:13:20.000Z,MM/dd/yyyy'T',HH:mm:ss.SSS'Z' +2023/11/14T,22:13:20.000Z,yyyy/MM/dd'T',HH:mm:ss.SSS'Z' +2023-11-14T,22:13:20,yyyy-MM-dd'T',HH:mm:ss +11/14/2023T,22:13:20,MM/dd/yyyy'T',HH:mm:ss +2023/11/14T,22:13:20,yyyy/MM/dd'T',HH:mm:ss +2023-11-14T,22:13:20.000,yyyy-MM-dd'T',HH:mm:ss.SSS +11/14/2023T,22:13:20.000,MM/dd/yyyy'T',HH:mm:ss.SSS +2023/11/14T,22:13:20.000,yyyy/MM/dd'T',HH:mm:ss.SSS +2023-11-14 ,22:13,yyyy-MM-dd ,HH:mm +11/14/2023 ,22:13,MM/dd/yyyy ,HH:mm +2023/11/14 ,22:13,yyyy/MM/dd ,HH:mm +20231114,221320,yyyyMMdd,HHmmss +20231114 ,221320,yyyyMMdd ,HHmmss +2023-11-14 ,22:13:20.000,yyyy-MM-dd ,HH:mm:ss.SSS +11/14/2023 ,22:13:20.000,MM/dd/yyyy ,HH:mm:ss.SSS +2023/11/14 ,22:13:20.000,yyyy/MM/dd ,HH:mm:ss.SSS +2023-11-14T,22:13:20.123456789Z,yyyy-MM-dd'T',HH:mm:ss.SSSSSSSSS'Z' +11/14/2023T,22:13:20.123456789Z,MM/dd/yyyy'T',HH:mm:ss.SSSSSSSSS'Z' +2023/11/14T,22:13:20.123456789Z,yyyy/MM/dd'T',HH:mm:ss.SSSSSSSSS'Z' +2023-11-14T,22:13:20.123456789,yyyy-MM-dd'T',HH:mm:ss.SSSSSSSSS +11/14/2023T,22:13:20.123456789,MM/dd/yyyy'T',HH:mm:ss.SSSSSSSSS +2023/11/14T,22:13:20.123456789,yyyy/MM/dd'T',HH:mm:ss.SSSSSSSSS +2023-11-14 ,22:13:20.123456789,yyyy-MM-dd ,HH:mm:ss.SSSSSSSSS +11/14/2023 ,22:13:20.123456789,MM/dd/yyyy ,HH:mm:ss.SSSSSSSSS +2023/11/14 ,22:13:20.123456789,yyyy/MM/dd ,HH:mm:ss.SSSSSSSSS +2023-11-14T,10:13:20AM,yyyy-MM-dd'T',HH:mm:ssa +2023-11-14T,10:13:20PM,yyyy-MM-dd'T',HH:mm:ssa +11/14/2023T,10:13:20AM,MM/dd/yyyy'T',HH:mm:ssa +11/14/2023T,10:13:20PM,MM/dd/yyyy'T',HH:mm:ssa +2023/11/14T,10:13:20AM,yyyy/MM/dd'T',HH:mm:ssa +2023/11/14T,10:13:20PM,yyyy/MM/dd'T',HH:mm:ssa +2023-11-14 ,10:13:20AM,yyyy-MM-dd ,HH:mm:ssa +2023-11-14 ,10:13:20PM,yyyy-MM-dd ,HH:mm:ssa +11/14/2023 ,10:13:20AM,MM/dd/yyyy ,HH:mm:ssa +11/14/2023 ,10:13:20PM,MM/dd/yyyy ,HH:mm:ssa +2023/11/14 ,10:13:20AM,yyyy/MM/dd ,HH:mm:ssa +2023/11/14 ,10:13:20PM,yyyy/MM/dd ,HH:mm:ssa +2023-11-14T,10:13:20.000AM,yyyy-MM-dd'T',HH:mm:ss.SSSa +2023-11-14T,10:13:20.000PM,yyyy-MM-dd'T',HH:mm:ss.SSSa +11/14/2023T,10:13:20.000AM,MM/dd/yyyy'T',HH:mm:ss.SSSa +11/14/2023T,10:13:20.000PM,MM/dd/yyyy'T',HH:mm:ss.SSSa +2023/11/14T,10:13:20.000AM,yyyy/MM/dd'T',HH:mm:ss.SSSa +2023/11/14T,10:13:20.000PM,yyyy/MM/dd'T',HH:mm:ss.SSSa +2023-11-14 ,10:13:20.000AM,yyyy-MM-dd ,HH:mm:ss.SSSa +2023-11-14 ,10:13:20.000PM,yyyy-MM-dd ,HH:mm:ss.SSSa +11/14/2023 ,10:13:20.000AM,MM/dd/yyyy ,HH:mm:ss.SSSa +11/14/2023 ,10:13:20.000PM,MM/dd/yyyy ,HH:mm:ss.SSSa +2023/11/14 ,10:13:20.000AM,yyyy/MM/dd ,HH:mm:ss.SSSa +2023/11/14 ,10:13:20.000PM,yyyy/MM/dd ,HH:mm:ss.SSSa +2023-11-14T,10:13:20.123456789AM,yyyy-MM-dd'T',HH:mm:ss.SSSSSSSSSa +2023-11-14T,10:13:20.123456789PM,yyyy-MM-dd'T',HH:mm:ss.SSSSSSSSSa +11/14/2023T,10:13:20.123456789AM,MM/dd/yyyy'T',HH:mm:ss.SSSSSSSSSa +11/14/2023T,10:13:20.123456789PM,MM/dd/yyyy'T',HH:mm:ss.SSSSSSSSSa +2023/11/14T,10:13:20.123456789AM,yyyy/MM/dd'T',HH:mm:ss.SSSSSSSSSa +2023/11/14T,10:13:20.123456789PM,yyyy/MM/dd'T',HH:mm:ss.SSSSSSSSSa +2023-11-14 ,10:13:20.123456789AM,yyyy-MM-dd ,HH:mm:ss.SSSSSSSSSa +2023-11-14 ,10:13:20.123456789PM,yyyy-MM-dd ,HH:mm:ss.SSSSSSSSSa +11/14/2023 ,10:13:20.123456789AM,MM/dd/yyyy ,HH:mm:ss.SSSSSSSSSa +11/14/2023 ,10:13:20.123456789PM,MM/dd/yyyy ,HH:mm:ss.SSSSSSSSSa +2023/11/14 ,10:13:20.123456789AM,yyyy/MM/dd ,HH:mm:ss.SSSSSSSSSa +2023/11/14 ,10:13:20.123456789PM,yyyy/MM/dd ,HH:mm:ss.SSSSSSSSSa \ No newline at end of file