Skip to content

Commit

Permalink
Merge pull request #651 from the-thing/650/truncated_filestore_messages
Browse files Browse the repository at this point in the history 
`FileStore` message length fix
  • Loading branch information
@chrjohn
chrjohn authored Dec 7, 2023
2 parents 8cf71f9 + ddeb9bb commit e050d3d
Show file tree
Hide file tree
Showing 10 changed files with 289 additions and 48 deletions.
4 changes: 4 additions & 0 deletions quickfixj-base/src/main/java/org/quickfixj/CharsetSupport.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,10 @@ public static void setCharset(String charset) throws UnsupportedEncodingExceptio
CharsetSupport.isStringEquivalent = isStringEquivalent(charsetInstance);
}

public static void setDefaultCharset() throws UnsupportedEncodingException {
setCharset(getDefaultCharset());
}

public static String getCharset() {
return charset;
}
Expand Down
27 changes: 13 additions & 14 deletions quickfixj-core/src/main/java/quickfix/CachedFileStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.io.BufferedOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
Expand Down Expand Up @@ -86,8 +87,6 @@ public class CachedFileStore implements MessageStore {

private FileOutputStream headerFileOutputStream;

private final String charsetEncoding = CharsetSupport.getCharset();

CachedFileStore(String path, SessionID sessionID, boolean syncWrites) throws IOException {
this.syncWrites = syncWrites;

Expand Down Expand Up @@ -308,16 +307,15 @@ public boolean get(int sequence, String message) {
throw new UnsupportedOperationException("not supported");
}

private String read(long offset, long size) throws IOException {
final byte[] data = new byte[(int) size];

messageFileReader.seek(offset);
if (messageFileReader.read(data) != size) {
throw new IOException("Truncated input while reading message: "
+ new String(data, charsetEncoding));
private String read(long offset, int size) throws IOException {
try {
final byte[] data = new byte[size];
messageFileReader.seek(offset);
messageFileReader.readFully(data);
return new String(data, CharsetSupport.getCharset());
} catch (EOFException eofe) { // can't read fully
throw new IOException("Truncated input while reading message: offset=" + offset + ", expected size=" + size, eofe);
}

return new String(data, charsetEncoding);
}

private Collection<String> getMessage(long startSequence, long endSequence) throws IOException {
Expand All @@ -326,7 +324,7 @@ private Collection<String> getMessage(long startSequence, long endSequence) thro
final List<long[]> offsetAndSizes = messageIndex.get(startSequence, endSequence);
for (final long[] offsetAndSize : offsetAndSizes) {
if (offsetAndSize != null) {
final String message = read(offsetAndSize[0], offsetAndSize[1]);
final String message = read(offsetAndSize[0], (int) offsetAndSize[1]);
messages.add(message);
}
}
Expand All @@ -341,7 +339,8 @@ private Collection<String> getMessage(long startSequence, long endSequence) thro
*/
public boolean set(int sequence, String message) throws IOException {
final long offset = messageFileWriter.getFilePointer();
final int size = message.length();
final byte[] messageBytes = message.getBytes(CharsetSupport.getCharset());
final int size = messageBytes.length;
messageIndex.put((long) sequence, new long[] { offset, size });
headerDataOutputStream.writeInt(sequence);
headerDataOutputStream.writeLong(offset);
Expand All @@ -350,7 +349,7 @@ public boolean set(int sequence, String message) throws IOException {
if (syncWrites) {
headerFileOutputStream.getFD().sync();
}
messageFileWriter.write(message.getBytes(CharsetSupport.getCharset()));
messageFileWriter.write(messageBytes);
return true;
}

Expand Down
8 changes: 4 additions & 4 deletions quickfixj-core/src/main/java/quickfix/FileStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@ public class FileStore implements MessageStore, Closeable {
private final String sessionFileName;
private final boolean syncWrites;
private final int maxCachedMsgs;
private final String charsetEncoding = CharsetSupport.getCharset();
private RandomAccessFile messageFileReader;
private RandomAccessFile messageFileWriter;
private DataOutputStream headerDataOutputStream;
Expand Down Expand Up @@ -350,7 +349,7 @@ private String getMessage(long offset, int size, int i) throws IOException {
final byte[] data = new byte[size];
messageFileReader.seek(offset);
messageFileReader.readFully(data);
return new String(data, charsetEncoding);
return new String(data, CharsetSupport.getCharset());
} catch (EOFException eofe) { // can't read fully
throw new IOException("Truncated input while reading message: messageIndex=" + i
+ ", offset=" + offset + ", expected size=" + size, eofe);
Expand All @@ -363,7 +362,8 @@ private String getMessage(long offset, int size, int i) throws IOException {
@Override
public boolean set(int sequence, String message) throws IOException {
final long offset = messageFileWriter.getFilePointer();
final int size = message.length();
final byte[] messageBytes = message.getBytes(CharsetSupport.getCharset());
final int size = messageBytes.length;
if (messageIndex != null) {
updateMessageIndex(sequence, offset, size);
}
Expand All @@ -374,7 +374,7 @@ public boolean set(int sequence, String message) throws IOException {
if (syncWrites) {
headerFileOutputStream.getFD().sync();
}
messageFileWriter.write(message.getBytes(CharsetSupport.getCharset()));
messageFileWriter.write(messageBytes);
return true;
}

Expand Down
41 changes: 41 additions & 0 deletions quickfixj-core/src/test/java/quickfix/AbstractMessageStoreTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,14 @@
package quickfix;

import junit.framework.TestCase;
import org.quickfixj.CharsetSupport;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public abstract class AbstractMessageStoreTest extends TestCase {

private SessionID sessionID;
private MessageStore store;

Expand Down Expand Up @@ -167,6 +170,44 @@ public void testRefreshMessageStore() throws Exception {
}
}

public void testSetAndGetMessageWithAsciiCharacters() throws IOException {
MessageStore underTest = getStore();

if (underTest instanceof SleepycatStore) {
return;
}

underTest.set(1, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVXYZ0123456789");

List<String> messages = new ArrayList<>();
underTest.get(1, 1, messages);

assertEquals(1, messages.size());
assertEquals("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVXYZ0123456789", messages.get(0));
}

public void testSetAndGetMessageWithUnicodeCharacters() throws IOException {
MessageStore underTest = getStore();

if (underTest instanceof SleepycatStore) {
return;
}

CharsetSupport.setCharset("UTF-8");

try {
underTest.set(1, "a \u00A9 \u2603 \uD834\uDF06");

List<String> messages = new ArrayList<>();
underTest.get(1, 1, messages);

assertEquals(1, messages.size());
assertEquals("a \u00A9 \u2603 \uD834\uDF06", messages.get(0));
} finally {
CharsetSupport.setDefaultCharset();
}
}

protected void closeMessageStore(MessageStore store) throws IOException {
// does nothing, by default
}
Expand Down
191 changes: 188 additions & 3 deletions quickfixj-core/src/test/java/quickfix/FieldTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,13 @@

package quickfix;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import java.io.UnsupportedEncodingException;
import java.math.BigDecimal;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.util.Arrays;
import java.util.Date;

import org.junit.Test;
import org.quickfixj.CharsetSupport;
Expand All @@ -39,6 +41,10 @@
import quickfix.fix50.MarketDataIncrementalRefresh;
import quickfix.fix50.NewOrderSingle;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

public class FieldTest {

@Test
Expand All @@ -52,6 +58,185 @@ public void testMessageSetGetString() {
assertEquals(side1, side2);
}

private void testFieldCalcuations(String value, int checksum, int length) {
Field<String> field = new Field<>(12, value);
field.setObject(value);
assertEquals("12=" + value, field.toString());
assertEquals(checksum, field.getChecksum());
assertEquals(length, field.getLength());

value = value.substring(0, value.length() - 1) + (char)(value.charAt(value.length() - 1) + 1);
checksum = (checksum + 1) & 0xFF;
field.setObject(value);
assertEquals("12=" + value, field.toString());
assertEquals(checksum, field.getChecksum());
assertEquals(length, field.getLength());

field.setTag(13);
checksum = (checksum + 1) & 0xFF;
assertEquals("13=" + value, field.toString());
assertEquals(checksum, field.getChecksum());
assertEquals(length, field.getLength());
}

@Test
public void testFieldCalculationsWithDefaultCharset() {
testFieldCalcuations("VALUE", 30, 9);
}

@Test
public void testFieldCalculationsWithUTF8Charset() throws UnsupportedEncodingException {
CharsetSupport.setCharset("UTF-8");
try {
testFieldCalcuations("\u6D4B\u9A8C\u6570\u636E", 50, 16);
} finally {
CharsetSupport.setDefaultCharset();
}
}

@Test
public void testDateField() {
DateField field = new DateField(11);
Date date = new Date();
field.setValue(date);
assertEquals(11, field.getTag());
assertEquals(date, field.getValue());
field = new DateField(11, date);
assertEquals(11, field.getTag());
assertEquals(date, field.getValue());
}

@Test
public void testUtcDateOnlyField() {
UtcDateOnlyField field = new UtcDateOnlyField(11);
LocalDate date = LocalDate.now();
field.setValue(date);
assertEquals(11, field.getTag());
assertEquals(date, field.getValue());
field = new UtcDateOnlyField(11, date);
assertEquals(11, field.getTag());
assertEquals(date, field.getValue());
}

@Test
public void testUtcTimeOnlyField() {
UtcTimeOnlyField field = new UtcTimeOnlyField(11);
LocalTime date = LocalTime.now();
field.setValue(date);
assertEquals(11, field.getTag());
assertEquals(date, field.getValue());
field = new UtcTimeOnlyField(11, date);
assertEquals(11, field.getTag());
assertEquals(date, field.getValue());
}

@Test
public void testUtcTimeStampField() {
UtcTimeStampField field = new UtcTimeStampField(11);
LocalDateTime date = LocalDateTime.now();
field.setValue(date);
assertEquals(11, field.getTag());
assertEquals(date, field.getValue());
field = new UtcTimeStampField(11, date);
assertEquals(11, field.getTag());
assertEquals(date, field.getValue());
}

@Test
public void testBooleanField() {
BooleanField field = new BooleanField(11);
field.setValue(true);
assertEquals(11, field.getTag());
assertTrue(field.getValue());
field.setValue(Boolean.FALSE);
assertEquals(11, field.getTag());
assertFalse(field.getValue());
field = new BooleanField(22, true);
assertEquals(22, field.getTag());
assertTrue(field.getValue());
field = new BooleanField(33, Boolean.TRUE);
assertEquals(33, field.getTag());
assertTrue(field.getValue());
}

@Test
public void testDoubleField() {
DoubleField field = new DoubleField(11);
field.setValue(12.3);
assertEquals(11, field.getTag());
assertEquals(12.3, field.getValue(), 0);
field.setValue(new Double(23.4));
assertEquals(11, field.getTag());
assertEquals(23.4, field.getValue(), 0);
field = new DoubleField(22, 34.5);
assertEquals(22, field.getTag());
assertEquals(34.5, field.getValue(), 0);
field = new DoubleField(33, new Double(45.6));
assertEquals(33, field.getTag());
assertEquals(45.6, field.getValue(), 0);
}

@Test(expected = NumberFormatException.class)
public void testDoubleFieldException() {
DoubleField field = new DoubleField(11, Double.NaN);
}

@Test
public void testDecimalField() {
DecimalField field = new DecimalField(11);
field.setValue(12.3);
assertEquals(11, field.getTag());
assertEquals(BigDecimal.valueOf(12.3), field.getValue());
field.setValue(23.4);
assertEquals(11, field.getTag());
assertEquals(BigDecimal.valueOf(23.4), field.getValue());
field = new DecimalField(22, 34.5);
assertEquals(22, field.getTag());
assertEquals(BigDecimal.valueOf(34.5), field.getValue());
field = new DecimalField(33, 45.6);
assertEquals(33, field.getTag());
assertEquals(BigDecimal.valueOf(45.6), field.getValue());
}

@Test(expected = NumberFormatException.class)
public void testDecimalFieldException() {
DecimalField field = new DecimalField(11, Double.POSITIVE_INFINITY);
}

@Test
public void testCharField() {
CharField field = new CharField(11);
field.setValue('x');
assertEquals(11, field.getTag());
assertEquals('x', field.getValue());
field.setValue(Character.valueOf('X'));
assertEquals(11, field.getTag());
assertEquals('X', field.getValue());
field = new CharField(22, 'a');
assertEquals(22, field.getTag());
assertEquals('a', field.getValue());
field = new CharField(33, Character.valueOf('A'));
assertEquals(33, field.getTag());
assertEquals('A', field.getValue());
}

@Test
public void testIntField() {
IntField field = new IntField(11);
field.setValue(12);
assertEquals(11, field.getTag());
assertEquals(12, field.getValue());
field.setValue(Integer.valueOf(23));
assertEquals(11, field.getTag());
assertEquals(23, field.getValue());
field = new IntField(22, 23);
assertEquals(22, field.getTag());
assertEquals(23, field.getValue());
field = new IntField(33, Integer.valueOf(44));
assertEquals(33, field.getTag());
assertEquals(44, field.getValue());
}

@Test
public void testBytesField() {
byte[] data = "rawdata".getBytes();
Expand Down
Loading

0 comments on commit e050d3d

Please sign in to comment.