Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 55 additions & 4 deletions store/mongo/mongo_store.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package mongo
import (
"context"
"fmt"
"sync"
"time"

"github.com/pkg/errors"
Expand All @@ -36,6 +37,8 @@ type mongoStoreFactory struct {
}

type mongoStore struct {
// Serialize database writes and the corresponding cache access.
mu sync.Mutex
sessionID quickfix.SessionID
cache quickfix.MessageStore
mongoURL string
Expand Down Expand Up @@ -158,6 +161,9 @@ type mongoQuickFixEntryData struct {

// Reset deletes the store records and sets the seqnums back to 1.
func (store *mongoStore) Reset() error {
store.mu.Lock()
defer store.mu.Unlock()

msgFilter := generateMessageFilter(&store.sessionID)
_, err := store.db.Database(store.mongoDatabase).Collection(store.messagesCollection).DeleteMany(context.Background(), msgFilter)

Expand All @@ -180,6 +186,9 @@ func (store *mongoStore) Reset() error {

// Refresh reloads the store from the database.
func (store *mongoStore) Refresh() error {
store.mu.Lock()
defer store.mu.Unlock()

if err := store.cache.Reset(); err != nil {
return err
}
Expand Down Expand Up @@ -225,16 +234,29 @@ func (store *mongoStore) populateCache() error {

// NextSenderMsgSeqNum returns the next MsgSeqNum that will be sent.
func (store *mongoStore) NextSenderMsgSeqNum() int {
store.mu.Lock()
defer store.mu.Unlock()

return store.cache.NextSenderMsgSeqNum()
}

// NextTargetMsgSeqNum returns the next MsgSeqNum that should be received.
func (store *mongoStore) NextTargetMsgSeqNum() int {
store.mu.Lock()
defer store.mu.Unlock()

return store.cache.NextTargetMsgSeqNum()
}

// SetNextSenderMsgSeqNum sets the next MsgSeqNum that will be sent.
func (store *mongoStore) SetNextSenderMsgSeqNum(next int) error {
store.mu.Lock()
defer store.mu.Unlock()

return store.setNextSenderMsgSeqNum(next)
}

func (store *mongoStore) setNextSenderMsgSeqNum(next int) error {
msgFilter := generateMessageFilter(&store.sessionID)
sessionUpdate := generateMessageFilter(&store.sessionID)
sessionUpdate.IncomingSeqNum = store.cache.NextTargetMsgSeqNum()
Expand All @@ -248,6 +270,13 @@ func (store *mongoStore) SetNextSenderMsgSeqNum(next int) error {

// SetNextTargetMsgSeqNum sets the next MsgSeqNum that should be received.
func (store *mongoStore) SetNextTargetMsgSeqNum(next int) error {
store.mu.Lock()
defer store.mu.Unlock()

return store.setNextTargetMsgSeqNum(next)
}

func (store *mongoStore) setNextTargetMsgSeqNum(next int) error {
msgFilter := generateMessageFilter(&store.sessionID)
sessionUpdate := generateMessageFilter(&store.sessionID)
sessionUpdate.IncomingSeqNum = next
Expand All @@ -261,22 +290,35 @@ func (store *mongoStore) SetNextTargetMsgSeqNum(next int) error {

// IncrNextSenderMsgSeqNum increments the next MsgSeqNum that will be sent.
func (store *mongoStore) IncrNextSenderMsgSeqNum() error {
if err := store.SetNextSenderMsgSeqNum(store.cache.NextSenderMsgSeqNum() + 1); err != nil {
store.mu.Lock()
defer store.mu.Unlock()

return store.incrNextSenderMsgSeqNum()
}

func (store *mongoStore) incrNextSenderMsgSeqNum() error {
if err := store.setNextSenderMsgSeqNum(store.cache.NextSenderMsgSeqNum() + 1); err != nil {
return errors.Wrap(err, "save sequence number")
}
return nil
}

// IncrNextTargetMsgSeqNum increments the next MsgSeqNum that should be received.
func (store *mongoStore) IncrNextTargetMsgSeqNum() error {
if err := store.SetNextTargetMsgSeqNum(store.cache.NextTargetMsgSeqNum() + 1); err != nil {
store.mu.Lock()
defer store.mu.Unlock()

if err := store.setNextTargetMsgSeqNum(store.cache.NextTargetMsgSeqNum() + 1); err != nil {
return errors.Wrap(err, "save sequence number")
}
return nil
}

// CreationTime returns the creation time of the store.
func (store *mongoStore) CreationTime() time.Time {
store.mu.Lock()
defer store.mu.Unlock()

return store.cache.CreationTime()
}

Expand All @@ -285,6 +327,13 @@ func (store *mongoStore) SetCreationTime(_ time.Time) {
}

func (store *mongoStore) SaveMessage(seqNum int, msg []byte) (err error) {
store.mu.Lock()
defer store.mu.Unlock()

return store.saveMessage(seqNum, msg)
}

func (store *mongoStore) saveMessage(seqNum int, msg []byte) (err error) {
msgFilter := generateMessageFilter(&store.sessionID)
msgFilter.Msgseq = seqNum
msgFilter.Message = msg
Expand All @@ -293,13 +342,15 @@ func (store *mongoStore) SaveMessage(seqNum int, msg []byte) (err error) {
}

func (store *mongoStore) SaveMessageAndIncrNextSenderMsgSeqNum(seqNum int, msg []byte) error {
store.mu.Lock()
defer store.mu.Unlock()

if !store.allowTransactions {
err := store.SaveMessage(seqNum, msg)
err := store.saveMessage(seqNum, msg)
if err != nil {
return err
}
return store.IncrNextSenderMsgSeqNum()
return store.incrNextSenderMsgSeqNum()
}

// If the mongodb supports replicasets, perform this operation as a transaction instead-
Expand Down
227 changes: 227 additions & 0 deletions store/mongo/mongo_store_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,26 @@
package mongo

import (
"context"
"fmt"
"log"
"os"
"strings"
"sync"
"sync/atomic"
"testing"
"time"

"github.com/quickfixgo/quickfix"
"github.com/quickfixgo/quickfix/internal/testsuite"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/integration/mtest"
"go.mongodb.org/mongo-driver/mongo/options"
)

// MongoStoreTestSuite runs all tests in the message.StoreTestSuite against the MongoStore implementation.
Expand Down Expand Up @@ -73,3 +83,220 @@ func (suite *MongoStoreTestSuite) TearDownTest() {
func TestMongoStoreTestSuite(t *testing.T) {
suite.Run(t, new(MongoStoreTestSuite))
}

func newConcurrentMongoStore(t *testing.T, monitor *event.CommandMonitor) *mongoStore {
t.Helper()
uri := os.Getenv("MONGODB_TEST_CXN")
if uri == "" {
t.Skip("MONGODB_TEST_CXN is not set")
}
client, err := mongo.Connect(context.Background(), options.Client().ApplyURI(uri).
SetReplicaSet("replicaset").SetMonitor(monitor).SetTimeout(10*time.Second))
require.NoError(t, err)
sessionID := quickfix.SessionID{BeginString: "FIX.4.4", SenderCompID: "SENDER", TargetCompID: "TARGET"}
cache, err := quickfix.NewMemoryStoreFactory().Create(sessionID)
require.NoError(t, err)
store := &mongoStore{
sessionID: sessionID,
cache: cache,
db: client,
mongoDatabase: "quickfix_concurrency_" + primitive.NewObjectID().Hex(),
messagesCollection: "messages",
sessionsCollection: "sessions",
allowTransactions: true,
}
t.Cleanup(func() {
require.NoError(t, client.Database(store.mongoDatabase).Drop(context.Background()))
require.NoError(t, store.Close())
})
require.NoError(t, store.populateCache())
require.NoError(t, client.Database(store.mongoDatabase).CreateCollection(context.Background(), store.messagesCollection))
return store
}

func waitForMongoOperation(t *testing.T, result <-chan error) error {
t.Helper()
select {
case err := <-result:
return err
case <-time.After(10 * time.Second):
t.Fatal("MongoStore operation did not complete")
return nil
}
}

func TestMongoStoreConcurrentSendAndReceive(t *testing.T) {
for _, transactions := range []bool{true, false} {
t.Run(fmt.Sprintf("transactions=%t", transactions), func(t *testing.T) {
testMongoStoreConcurrentSend(t, transactions, (*mongoStore).IncrNextTargetMsgSeqNum, 2, 2)
})
}
}

func TestMongoStoreConcurrentResetAndRefresh(t *testing.T) {
t.Run("reset", func(t *testing.T) {
testMongoStoreConcurrentSend(t, true, (*mongoStore).Reset, 1, 1)
})
t.Run("refresh", func(t *testing.T) {
testMongoStoreConcurrentSend(t, true, (*mongoStore).Refresh, 2, 1)
})
}

func testMongoStoreConcurrentSend(t *testing.T, transactions bool, update func(*mongoStore) error, wantSender, wantTarget int) {
t.Helper()
inserted := make(chan struct{})
resume := make(chan struct{})
release := sync.OnceFunc(func() { close(resume) })
defer release()
var pauseInsert atomic.Bool
store := newConcurrentMongoStore(t, &event.CommandMonitor{
Succeeded: func(_ context.Context, ev *event.CommandSucceededEvent) {
if ev.CommandName == "insert" && pauseInsert.CompareAndSwap(true, false) {
close(inserted)
<-resume
}
},
})
store.allowTransactions = transactions
pauseInsert.Store(true)
sent := make(chan error, 1)
go func() { sent <- store.SaveMessageAndIncrNextSenderMsgSeqNum(1, []byte("message")) }()
select {
case <-inserted:
case <-time.After(10 * time.Second):
t.Fatal("message insert did not complete")
}

// With transactions, the insert establishes the sender's snapshot.
// Other session updates must wait until the entire send completes.
received := make(chan error, 1)
started := make(chan struct{})
go func() {
close(started)
received <- update(store)
}()
<-started
var receiveErr error
var receivedBeforeRelease bool
select {
case receiveErr = <-received:
receivedBeforeRelease = true
case <-time.After(100 * time.Millisecond):
}
release()
sendErr := waitForMongoOperation(t, sent)
if !receivedBeforeRelease {
receiveErr = waitForMongoOperation(t, received)
}
require.NoError(t, sendErr)
require.NoError(t, receiveErr)
require.False(t, receivedBeforeRelease, "session update completed before the send")
require.Equal(t, wantSender, store.NextSenderMsgSeqNum())
require.Equal(t, wantTarget, store.NextTargetMsgSeqNum())
require.NoError(t, store.Refresh())
require.Equal(t, wantSender, store.NextSenderMsgSeqNum())
require.Equal(t, wantTarget, store.NextTargetMsgSeqNum())
messages, err := store.GetMessages(1, 1)
require.NoError(t, err)
if wantSender == 1 {
require.Empty(t, messages)
} else {
require.Equal(t, [][]byte{[]byte("message")}, messages)
}
}

func TestMongoStoreConcurrentSequenceNumbers(t *testing.T) {
for _, transactions := range []bool{true, false} {
t.Run(fmt.Sprintf("transactions=%t", transactions), func(t *testing.T) {
store := newConcurrentMongoStore(t, nil)
store.allowTransactions = transactions
const count = 32
messages := make([][]byte, count)
for i := range messages {
messages[i] = []byte(fmt.Sprintf("message %d", i+1))
}
results := make(chan error, 2)
go func() {
for i, msg := range messages {
if err := store.SaveMessageAndIncrNextSenderMsgSeqNum(i+1, msg); err != nil {
results <- err
return
}
}
results <- nil
}()
go func() {
for i := 0; i < count; i++ {
if err := store.IncrNextTargetMsgSeqNum(); err != nil {
results <- err
return
}
}
results <- nil
}()
stopReading := make(chan struct{})
readerDone := make(chan struct{})
stop := sync.OnceFunc(func() { close(stopReading) })
defer stop()
go func() {
defer close(readerDone)
for {
select {
case <-stopReading:
return
default:
store.NextSenderMsgSeqNum()
store.NextTargetMsgSeqNum()
store.CreationTime()
}
}
}()
firstErr := waitForMongoOperation(t, results)
secondErr := waitForMongoOperation(t, results)
stop()
<-readerDone
require.NoError(t, firstErr)
require.NoError(t, secondErr)
require.Equal(t, count+1, store.NextSenderMsgSeqNum())
require.Equal(t, count+1, store.NextTargetMsgSeqNum())
require.NoError(t, store.Refresh())
require.Equal(t, count+1, store.NextSenderMsgSeqNum())
require.Equal(t, count+1, store.NextTargetMsgSeqNum())
actual, err := store.GetMessages(1, count)
require.NoError(t, err)
require.Equal(t, messages, actual)
})
}
}

func TestMongoStoreSendFailureUnlocks(t *testing.T) {
mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock))
for step, command := range []string{"insert", "update", "commitTransaction"} {
mt.Run(command, func(mt *mtest.T) {
cache, err := quickfix.NewMemoryStoreFactory().Create(quickfix.SessionID{})
require.NoError(mt, err)
store := &mongoStore{
cache: cache, db: mt.Client, mongoDatabase: mt.DB.Name(),
messagesCollection: "messages", sessionsCollection: "sessions", allowTransactions: true,
}
for i := 0; i < step; i++ {
mt.AddMockResponses(mtest.CreateSuccessResponse(bson.E{Key: "n", Value: 1}))
}
mt.AddMockResponses(
mtest.CreateCommandErrorResponse(mtest.CommandError{Code: 42, Message: "send failed"}),
mtest.CreateSuccessResponse(), // Abort the failed transaction, if necessary.
)
err = store.SaveMessageAndIncrNextSenderMsgSeqNum(1, []byte("message"))
var commandErr mongo.CommandError
require.ErrorAs(mt, err, &commandErr)
require.EqualValues(mt, 42, commandErr.Code)
mt.ClearMockResponses()
mt.AddMockResponses(mtest.CreateSuccessResponse(bson.E{Key: "n", Value: 1}))
result := make(chan error, 1)
go func() { result <- store.IncrNextTargetMsgSeqNum() }()
require.NoError(mt, waitForMongoOperation(mt.T, result))
require.Equal(mt, 1, store.NextSenderMsgSeqNum())
require.Equal(mt, 2, store.NextTargetMsgSeqNum())
})
}
}
Loading