1package store

2

3import (

4 "os"

5 "path/filepath"

6 "reflect"

7 "strings"

8 "testing"

9 "time"

10

11 "github.com/mjl-/bstore"

12

13 "github.com/mjl-/mox/mlog"

14 "github.com/mjl-/mox/mox-"

15)

16

17func TestThreadingUpgrade(t *testing.T) {

18 log := mlog.New("store", nil)

19 os.RemoveAll("../testdata/store/data")

20 mox.ConfigStaticPath = filepath.FromSlash("../testdata/store/mox.conf")

21 mox.MustLoadConfig(true, false)

22 acc, err := OpenAccount(log, "mjl")

23 tcheck(t, err, "open account")

24 defer func() {

25 err = acc.Close()

26 tcheck(t, err, "closing account")

27 acc.CheckClosed()

28 }()

29 defer Switchboard()()

30

31 // New account already has threading. Add some messages, check the threading.

32 deliver := func(recv time.Time, s string, expThreadID int64) Message {

33 t.Helper()

34 f, err := CreateMessageTemp(log, "account-test")

35 tcheck(t, err, "temp file")

36 defer os.Remove(f.Name())

37 defer f.Close()

38

39 s = strings.ReplaceAll(s, "\n", "\r\n")

40 m := Message{

41 Size: int64(len(s)),

42 MsgPrefix: []byte(s),

43 Received: recv,

44 }

45 err = acc.DeliverMailbox(log, "Inbox", &m, f)

46 tcheck(t, err, "deliver")

47 if expThreadID == 0 {

48 expThreadID = m.ID

49 }

50 if m.ThreadID != expThreadID {

51 t.Fatalf("got threadid %d, expected %d", m.ThreadID, expThreadID)

52 }

53 return m

54 }

55

56 now := time.Now()

57

58 m0 := deliver(now, "Message-ID: <m0@localhost>\nSubject: test1\n\ntest\n", 0)

59 m1 := deliver(now, "Message-ID: <m1@localhost>\nReferences: <m0@localhost>\nSubject: test1\n\ntest\n", m0.ID) // References.

60 m2 := deliver(now, "Message-ID: <m2@localhost>\nReferences: <m0@localhost>\nSubject: other\n\ntest\n", 0) // References, but different subject.

61 m3 := deliver(now, "Message-ID: <m3@localhost>\nIn-Reply-To: <m0@localhost>\nSubject: test1\n\ntest\n", m0.ID) // In-Reply-To.

62 m4 := deliver(now, "Message-ID: <m4@localhost>\nSubject: re: test1\n\ntest\n", m0.ID) // Subject.

63 m5 := deliver(now, "Message-ID: <m5@localhost>\nSubject: test1 (fwd)\n\ntest\n", m0.ID) // Subject.

64 m6 := deliver(now, "Message-ID: <m6@localhost>\nSubject: [fwd: test1]\n\ntest\n", m0.ID) // Subject.

65 m7 := deliver(now, "Message-ID: <m7@localhost>\nSubject: test1\n\ntest\n", 0) // Only subject, but not a response.

66

67 // Thread with a cyclic head, a self-referencing message.

68 c1 := deliver(now, "Message-ID: <c1@localhost>\nReferences: <c2@localhost>\nSubject: cycle0\n\ntest\n", 0) // Head cycle with m8.

69 c2 := deliver(now, "Message-ID: <c2@localhost>\nReferences: <c1@localhost>\nSubject: cycle0\n\ntest\n", c1.ID) // Head cycle with c1.

70 c3 := deliver(now, "Message-ID: <c3@localhost>\nReferences: <c1@localhost>\nSubject: cycle0\n\ntest\n", c1.ID) // Connected to one of the cycle elements.

71 c4 := deliver(now, "Message-ID: <c4@localhost>\nReferences: <c2@localhost>\nSubject: cycle0\n\ntest\n", c1.ID) // Connected to other cycle element.

72 c5 := deliver(now, "Message-ID: <c5@localhost>\nReferences: <c4@localhost>\nSubject: cycle0\n\ntest\n", c1.ID)

73 c5b := deliver(now, "Message-ID: <c5@localhost>\nReferences: <c4@localhost>\nSubject: cycle0\n\ntest\n", c1.ID) // Duplicate, e.g. Sent item, internal cycle during upgrade.

74 c6 := deliver(now, "Message-ID: <c6@localhost>\nReferences: <c5@localhost>\nSubject: cycle0\n\ntest\n", c1.ID)

75 c7 := deliver(now, "Message-ID: <c7@localhost>\nReferences: <c5@localhost> <c7@localhost>\nSubject: cycle0\n\ntest\n", c1.ID) // Self-referencing message that also points to actual parent.

76

77 // More than 2 messages to make a cycle.

78 d0 := deliver(now, "Message-ID: <d0@localhost>\nReferences: <d2@localhost>\nSubject: cycle1\n\ntest\n", 0)

79 d1 := deliver(now, "Message-ID: <d1@localhost>\nReferences: <d0@localhost>\nSubject: cycle1\n\ntest\n", d0.ID)

80 d2 := deliver(now, "Message-ID: <d2@localhost>\nReferences: <d1@localhost>\nSubject: cycle1\n\ntest\n", d0.ID)

81

82 // Cycle with messages delivered later. During import/upgrade, they will all be one thread.

83 e0 := deliver(now, "Message-ID: <e0@localhost>\nReferences: <e1@localhost>\nSubject: cycle2\n\ntest\n", 0)

84 e1 := deliver(now, "Message-ID: <e1@localhost>\nReferences: <e2@localhost>\nSubject: cycle2\n\ntest\n", 0)

85 e2 := deliver(now, "Message-ID: <e2@localhost>\nReferences: <e0@localhost>\nSubject: cycle2\n\ntest\n", e0.ID)

86

87 // Three messages in a cycle (f1, f2, f3), with one with an additional ancestor (f4) which is ignored due to the cycle. Has different threads during import.

88 f0 := deliver(now, "Message-ID: <f0@localhost>\nSubject: cycle3\n\ntest\n", 0)

89 f1 := deliver(now, "Message-ID: <f1@localhost>\nReferences: <f0@localhost> <f2@localhost>\nSubject: cycle3\n\ntest\n", f0.ID)

90 f2 := deliver(now, "Message-ID: <f2@localhost>\nReferences: <f3@localhost>\nSubject: cycle3\n\ntest\n", 0)

91 f3 := deliver(now, "Message-ID: <f3@localhost>\nReferences: <f1@localhost>\nSubject: cycle3\n\ntest\n", f0.ID)

92

93 // Duplicate single message (no larger thread).

94 g0 := deliver(now, "Message-ID: <g0@localhost>\nSubject: dup\n\ntest\n", 0)

95 g0b := deliver(now, "Message-ID: <g0@localhost>\nSubject: dup\n\ntest\n", g0.ID)

96

97 // Duplicate message with a child message.

98 h0 := deliver(now, "Message-ID: <h0@localhost>\nSubject: dup2\n\ntest\n", 0)

99 h0b := deliver(now, "Message-ID: <h0@localhost>\nSubject: dup2\n\ntest\n", h0.ID)

100 h1 := deliver(now, "Message-ID: <h1@localhost>\nReferences: <h0@localhost>\nSubject: dup2\n\ntest\n", h0.ID)

101

102 // Message has itself as reference.

103 s0 := deliver(now, "Message-ID: <s0@localhost>\nReferences: <s0@localhost>\nSubject: self-referencing message\n\ntest\n", 0)

104

105 // Message with \0 in subject, should get an empty base subject.

106 b0 := deliver(now, "Message-ID: <b0@localhost>\nSubject: bad\u0000subject\n\ntest\n", 0)

107 b1 := deliver(now, "Message-ID: <b1@localhost>\nSubject: bad\u0000subject\n\ntest\n", 0) // Not matched.

108

109 // Interleaved duplicate threaded messages. First child, then parent, then duplicate parent, then duplicat child again.

110 i0 := deliver(now, "Message-ID: <i0@localhost>\nReferences: <i1@localhost>\nSubject: interleaved duplicate\n\ntest\n", 0)

111 i1 := deliver(now, "Message-ID: <i1@localhost>\nSubject: interleaved duplicate\n\ntest\n", 0)

112 i2 := deliver(now, "Message-ID: <i1@localhost>\nSubject: interleaved duplicate\n\ntest\n", i1.ID)

113 i3 := deliver(now, "Message-ID: <i0@localhost>\nReferences: <i1@localhost>\nSubject: interleaved duplicate\n\ntest\n", i0.ID)

114

115 j0 := deliver(now, "Message-ID: <j0@localhost>\nReferences: <>\nSubject: empty id in references\n\ntest\n", 0)

116

117 dbpath := acc.DBPath

118 err = acc.Close()

119 tcheck(t, err, "close account")

120 acc.CheckClosed()

121

122 // Now clear the threading upgrade, and the threading fields and close the account.

123 // We open the database file directly, so we don't trigger the consistency checker.

124 db, err := bstore.Open(ctxbg, dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)

125 err = db.Write(ctxbg, func(tx *bstore.Tx) error {

126 up := Upgrade{ID: 1}

127 err := tx.Delete(&up)

128 tcheck(t, err, "delete upgrade")

129

130 q := bstore.QueryTx[Message](tx)

131 _, err = q.UpdateFields(map[string]any{

132 "MessageID": "",

133 "SubjectBase": "",

134 "ThreadID": int64(0),

135 "ThreadParentIDs": []int64(nil),

136 "ThreadMissingLink": false,

137 })

138 return err

139 })

140 tcheck(t, err, "reset threading fields")

141 err = db.Close()

142 tcheck(t, err, "closing db")

143

144 // Open the account again, that should get the account upgraded. Wait for upgrade to finish.

145 acc, err = OpenAccount(log, "mjl")

146 tcheck(t, err, "open account")

147 err = acc.ThreadingWait(log)

148 tcheck(t, err, "wait for threading")

149

150 check := func(id int64, expThreadID int64, expParentIDs []int64, expMissingLink bool) {

151 t.Helper()

152

153 m := Message{ID: id}

154 err := acc.DB.Get(ctxbg, &m)

155 tcheck(t, err, "get message")

156 if m.ThreadID != expThreadID || !reflect.DeepEqual(m.ThreadParentIDs, expParentIDs) || m.ThreadMissingLink != expMissingLink {

157 t.Fatalf("got thread id %d, parent ids %v, missing link %v, expected %d %v %v", m.ThreadID, m.ThreadParentIDs, m.ThreadMissingLink, expThreadID, expParentIDs, expMissingLink)

158 }

159 }

160

161 parents0 := []int64{m0.ID}

162 check(m0.ID, m0.ID, nil, false)

163 check(m1.ID, m0.ID, parents0, false)

164 check(m2.ID, m2.ID, nil, true)

165 check(m3.ID, m0.ID, parents0, false)

166 check(m4.ID, m0.ID, parents0, true)

167 check(m5.ID, m0.ID, parents0, true)

168 check(m6.ID, m0.ID, parents0, true)

169 check(m7.ID, m7.ID, nil, false)

170

171 check(c1.ID, c1.ID, nil, true) // Head of cycle, hence missing link

172 check(c2.ID, c1.ID, []int64{c1.ID}, false)

173 check(c3.ID, c1.ID, []int64{c1.ID}, false)

174 check(c4.ID, c1.ID, []int64{c2.ID, c1.ID}, false)

175 check(c5.ID, c1.ID, []int64{c4.ID, c2.ID, c1.ID}, false)

176 check(c5b.ID, c1.ID, []int64{c5.ID, c4.ID, c2.ID, c1.ID}, true)

177 check(c6.ID, c1.ID, []int64{c5.ID, c4.ID, c2.ID, c1.ID}, false)

178 check(c7.ID, c1.ID, []int64{c5.ID, c4.ID, c2.ID, c1.ID}, true)

179

180 check(d0.ID, d0.ID, nil, true)

181 check(d1.ID, d0.ID, []int64{d0.ID}, false)

182 check(d2.ID, d0.ID, []int64{d1.ID, d0.ID}, false)

183

184 check(e0.ID, e0.ID, nil, true)

185 check(e1.ID, e0.ID, []int64{e2.ID, e0.ID}, false)

186 check(e2.ID, e0.ID, []int64{e0.ID}, false)

187

188 check(f0.ID, f0.ID, nil, false)

189 check(f1.ID, f1.ID, nil, true)

190 check(f2.ID, f1.ID, []int64{f3.ID, f1.ID}, false)

191 check(f3.ID, f1.ID, []int64{f1.ID}, false)

192

193 check(g0.ID, g0.ID, nil, false)

194 check(g0b.ID, g0.ID, []int64{g0.ID}, true)

195

196 check(h0.ID, h0.ID, nil, false)

197 check(h0b.ID, h0.ID, []int64{h0.ID}, true)

198 check(h1.ID, h0.ID, []int64{h0.ID}, false)

199

200 check(s0.ID, s0.ID, nil, true)

201

202 check(b0.ID, b0.ID, nil, false)

203 check(b1.ID, b1.ID, nil, false)

204

205 check(i0.ID, i1.ID, []int64{i1.ID}, false)

206 check(i1.ID, i1.ID, nil, false)

207 check(i2.ID, i1.ID, []int64{i1.ID}, true)

208 check(i3.ID, i1.ID, []int64{i0.ID, i1.ID}, true)

209

210 check(j0.ID, j0.ID, nil, false)

211}

212