1package imapserver
2
3import (
4 "fmt"
5 "strings"
6 "testing"
7
8 "github.com/mjl-/bstore"
9
10 "github.com/mjl-/mox/imapclient"
11 "github.com/mjl-/mox/mox-"
12 "github.com/mjl-/mox/store"
13)
14
15func TestCondstore(t *testing.T) {
16 testCondstoreQresync(t, false)
17}
18
19func TestQresync(t *testing.T) {
20 testCondstoreQresync(t, true)
21}
22
23func testCondstoreQresync(t *testing.T, qresync bool) {
24 defer mockUIDValidity()()
25 tc := start(t)
26 defer tc.close()
27
28 // todo: check whether marking \seen will cause modseq to be returned in case of qresync.
29
30 // Check basic requirements of CONDSTORE.
31
32 capability := "Condstore"
33 if qresync {
34 capability = "Qresync"
35 }
36
37 tc.client.Login("mjl@mox.example", "testtest")
38 tc.client.Enable(capability)
39 tc.transactf("ok", "Select inbox")
40 tc.xuntaggedOpt(false, imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(1), More: "x"}})
41
42 // First some tests without any messages.
43
44 tc.transactf("ok", "Status inbox (Highestmodseq)")
45 tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"HIGHESTMODSEQ": 1}})
46
47 // No messages, no matches.
48 tc.transactf("ok", "Uid Fetch 1:* (Flags) (Changedsince 12345)")
49 tc.xuntagged()
50
51 // Also no messages with modseq 1, which we internally turn into modseq 0.
52 tc.transactf("ok", "Uid Fetch 1:* (Flags) (Changedsince 1)")
53 tc.xuntagged()
54
55 // Also try with modseq attribute.
56 tc.transactf("ok", "Uid Fetch 1:* (Flags Modseq) (Changedsince 1)")
57 tc.xuntagged()
58
59 // Search with modseq search criteria.
60 tc.transactf("ok", "Search Modseq 0") // Zero is valid, matches all.
61 tc.xsearch()
62
63 tc.transactf("ok", "Search Modseq 1") // Converted to zero internally.
64 tc.xsearch()
65
66 tc.transactf("ok", "Search Modseq 12345")
67 tc.xsearch()
68
69 tc.transactf("ok", `Search Modseq "/Flags/\\Draft" All 12345`)
70 tc.xsearch()
71
72 tc.transactf("ok", `Search Or Modseq 12345 Modseq 54321`)
73 tc.xsearch()
74
75 // esearch
76 tc.transactf("ok", "Search Return (All) Modseq 123")
77 tc.xesearch(imapclient.UntaggedEsearch{})
78
79 // Now we add, delete, expunge, modify some message flags and check if the
80 // responses are correct. We check in both a condstore-enabled and one without that
81 // we get the correct notifications.
82
83 // First we add 3 messages as if they were added before we implemented CONDSTORE.
84 // Later on, we'll update the second, and delete the third, leaving the first
85 // unmodified. Those messages have modseq 0 in the database. We use append for
86 // convenience, then adjust the records in the database.
87 // We have a workaround below to prevent triggering the consistency checker.
88 tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
89 tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
90 tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
91 _, err := bstore.QueryDB[store.Message](ctxbg, tc.account.DB).UpdateFields(map[string]any{
92 "ModSeq": 0,
93 "CreateSeq": 0,
94 })
95 tcheck(t, err, "clearing modseq from messages")
96 err = tc.account.DB.Update(ctxbg, &store.SyncState{ID: 1, LastModSeq: 1})
97 tcheck(t, err, "resetting modseq state")
98
99 tc.client.Create("otherbox")
100
101 // tc2 is a client without condstore, so no modseq responses.
102 tc2 := startNoSwitchboard(t)
103 defer tc2.close()
104 tc2.client.Login("mjl@mox.example", "testtest")
105 tc2.client.Select("inbox")
106
107 // tc3 is a client with condstore, so with modseq responses.
108 tc3 := startNoSwitchboard(t)
109 defer tc3.close()
110 tc3.client.Login("mjl@mox.example", "testtest")
111 tc3.client.Enable(capability)
112 tc3.client.Select("inbox")
113
114 var clientModseq int64 = 1 // We track the client-side modseq for inbox. Not a store.ModSeq.
115
116 // Add messages to: inbox, otherbox, inbox, inbox.
117 // We have these messages in order of modseq: 2+1 in inbox, 1 in otherbox, 2 in inbox.
118 // The original two in inbox appear to have modseq 1 (with 0 stored in the database).
119 // The ones we insert below will start with modseq 2. So we'll have modseq 1-5.
120 tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
121 tc.xuntagged(imapclient.UntaggedExists(4))
122 tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 4})
123
124 tc.transactf("ok", "Append otherbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
125 tc.xuntagged()
126 tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 2, UID: 1})
127
128 tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
129 tc.xuntagged(imapclient.UntaggedExists(5))
130 tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 5})
131
132 tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
133 tc.xuntagged(imapclient.UntaggedExists(6))
134 tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 6})
135
136 tc2.transactf("ok", "Noop")
137 noflags := imapclient.FetchFlags(nil)
138 tc2.xuntagged(
139 imapclient.UntaggedExists(6),
140 imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(4), noflags}},
141 imapclient.UntaggedFetch{Seq: 5, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags}},
142 imapclient.UntaggedFetch{Seq: 6, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags}},
143 )
144
145 tc3.transactf("ok", "Noop")
146 tc3.xuntagged(
147 imapclient.UntaggedExists(6),
148 imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(4), noflags, imapclient.FetchModSeq(clientModseq + 1)}},
149 imapclient.UntaggedFetch{Seq: 5, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(clientModseq + 3)}},
150 imapclient.UntaggedFetch{Seq: 6, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq + 4)}},
151 )
152
153 mox.SetPedantic(true)
154 tc.transactf("bad", `Fetch 1 Flags (Changedsince 0)`) // 0 not allowed in syntax.
155 mox.SetPedantic(false)
156 tc.transactf("ok", "Uid fetch 1 (Flags) (Changedsince 0)")
157 tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(clientModseq)}})
158
159 clientModseq += 4 // Four messages, over two mailboxes, modseq is per account.
160
161 // Check highestmodseq for mailboxes.
162 tc.transactf("ok", "Status inbox (highestmodseq)")
163 tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"HIGHESTMODSEQ": clientModseq}})
164
165 tc.transactf("ok", "Status otherbox (highestmodseq)")
166 tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "otherbox", Attrs: map[string]int64{"HIGHESTMODSEQ": 3}})
167
168 // Check highestmodseq when we select.
169 tc.transactf("ok", "Examine otherbox")
170 tc.xuntaggedOpt(false, imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(3), More: "x"}})
171
172 tc.transactf("ok", "Select inbox")
173 tc.xuntaggedOpt(false, imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(clientModseq), More: "x"}})
174
175 // Check fetch modseq response and changedsince.
176 tc.transactf("ok", `Fetch 1 (Modseq)`)
177 tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchModSeq(1)}})
178
179 // Without modseq attribute, even with condseq enabled, there is no modseq response.
180 // For QRESYNC, we must always send MODSEQ for UID FETCH commands, but not for FETCH commands. ../rfc/7162:1427
181 tc.transactf("ok", `Uid Fetch 1 Flags`)
182 if qresync {
183 tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(1)}})
184 } else {
185 tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags}})
186 }
187 tc.transactf("ok", `Fetch 1 Flags`)
188 tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags}})
189
190 // When CHANGEDSINCE is present, MODSEQ is automatically added to the response.
191 // ../rfc/7162:871
192 // ../rfc/7162:877
193 tc.transactf("ok", `Fetch 1 Flags (Changedsince 1)`)
194 tc.xuntagged()
195 tc.transactf("ok", `Fetch 1,4 Flags (Changedsince 1)`)
196 tc.xuntagged(imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(4), noflags, imapclient.FetchModSeq(2)}})
197 tc.transactf("ok", `Fetch 2 Flags (Changedsince 2)`)
198 tc.xuntagged()
199
200 // store and uid store.
201
202 // unchangedsince 0 never passes the check. ../rfc/7162:640
203 tc.transactf("ok", `Store 1 (Unchangedsince 0) +Flags ()`)
204 tc.xcodeArg(imapclient.CodeModified(xparseNumSet("1")))
205 tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(1)}})
206
207 // Modseq is 2 for first condstore-aware-appended message, so also no match.
208 tc.transactf("ok", `Uid Store 4 (Unchangedsince 1) +Flags ()`)
209 tc.xcodeArg(imapclient.CodeModified(xparseNumSet("4")))
210
211 // Modseq is 1 for original message.
212 tc.transactf("ok", `Store 1 (Unchangedsince 1) +Flags (label1)`)
213 tc.xcode("") // No MODIFIED.
214 clientModseq++
215 tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags{"label1"}, imapclient.FetchModSeq(clientModseq)}})
216 tc2.transactf("ok", "Noop")
217 tc2.xuntagged(
218 imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags{"label1"}}},
219 )
220 tc3.transactf("ok", "Noop")
221 tc3.xuntagged(
222 imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags{"label1"}, imapclient.FetchModSeq(clientModseq)}},
223 )
224
225 // Modify same message twice. Check that second application doesn't fail due to
226 // modseq change made in the first application. ../rfc/7162:823
227 tc.transactf("ok", `Uid Store 1,1 (Unchangedsince %d) -Flags (label1)`, clientModseq)
228 clientModseq++
229 tc.xcode("") // No MODIFIED.
230 tc.xuntagged(
231 imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil), imapclient.FetchModSeq(clientModseq)}},
232 )
233 // We do broadcast the changes twice. Not great, but doesn't hurt. This isn't common.
234 tc2.transactf("ok", "Noop")
235 tc2.xuntagged(
236 imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil)}},
237 )
238 tc3.transactf("ok", "Noop")
239 tc3.xuntagged(
240 imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil), imapclient.FetchModSeq(clientModseq)}},
241 )
242
243 // Modify without actually changing flags, there will be no new modseq and no broadcast.
244 tc.transactf("ok", `Store 1 (Unchangedsince %d) -Flags (label1)`, clientModseq)
245 tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil), imapclient.FetchModSeq(clientModseq)}})
246 tc.xcode("") // No MODIFIED.
247 tc2.transactf("ok", "Noop")
248 tc2.xuntagged()
249 tc3.transactf("ok", "Noop")
250 tc3.xuntagged()
251
252 // search with modseq criteria and modseq in response
253 tc.transactf("ok", "Search Modseq %d", clientModseq)
254 tc.xsearchmodseq(clientModseq, 1)
255
256 tc.transactf("ok", "Uid Search Or Modseq %d Modseq %d", clientModseq, clientModseq)
257 tc.xsearchmodseq(clientModseq, 1)
258
259 // esearch
260 tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq %d", clientModseq)
261 tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 1, All: esearchall0("1"), ModSeq: clientModseq})
262
263 uint32ptr := func(v uint32) *uint32 {
264 return &v
265 }
266 tc.transactf("ok", "Search Return (Count) 1:* Modseq 0")
267 tc.xesearch(imapclient.UntaggedEsearch{Count: uint32ptr(6), ModSeq: clientModseq})
268
269 tc.transactf("ok", "Search Return (Min Max) 1:* Modseq 0")
270 tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 6, ModSeq: clientModseq})
271
272 tc.transactf("ok", "Search Return (Min) 1:* Modseq 0")
273 tc.xesearch(imapclient.UntaggedEsearch{Min: 1, ModSeq: clientModseq})
274
275 // expunge, we expunge the third and fourth messages. The third was originally with
276 // modseq 0, the fourth was added with condstore-aware append.
277 tc.transactf("ok", `Store 3:4 +Flags (\Deleted)`)
278 clientModseq++
279 tc2.transactf("ok", "Noop")
280 tc3.transactf("ok", "Noop")
281 tc.transactf("ok", "Expunge")
282 clientModseq++
283 if qresync {
284 tc.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("3:4")})
285 } else {
286 tc.xuntagged(imapclient.UntaggedExpunge(3), imapclient.UntaggedExpunge(3))
287 }
288 tc.xcodeArg(imapclient.CodeHighestModSeq(clientModseq))
289 tc2.transactf("ok", "Noop")
290 tc2.xuntagged(imapclient.UntaggedExpunge(3), imapclient.UntaggedExpunge(3))
291 tc3.transactf("ok", "Noop")
292 if qresync {
293 tc3.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("3:4")})
294 } else {
295 tc3.xuntagged(imapclient.UntaggedExpunge(3), imapclient.UntaggedExpunge(3))
296 }
297
298 // Again after expunge: status, select, conditional store/fetch/search
299 tc.transactf("ok", "Status inbox (Highestmodseq Messages Unseen Deleted)")
300 tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"MESSAGES": 4, "UNSEEN": 4, "DELETED": 0, "HIGHESTMODSEQ": clientModseq}})
301
302 tc.transactf("ok", "Close")
303 tc.transactf("ok", "Select inbox")
304 tc.xuntaggedOpt(false,
305 imapclient.UntaggedExists(4),
306 imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(clientModseq), More: "x"}},
307 )
308
309 tc.transactf("ok", `Fetch 1:* (Modseq)`)
310 tc.xuntagged(
311 imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchModSeq(7)}},
312 imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), imapclient.FetchModSeq(1)}},
313 imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), imapclient.FetchModSeq(4)}},
314 imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), imapclient.FetchModSeq(5)}},
315 )
316 // Expunged messages, with higher modseq, should not show up.
317 tc.transactf("ok", "Uid Fetch 1:* (flags) (Changedsince 7)")
318 tc.xuntagged()
319
320 // search
321 tc.transactf("ok", "Search Modseq 7")
322 tc.xsearchmodseq(7, 1)
323 tc.transactf("ok", "Search Modseq 8")
324 tc.xsearch()
325
326 // esearch
327 tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq 7")
328 tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 1, All: esearchall0("1"), ModSeq: 7})
329 tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq 8")
330 tc.xuntagged(imapclient.UntaggedEsearch{Correlator: tc.client.LastTag})
331
332 // store, cannot modify expunged messages.
333 tc.transactf("ok", `Uid Store 3,4 (Unchangedsince %d) +Flags (label2)`, clientModseq)
334 tc.xuntagged()
335 tc.xcode("") // Not MODIFIED.
336 tc.transactf("ok", `Uid Store 3,4 +Flags (label2)`)
337 tc.xuntagged()
338 tc.xcode("") // Not MODIFIED.
339
340 // Check all condstore-enabling commands (and their syntax), ../rfc/7162:368
341
342 // We start a new connection, do the thing that should enable condstore, then
343 // change flags of a message in another connection, do a noop in the new connection
344 // which should result in an untagged fetch that includes modseq, the indicator
345 // that condstore was indeed enabled. It's a bit complicated, but i don't think
346 // there is a clearly specified mechanism to find out which capabilities are
347 // enabled at any point.
348 var tagcount int
349 checkCondstoreEnabled := func(fn func(xtc *testconn)) {
350 t.Helper()
351
352 xtc := startNoSwitchboard(t)
353 // We have modified modseq & createseq to 0 above for testing that case. Don't
354 // trigger the consistency checker.
355 store.CheckConsistencyOnClose = false
356 defer func() {
357 xtc.close()
358 store.CheckConsistencyOnClose = true
359 }()
360 xtc.client.Login("mjl@mox.example", "testtest")
361 fn(xtc)
362 tagcount++
363 label := fmt.Sprintf("l%d", tagcount)
364 tc.transactf("ok", "Store 4 Flags (%s)", label)
365 clientModseq++
366 xtc.transactf("ok", "Noop")
367 xtc.xuntagged(imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), imapclient.FetchFlags{label}, imapclient.FetchModSeq(clientModseq)}})
368 }
369 // SELECT/EXAMINE with CONDSTORE parameter, ../rfc/7162:373
370 checkCondstoreEnabled(func(xtc *testconn) {
371 t.Helper()
372 xtc.transactf("ok", "Select inbox (Condstore)")
373 })
374 // STATUS with HIGHESTMODSEQ attribute, ../rfc/7162:375
375 checkCondstoreEnabled(func(xtc *testconn) {
376 t.Helper()
377 xtc.transactf("ok", "Status otherbox (Highestmodseq)")
378 xtc.transactf("ok", "Select inbox")
379 })
380 // FETCH with MODSEQ ../rfc/7162:377
381 checkCondstoreEnabled(func(xtc *testconn) {
382 t.Helper()
383 xtc.transactf("ok", "Select inbox")
384 xtc.transactf("ok", "Fetch 4 (Modseq)")
385 })
386 // SEARCH with MODSEQ ../rfc/7162:377
387 checkCondstoreEnabled(func(xtc *testconn) {
388 t.Helper()
389 xtc.transactf("ok", "Select inbox")
390 xtc.transactf("ok", "Search 4 Modseq 1")
391 })
392 // FETCH with CHANGEDSINCE ../rfc/7162:380
393 checkCondstoreEnabled(func(xtc *testconn) {
394 t.Helper()
395 xtc.transactf("ok", "Select inbox")
396 xtc.transactf("ok", "Fetch 4 (Flags) (Changedsince %d)", clientModseq)
397 })
398 // STORE with UNCHANGEDSINCE ../rfc/7162:382
399 checkCondstoreEnabled(func(xtc *testconn) {
400 t.Helper()
401 xtc.transactf("ok", "Select inbox")
402 xtc.transactf("ok", "Store 4 (Unchangedsince 0) Flags ()")
403 })
404 // ENABLE CONDSTORE ../rfc/7162:384
405 checkCondstoreEnabled(func(xtc *testconn) {
406 t.Helper()
407 xtc.transactf("ok", "Enable Condstore")
408 xtc.transactf("ok", "Select inbox")
409 })
410 // ENABLE QRESYNC ../rfc/7162:1390
411 checkCondstoreEnabled(func(xtc *testconn) {
412 t.Helper()
413 xtc.transactf("ok", "Enable Qresync")
414 xtc.transactf("ok", "Select inbox")
415 })
416 tc.transactf("ok", "Store 4 Flags ()")
417 clientModseq++
418
419 if qresync {
420 testQresync(t, tc, clientModseq)
421 }
422
423 // Continue with some tests that further change the data.
424 // First we copy messages to a new mailbox, and check we get new modseq for those
425 // messages.
426 tc.transactf("ok", "Select otherbox")
427 tc2.transactf("ok", "Noop")
428 tc3.transactf("ok", "Noop")
429 tc.transactf("ok", "Copy 1 inbox")
430 clientModseq++
431 tc2.transactf("ok", "Noop")
432 tc3.transactf("ok", "Noop")
433 tc2.xuntagged(
434 imapclient.UntaggedExists(5),
435 imapclient.UntaggedFetch{Seq: 5, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(7), noflags}},
436 )
437 tc3.xuntagged(
438 imapclient.UntaggedExists(5),
439 imapclient.UntaggedFetch{Seq: 5, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(7), noflags, imapclient.FetchModSeq(clientModseq)}},
440 )
441
442 // Then we move some messages, and check if we get expunged/vanished in original
443 // and untagged fetch with modseq in destination mailbox.
444 // tc2o is a client without condstore, so no modseq responses.
445 tc2o := startNoSwitchboard(t)
446 defer tc2o.close()
447 tc2o.client.Login("mjl@mox.example", "testtest")
448 tc2o.client.Select("otherbox")
449
450 // tc3o is a client with condstore, so with modseq responses.
451 tc3o := startNoSwitchboard(t)
452 defer tc3o.close()
453 tc3o.client.Login("mjl@mox.example", "testtest")
454 tc3o.client.Enable(capability)
455 tc3o.client.Select("otherbox")
456
457 tc.transactf("ok", "Select inbox")
458 tc.transactf("ok", "Uid Move 2:4 otherbox") // Only UID 2, because UID 3 and 4 have already been expunged.
459 clientModseq++
460 if qresync {
461 tc.xuntaggedOpt(false, imapclient.UntaggedVanished{UIDs: xparseNumSet("2")})
462 tc.xcodeArg(imapclient.CodeHighestModSeq(clientModseq))
463 } else {
464 tc.xuntaggedOpt(false, imapclient.UntaggedExpunge(2))
465 tc.xcode("")
466 }
467 tc2.transactf("ok", "Noop")
468 tc2.xuntagged(imapclient.UntaggedExpunge(2))
469 tc3.transactf("ok", "Noop")
470 if qresync {
471 tc3.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("2")})
472 } else {
473 tc3.xuntagged(imapclient.UntaggedExpunge(2))
474 }
475 tc2o.transactf("ok", "Noop")
476 tc2o.xuntagged(
477 imapclient.UntaggedExists(2),
478 imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), noflags}},
479 )
480 tc3o.transactf("ok", "Noop")
481 tc3o.xuntagged(
482 imapclient.UntaggedExists(2),
483 imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), noflags, imapclient.FetchModSeq(clientModseq)}},
484 )
485
486 // Restore valid modseq/createseq for the consistency checker.
487 _, err = bstore.QueryDB[store.Message](ctxbg, tc.account.DB).FilterEqual("CreateSeq", int64(0)).UpdateNonzero(store.Message{CreateSeq: 2})
488 tcheck(t, err, "updating modseq/createseq to valid values")
489 _, err = bstore.QueryDB[store.Message](ctxbg, tc.account.DB).FilterEqual("ModSeq", int64(0)).UpdateNonzero(store.Message{ModSeq: 2})
490 tcheck(t, err, "updating modseq/createseq to valid values")
491 tc2o.close()
492 tc2o = nil
493 tc3o.close()
494 tc3o = nil
495
496 // Then we rename inbox, which is special because it moves messages away instead of
497 // actually moving the mailbox. The mailbox stays and is cleared, so we check if we
498 // get expunged/vanished messages.
499 tc.transactf("ok", "Rename inbox oldbox")
500 // todo spec: server doesn't respond with untagged responses, find rfc reference that says this is ok.
501 tc2.transactf("ok", "Noop")
502 tc2.xuntagged(
503 imapclient.UntaggedList{Separator: '/', Mailbox: "oldbox"},
504 imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1),
505 )
506 tc3.transactf("ok", "Noop")
507 if qresync {
508 tc3.xuntagged(
509 imapclient.UntaggedList{Separator: '/', Mailbox: "oldbox"},
510 imapclient.UntaggedVanished{UIDs: xparseNumSet("1,5:7")},
511 )
512 } else {
513 tc3.xuntagged(
514 imapclient.UntaggedList{Separator: '/', Mailbox: "oldbox"},
515 imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1),
516 )
517 }
518
519 // Then we delete otherbox (we cannot delete inbox). We don't keep any history for removed mailboxes, so not actually a special case.
520 tc.transactf("ok", "Delete otherbox")
521}
522
523func testQresync(t *testing.T, tc *testconn, clientModseq int64) {
524 // Vanished on non-uid fetch is not allowed. ../rfc/7162:1693
525 tc.transactf("bad", "fetch 1:* (Flags) (Changedsince 1 Vanished)")
526
527 // Vanished without changedsince is not allowed. ../rfc/7162:1701
528 tc.transactf("bad", "Uid Fetch 1:* (Flags) (Vanished)")
529
530 // Vanished not allowed without first enabling qresync. ../rfc/7162:1697
531 xtc := startNoSwitchboard(t)
532 xtc.client.Login("mjl@mox.example", "testtest")
533 xtc.transactf("ok", "Select inbox (Condstore)")
534 xtc.transactf("bad", "Uid Fetch 1:* (Flags) (Changedsince 1 Vanished)")
535 // Prevent triggering the consistency checker, we still have modseq/createseq at 0.
536 store.CheckConsistencyOnClose = false
537 xtc.close()
538 store.CheckConsistencyOnClose = true
539 xtc = nil
540
541 // Check that we get proper vanished responses.
542 tc.transactf("ok", "Uid Fetch 1:* (Flags) (Changedsince 1 Vanished)")
543 noflags := imapclient.FetchFlags(nil)
544 tc.xuntagged(
545 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
546 imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
547 imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
548 imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
549 )
550
551 // select/examine with qresync parameters, including the various optional fields.
552 tc.transactf("ok", "Close")
553
554 // Must enable qresync explicitly before using. ../rfc/7162:1446
555 xtc = startNoSwitchboard(t)
556 xtc.client.Login("mjl@mox.example", "testtest")
557 xtc.transactf("bad", "Select inbox (Qresync 1 0)")
558 // Prevent triggering the consistency checker, we still have modseq/createseq at 0.
559 store.CheckConsistencyOnClose = false
560 xtc.close()
561 store.CheckConsistencyOnClose = true
562 xtc = nil
563
564 tc.transactf("bad", "Select inbox (Qresync (0 1))") // Both args must be > 0.
565 tc.transactf("bad", "Select inbox (Qresync (1 0))") // Both args must be > 0.
566 tc.transactf("bad", "Select inbox (Qresync)") // Two args are minimum.
567 tc.transactf("bad", "Select inbox (Qresync (1))") // Two args are minimum.
568 tc.transactf("bad", "Select inbox (Qresync (1 1 1:*))") // Known UIDs, * not allowed.
569 tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1:* 1:6)))") // Known seqset cannot have *.
570 tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1:6 1:*)))") // Known uidset cannot have *.
571 tc.transactf("bad", "Select inbox (Qresync (1 1) qresync (1 1))") // Duplicate qresync.
572
573 flags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent l1 l2 l3 l4 l5 l6 l7 l8 label1`, " ")
574 permflags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent \*`, " ")
575 uflags := imapclient.UntaggedFlags(flags)
576 upermflags := imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "PERMANENTFLAGS", CodeArg: imapclient.CodeList{Code: "PERMANENTFLAGS", Args: permflags}, More: "x"}}
577
578 baseUntagged := []imapclient.Untagged{
579 uflags,
580 upermflags,
581 imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox"},
582 imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UIDNEXT", CodeArg: imapclient.CodeUint{Code: "UIDNEXT", Num: 7}, More: "x"}},
583 imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UIDVALIDITY", CodeArg: imapclient.CodeUint{Code: "UIDVALIDITY", Num: 1}, More: "x"}},
584 imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UNSEEN", CodeArg: imapclient.CodeUint{Code: "UNSEEN", Num: 1}, More: "x"}},
585 imapclient.UntaggedRecent(0),
586 imapclient.UntaggedExists(4),
587 imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(clientModseq), More: "x"}},
588 }
589
590 makeUntagged := func(l ...imapclient.Untagged) []imapclient.Untagged {
591 return append(append([]imapclient.Untagged{}, baseUntagged...), l...)
592 }
593
594 // uidvalidity 1, highest known modseq 1, sends full current state.
595 tc.transactf("ok", "Select inbox (Qresync (1 1))")
596 tc.xuntagged(
597 makeUntagged(
598 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
599 imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
600 imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
601 imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
602 )...,
603 )
604
605 // Uidvalidity mismatch, server will not send any changes, so it's just a regular open.
606 tc.transactf("ok", "Close")
607 tc.transactf("ok", "Select inbox (Qresync (2 1))")
608 tc.xuntagged(baseUntagged...)
609
610 // We can tell which UIDs we know. First, send broader range then exist, should work.
611 tc.transactf("ok", "Close")
612 tc.transactf("ok", "Select inbox (Qresync (1 1 1:7))")
613 tc.xuntagged(
614 makeUntagged(
615 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
616 imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
617 imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
618 imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
619 )...,
620 )
621
622 // Now send just the ones that exist. We won't get the vanished messages.
623 tc.transactf("ok", "Close")
624 tc.transactf("ok", "Select inbox (Qresync (1 1 1,2,5:6))")
625 tc.xuntagged(
626 makeUntagged(
627 imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
628 imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
629 imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
630 )...,
631 )
632
633 // We'll only get updates for UIDs we specify.
634 tc.transactf("ok", "Close")
635 tc.transactf("ok", "Select inbox (Qresync (1 1 5))")
636 tc.xuntagged(
637 makeUntagged(
638 imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
639 )...,
640 )
641
642 // We'll only get updates for UIDs we specify. ../rfc/7162:1523
643 tc.transactf("ok", "Close")
644 tc.transactf("ok", "Select inbox (Qresync (1 1 3))")
645 tc.xuntagged(
646 makeUntagged(
647 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3")},
648 )...,
649 )
650
651 // If we specify the latest modseq, we'll get no changes.
652 tc.transactf("ok", "Close")
653 tc.transactf("ok", "Select inbox (Qresync (1 %d))", clientModseq)
654 tc.xuntagged(baseUntagged...)
655
656 // We can provide our own seqs & uids, and have server determine which uids we
657 // know. But the seqs & uids must be of equal length. First try with a few combinations
658 // that aren't valid. ../rfc/7162:1579
659 tc.transactf("ok", "Close")
660 tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1 1,2)))") // Not same length.
661 tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1,2 1)))") // Not same length.
662 tc.transactf("no", "Select inbox (Qresync (1 1 1:6 (1,2 1,1)))") // Not ascending.
663 tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1:* 1:4)))") // Star not allowed.
664
665 // With valid parameters, based on what a client would know at this stage.
666 tc.transactf("ok", "Select inbox (Qresync (1 1 1:6 (1,3,6 1,3,6)))")
667 tc.xuntagged(
668 makeUntagged(
669 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
670 imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
671 imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
672 imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
673 )...,
674 )
675
676 // The 3rd parameter is optional, try without.
677 tc.transactf("ok", "Close")
678 tc.transactf("ok", "Select inbox (Qresync (1 5 (1,3,6 1,3,6)))")
679 tc.xuntagged(
680 makeUntagged(
681 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
682 imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
683 imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
684 )...,
685 )
686
687 tc.transactf("ok", "Close")
688 tc.transactf("ok", "Select inbox (Qresync (1 8 (1,3,6 1,3,6)))")
689 tc.xuntagged(
690 makeUntagged(
691 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
692 imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
693 )...,
694 )
695
696 // Client will claim a highestmodseq but then include uids that have been removed
697 // since that time. Server detects this, sends full vanished history and continues
698 // working with modseq changed to 1 before the expunged uid.
699 tc.transactf("ok", "Close")
700 tc.transactf("ok", "Select inbox (Qresync (1 9 (1,3,6 1,3,6)))")
701 tc.xuntagged(
702 makeUntagged(
703 imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "ALERT", More: "Synchronization inconsistency in client detected. Client tried to sync with a UID that was removed at or after the MODSEQ it sent in the request. Sending all historic message removals for selected mailbox. Full synchronization recommended."}},
704 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
705 imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
706 )...,
707 )
708
709 // Client will claim a highestmodseq but then include uids that have been removed
710 // since that time. Server detects this, sends full vanished history and continues
711 // working with modseq changed to 1 before the expunged uid.
712 tc.transactf("ok", "Close")
713 tc.transactf("ok", "Select inbox (Qresync (1 18 (1,3,6 1,3,6)))")
714 tc.xuntagged(
715 makeUntagged(
716 imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "ALERT", More: "Synchronization inconsistency in client detected. Client tried to sync with a UID that was removed at or after the MODSEQ it sent in the request. Sending all historic message removals for selected mailbox. Full synchronization recommended."}},
717 imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
718 imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
719 )...,
720 )
721}
722