1package imapserver
2
3// todo: if fetch fails part-way through the command, we wouldn't be storing the messages that were parsed. should we try harder to get parsed form of messages stored in db?
4
5import (
6 "bytes"
7 "context"
8 "fmt"
9 "io"
10 "log/slog"
11 "maps"
12 "mime"
13 "net/textproto"
14 "slices"
15 "strings"
16
17 "github.com/mjl-/bstore"
18
19 "github.com/mjl-/mox/message"
20 "github.com/mjl-/mox/mlog"
21 "github.com/mjl-/mox/mox-"
22 "github.com/mjl-/mox/moxio"
23 "github.com/mjl-/mox/store"
24)
25
26// functions to handle fetch attribute requests are defined on fetchCmd.
27type fetchCmd struct {
28 conn *conn
29 isUID bool // If this is a UID FETCH command.
30 rtx *bstore.Tx // Read-only transaction, kept open while processing all messages.
31 updateSeen []store.UID // To mark as seen after processing all messages. UID instead of message ID since moved messages keep their ID and insert a new ID in the original mailbox.
32 hasChangedSince bool // Whether CHANGEDSINCE was set. Enables MODSEQ in response.
33 expungeIssued bool // Set if any message has been expunged. Can happen for expunged messages.
34
35 // For message currently processing.
36 mailboxID int64
37 uid store.UID
38
39 markSeen bool
40 needFlags bool
41 needModseq bool // Whether untagged responses needs modseq.
42 newPreviews map[store.UID]string // Save with messages when done.
43
44 // Loaded when first needed, closed when message was processed.
45 m *store.Message // Message currently being processed.
46 msgr *store.MsgReader
47 part *message.Part
48}
49
50// error when processing an attribute. we typically just don't respond with requested attributes that encounter a failure.
51type attrError struct{ err error }
52
53func (e attrError) Error() string {
54 return e.err.Error()
55}
56
57// raise error processing an attribute.
58func (cmd *fetchCmd) xerrorf(format string, args ...any) {
59 panic(attrError{fmt.Errorf(format, args...)})
60}
61
62func (cmd *fetchCmd) xcheckf(err error, format string, args ...any) {
63 if err != nil {
64 msg := fmt.Sprintf(format, args...)
65 cmd.xerrorf("%s: %w", msg, err)
66 }
67}
68
69// Fetch returns information about messages, be it email envelopes, headers,
70// bodies, full messages, flags.
71//
72// State: Selected
73func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
74 // Command: ../rfc/9051:4330 ../rfc/3501:2992 ../rfc/7162:864
75 // Examples: ../rfc/9051:4463 ../rfc/9051:4520 ../rfc/7162:880
76 // Response syntax: ../rfc/9051:6742 ../rfc/3501:4864 ../rfc/7162:2490
77
78 // Request syntax: ../rfc/9051:6553 ../rfc/3501:4748 ../rfc/4466:535 ../rfc/7162:2475
79 p.xspace()
80 nums := p.xnumSet()
81 p.xspace()
82 atts := p.xfetchAtts()
83 var changedSince int64
84 var haveChangedSince bool
85 var vanished bool
86 if p.space() {
87 // ../rfc/4466:542
88 // ../rfc/7162:2479
89 p.xtake("(")
90 seen := map[string]bool{}
91 for {
92 var w string
93 if isUID && p.conn.enabled[capQresync] {
94 // Vanished only valid for uid fetch, and only for qresync. ../rfc/7162:1693
95 w = p.xtakelist("CHANGEDSINCE", "VANISHED")
96 } else {
97 w = p.xtakelist("CHANGEDSINCE")
98 }
99 if seen[w] {
100 xsyntaxErrorf("duplicate fetch modifier %s", w)
101 }
102 seen[w] = true
103 switch w {
104 case "CHANGEDSINCE":
105 p.xspace()
106 changedSince = p.xnumber64()
107 // workaround: ios mail (16.5.1) was seen sending changedSince 0 on an existing account that got condstore enabled.
108 if changedSince == 0 && mox.Pedantic {
109 // ../rfc/7162:2551
110 xsyntaxErrorf("changedsince modseq must be > 0")
111 }
112 // CHANGEDSINCE is a CONDSTORE-enabling parameter. ../rfc/7162:380
113 p.conn.xensureCondstore(nil)
114 haveChangedSince = true
115 case "VANISHED":
116 vanished = true
117 }
118 if p.take(")") {
119 break
120 }
121 p.xspace()
122 }
123
124 // ../rfc/7162:1701
125 if vanished && !haveChangedSince {
126 xsyntaxErrorf("VANISHED can only be used with CHANGEDSINCE")
127 }
128 }
129 p.xempty()
130
131 // We only keep a wlock, only for initial checks and listing the uids. Then we
132 // unlock and work without a lock. So changes to the store can happen, and we need
133 // to deal with that. If we need to mark messages as seen, we do so after
134 // processing the fetch for all messages, in a single write transaction. We don't
135 // send untagged changes for those \seen flag changes before finishing this
136 // command, because we have to sequence all changes properly, and since we don't
137 // (want to) hold a wlock while processing messages (can be many!), other changes
138 // may have happened to the store. So instead, we'll silently mark messages as seen
139 // (the client should know this is happening anyway!), then broadcast the changes
140 // to everyone, including ourselves. A noop/idle command that may come next will
141 // return the \seen flag changes, in the correct order, with the correct modseq. We
142 // also cannot just apply pending changes while processing. It is not allowed at
143 // all for non-uid-fetch. It would also make life more complicated, e.g. we would
144 // perhaps have to check if newly added messages also match uid fetch set that was
145 // requested.
146
147 var uids []store.UID
148 var vanishedUIDs []store.UID
149
150 cmd := &fetchCmd{conn: c, isUID: isUID, hasChangedSince: haveChangedSince, mailboxID: c.mailboxID, newPreviews: map[store.UID]string{}}
151
152 defer func() {
153 if cmd.rtx == nil {
154 return
155 }
156 err := cmd.rtx.Rollback()
157 c.log.Check(err, "rollback rtx")
158 cmd.rtx = nil
159 }()
160
161 c.account.WithRLock(func() {
162 var err error
163 cmd.rtx, err = c.account.DB.Begin(context.TODO(), false)
164 cmd.xcheckf(err, "begin transaction")
165
166 // Ensure the mailbox still exists.
167 c.xmailboxID(cmd.rtx, c.mailboxID)
168
169 // With changedSince, the client is likely asking for a small set of changes. Use a
170 // database query to trim down the uids we need to look at. We need to go through
171 // the database for "VANISHED (EARLIER)" anyway, to see UIDs that aren't in the
172 // session anymore. Vanished must be used with changedSince. ../rfc/7162:871
173 if changedSince > 0 {
174 q := bstore.QueryTx[store.Message](cmd.rtx)
175 q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
176 q.FilterGreater("ModSeq", store.ModSeqFromClient(changedSince))
177 if !vanished {
178 q.FilterEqual("Expunged", false)
179 }
180 err := q.ForEach(func(m store.Message) error {
181 if m.UID >= c.uidnext {
182 return nil
183 }
184 if isUID {
185 if nums.xcontainsKnownUID(m.UID, c.searchResult, func() store.UID { return c.uidnext - 1 }) {
186 if m.Expunged {
187 vanishedUIDs = append(vanishedUIDs, m.UID)
188 } else {
189 uids = append(uids, m.UID)
190 }
191 }
192 } else {
193 seq := c.sequence(m.UID)
194 if seq > 0 && nums.containsSeq(seq, c.uids, c.searchResult) {
195 uids = append(uids, m.UID)
196 }
197 }
198 return nil
199 })
200 xcheckf(err, "looking up messages with changedsince")
201
202 // In case of vanished where we don't have the full history, we must send VANISHED
203 // for all uids matching nums. ../rfc/7162:1718
204 delModSeq, err := c.account.HighestDeletedModSeq(cmd.rtx)
205 xcheckf(err, "looking up highest deleted modseq")
206 if !vanished || changedSince >= delModSeq.Client() {
207 return
208 }
209
210 // We'll iterate through all UIDs in the numset, and add anything that isn't
211 // already in uids and vanishedUIDs. First sort the uids we already found, for fast
212 // lookup. We'll gather new UIDs in more, so we don't break the binary search.
213 slices.Sort(vanishedUIDs)
214 slices.Sort(uids)
215
216 more := map[store.UID]struct{}{} // We'll add them at the end.
217 checkVanished := func(uid store.UID) {
218 if uid < c.uidnext && uidSearch(uids, uid) <= 0 && uidSearch(vanishedUIDs, uid) <= 0 {
219 more[uid] = struct{}{}
220 }
221 }
222
223 // Now look through the requested uids. We may have a searchResult, handle it
224 // separately from a numset with potential stars, over which we can more easily
225 // iterate.
226 if nums.searchResult {
227 for _, uid := range c.searchResult {
228 checkVanished(uid)
229 }
230 } else {
231 xlastUID := c.newCachedLastUID(cmd.rtx, c.mailboxID, func(xerr error) { xuserErrorf("%s", xerr) })
232 iter := nums.xinterpretStar(xlastUID).newIter()
233 for {
234 num, ok := iter.Next()
235 if !ok {
236 break
237 }
238 checkVanished(store.UID(num))
239 }
240 }
241 vanishedUIDs = slices.AppendSeq(vanishedUIDs, maps.Keys(more))
242 slices.Sort(vanishedUIDs)
243 } else {
244 uids = c.xnumSetEval(cmd.rtx, isUID, nums)
245 }
246
247 })
248 // We are continuing without a lock, working off our snapshot of uids to process.
249
250 // First report all vanished UIDs. ../rfc/7162:1714
251 if len(vanishedUIDs) > 0 {
252 // Mention all vanished UIDs in compact numset form.
253 // ../rfc/7162:1985
254 // No hard limit on response sizes, but clients are recommended to not send more
255 // than 8k. We send a more conservative max 4k.
256 for _, s := range compactUIDSet(vanishedUIDs).Strings(4*1024 - 32) {
257 c.xbwritelinef("* VANISHED (EARLIER) %s", s)
258 }
259 }
260
261 defer cmd.msgclose() // In case of panic.
262
263 for _, cmd.uid = range uids {
264 cmd.conn.log.Debug("processing uid", slog.Any("uid", cmd.uid))
265 data, err := cmd.process(atts)
266 if err != nil {
267 cmd.conn.log.Infox("processing fetch attribute", err, slog.Any("uid", cmd.uid))
268 xuserErrorf("processing fetch attribute: %v", err)
269 }
270
271 // UIDFETCH in case of uidonly. ../rfc/9586:181
272 if c.uidonly {
273 fmt.Fprintf(cmd.conn.xbw, "* %d UIDFETCH ", cmd.uid)
274 } else {
275 fmt.Fprintf(cmd.conn.xbw, "* %d FETCH ", cmd.conn.xsequence(cmd.uid))
276 }
277 data.xwriteTo(cmd.conn, cmd.conn.xbw)
278 cmd.conn.xbw.Write([]byte("\r\n"))
279
280 cmd.msgclose()
281 }
282
283 // We've returned all data. Now we mark messages as seen in one go, in a new write
284 // transaction. We don't send untagged messages for the changes, since there may be
285 // unprocessed pending changes. Instead, we broadcast them to ourselve too, so a
286 // next noop/idle will return the flags to the client.
287
288 err := cmd.rtx.Rollback()
289 c.log.Check(err, "fetch read tx rollback")
290 cmd.rtx = nil
291
292 // ../rfc/9051:4432 We mark all messages that need it as seen at the end of the
293 // command, in a single transaction.
294 if len(cmd.updateSeen) > 0 || len(cmd.newPreviews) > 0 {
295 c.account.WithWLock(func() {
296 changes := make([]store.Change, 0, len(cmd.updateSeen)+1)
297
298 c.xdbwrite(func(wtx *bstore.Tx) {
299 mb, err := store.MailboxID(wtx, c.mailboxID)
300 if err == store.ErrMailboxExpunged {
301 xusercodeErrorf("NONEXISTENT", "mailbox has been expunged")
302 }
303 xcheckf(err, "get mailbox for updating counts after marking as seen")
304
305 var modseq store.ModSeq
306
307 for _, uid := range cmd.updateSeen {
308 m, err := bstore.QueryTx[store.Message](wtx).FilterNonzero(store.Message{MailboxID: c.mailboxID, UID: uid}).Get()
309 xcheckf(err, "get message")
310 if m.Expunged {
311 // Message has been deleted in the mean time.
312 cmd.expungeIssued = true
313 continue
314 }
315 if m.Seen {
316 // Message already marked as seen by another process.
317 continue
318 }
319
320 if modseq == 0 {
321 modseq, err = c.account.NextModSeq(wtx)
322 xcheckf(err, "get next mod seq")
323 }
324
325 oldFlags := m.Flags
326 mb.Sub(m.MailboxCounts())
327 m.Seen = true
328 mb.Add(m.MailboxCounts())
329 changes = append(changes, m.ChangeFlags(oldFlags, mb))
330
331 m.ModSeq = modseq
332 err = wtx.Update(&m)
333 xcheckf(err, "mark message as seen")
334 }
335
336 changes = append(changes, mb.ChangeCounts())
337
338 for uid, s := range cmd.newPreviews {
339 m, err := bstore.QueryTx[store.Message](wtx).FilterNonzero(store.Message{MailboxID: c.mailboxID, UID: uid}).Get()
340 xcheckf(err, "get message")
341 if m.Expunged {
342 // Message has been deleted in the mean time.
343 cmd.expungeIssued = true
344 continue
345 }
346
347 // note: we are not updating modseq.
348
349 m.Preview = &s
350 err = wtx.Update(&m)
351 xcheckf(err, "saving preview with message")
352 }
353
354 if modseq > 0 {
355 mb.ModSeq = modseq
356 err = wtx.Update(&mb)
357 xcheckf(err, "update mailbox with counts and modseq")
358 }
359 })
360
361 // Broadcast these changes also to ourselves, so we'll send the updated flags, but
362 // in the correct order, after other changes.
363 store.BroadcastChanges(c.account, changes)
364 })
365 }
366
367 if cmd.expungeIssued {
368 // ../rfc/2180:343
369 // ../rfc/9051:5102
370 c.xwriteresultf("%s OK [EXPUNGEISSUED] at least one message was expunged", tag)
371 } else {
372 c.ok(tag, cmdstr)
373 }
374}
375
376func (cmd *fetchCmd) xensureMessage() *store.Message {
377 if cmd.m != nil {
378 return cmd.m
379 }
380
381 // We do not filter by Expunged, the message may have been deleted in other
382 // sessions, but not in ours.
383 q := bstore.QueryTx[store.Message](cmd.rtx)
384 q.FilterNonzero(store.Message{MailboxID: cmd.mailboxID, UID: cmd.uid})
385 m, err := q.Get()
386 cmd.xcheckf(err, "get message for uid %d", cmd.uid)
387 cmd.m = &m
388 if m.Expunged {
389 cmd.expungeIssued = true
390 }
391 return cmd.m
392}
393
394func (cmd *fetchCmd) xensureParsed() (*store.MsgReader, *message.Part) {
395 if cmd.msgr != nil {
396 return cmd.msgr, cmd.part
397 }
398
399 m := cmd.xensureMessage()
400
401 cmd.msgr = cmd.conn.account.MessageReader(*m)
402 defer func() {
403 if cmd.part == nil {
404 err := cmd.msgr.Close()
405 cmd.conn.xsanity(err, "closing messagereader")
406 cmd.msgr = nil
407 }
408 }()
409
410 p, err := m.LoadPart(cmd.msgr)
411 xcheckf(err, "load parsed message")
412 cmd.part = &p
413 return cmd.msgr, cmd.part
414}
415
416// msgclose must be called after processing a message (after having written/used
417// its data), even in the case of a panic.
418func (cmd *fetchCmd) msgclose() {
419 cmd.m = nil
420 cmd.part = nil
421 if cmd.msgr != nil {
422 err := cmd.msgr.Close()
423 cmd.conn.xsanity(err, "closing messagereader")
424 cmd.msgr = nil
425 }
426}
427
428func (cmd *fetchCmd) process(atts []fetchAtt) (rdata listspace, rerr error) {
429 defer func() {
430 x := recover()
431 if x == nil {
432 return
433 }
434 err, ok := x.(attrError)
435 if !ok {
436 panic(x)
437 } else if rerr == nil {
438 rerr = err
439 }
440 }()
441
442 var data listspace
443 if !cmd.conn.uidonly {
444 data = append(data, bare("UID"), number(cmd.uid))
445 }
446
447 cmd.markSeen = false
448 cmd.needFlags = false
449 cmd.needModseq = false
450
451 for _, a := range atts {
452 data = append(data, cmd.xprocessAtt(a)...)
453 }
454
455 if cmd.markSeen {
456 cmd.updateSeen = append(cmd.updateSeen, cmd.uid)
457 }
458
459 if cmd.needFlags {
460 m := cmd.xensureMessage()
461 data = append(data, bare("FLAGS"), flaglist(m.Flags, m.Keywords))
462 }
463
464 // The wording around when to include the MODSEQ attribute is hard to follow and is
465 // specified and refined in several places.
466 //
467 // An additional rule applies to "QRESYNC servers" (we'll assume it only applies
468 // when QRESYNC is enabled on a connection): setting the \Seen flag also triggers
469 // sending MODSEQ, and so does a UID FETCH command. ../rfc/7162:1421
470 //
471 // For example, ../rfc/7162:389 says the server must include modseq in "all
472 // subsequent untagged fetch responses", then lists cases, but leaves out FETCH/UID
473 // FETCH. That appears intentional, it is not a list of examples, it is the full
474 // list, and the "all subsequent untagged fetch responses" doesn't mean "all", just
475 // those covering the listed cases. That makes sense, because otherwise all the
476 // other mentioning of cases elsewhere in the RFC would be too superfluous.
477 //
478 // ../rfc/7162:877 ../rfc/7162:388 ../rfc/7162:909 ../rfc/7162:1426
479 if cmd.needModseq || cmd.hasChangedSince || cmd.conn.enabled[capQresync] && cmd.isUID {
480 m := cmd.xensureMessage()
481 data = append(data, bare("MODSEQ"), listspace{bare(fmt.Sprintf("%d", m.ModSeq.Client()))})
482 }
483
484 return data, nil
485}
486
487// result for one attribute. if processing fails, e.g. because data was requested
488// that doesn't exist and cannot be represented in imap, the attribute is simply
489// not returned to the user. in this case, the returned value is a nil list.
490func (cmd *fetchCmd) xprocessAtt(a fetchAtt) []token {
491 switch a.field {
492 case "UID":
493 // Present by default without uidonly. For uidonly, we only add it when explicitly
494 // requested. ../rfc/9586:184
495 if cmd.conn.uidonly {
496 return []token{bare("UID"), number(cmd.uid)}
497 }
498
499 case "ENVELOPE":
500 _, part := cmd.xensureParsed()
501 envelope := xenvelope(part)
502 return []token{bare("ENVELOPE"), envelope}
503
504 case "INTERNALDATE":
505 // ../rfc/9051:6753 ../rfc/9051:6502
506 m := cmd.xensureMessage()
507 return []token{bare("INTERNALDATE"), dquote(m.Received.Format("_2-Jan-2006 15:04:05 -0700"))}
508
509 case "SAVEDATE":
510 m := cmd.xensureMessage()
511 // For messages in storage from before we implemented this extension, we don't have
512 // a savedate, and we return nil. This is normally meant to be per mailbox, but
513 // returning it per message should be fine. ../rfc/8514:191
514 var savedate token = nilt
515 if m.SaveDate != nil {
516 savedate = dquote(m.SaveDate.Format("_2-Jan-2006 15:04:05 -0700"))
517 }
518 return []token{bare("SAVEDATE"), savedate}
519
520 case "BODYSTRUCTURE":
521 _, part := cmd.xensureParsed()
522 bs := xbodystructure(cmd.conn.log, part, true)
523 return []token{bare("BODYSTRUCTURE"), bs}
524
525 case "BODY":
526 respField, t := cmd.xbody(a)
527 if respField == "" {
528 return nil
529 }
530 return []token{bare(respField), t}
531
532 case "BINARY.SIZE":
533 _, p := cmd.xensureParsed()
534 if len(a.sectionBinary) == 0 {
535 // Must return the size of the entire message but with decoded body.
536 // todo: make this less expensive and/or cache the result?
537 n, err := io.Copy(io.Discard, cmd.xbinaryMessageReader(p))
538 cmd.xcheckf(err, "reading message as binary for its size")
539 return []token{bare(cmd.sectionRespField(a)), number(uint32(n))}
540 }
541 p = cmd.xpartnumsDeref(a.sectionBinary, p)
542 if len(p.Parts) > 0 || p.Message != nil {
543 // ../rfc/9051:4385
544 cmd.xerrorf("binary only allowed on leaf parts, not multipart/* or message/rfc822 or message/global")
545 }
546 return []token{bare(cmd.sectionRespField(a)), number(p.DecodedSize)}
547
548 case "BINARY":
549 respField, t := cmd.xbinary(a)
550 if respField == "" {
551 return nil
552 }
553 return []token{bare(respField), t}
554
555 case "RFC822.SIZE":
556 m := cmd.xensureMessage()
557 return []token{bare("RFC822.SIZE"), number(m.Size)}
558
559 case "RFC822.HEADER":
560 ba := fetchAtt{
561 field: "BODY",
562 peek: true,
563 section: &sectionSpec{
564 msgtext: &sectionMsgtext{s: "HEADER"},
565 },
566 }
567 respField, t := cmd.xbody(ba)
568 if respField == "" {
569 return nil
570 }
571 return []token{bare(a.field), t}
572
573 case "RFC822":
574 ba := fetchAtt{
575 field: "BODY",
576 section: &sectionSpec{},
577 }
578 respField, t := cmd.xbody(ba)
579 if respField == "" {
580 return nil
581 }
582 return []token{bare(a.field), t}
583
584 case "RFC822.TEXT":
585 ba := fetchAtt{
586 field: "BODY",
587 section: &sectionSpec{
588 msgtext: &sectionMsgtext{s: "TEXT"},
589 },
590 }
591 respField, t := cmd.xbody(ba)
592 if respField == "" {
593 return nil
594 }
595 return []token{bare(a.field), t}
596
597 case "FLAGS":
598 cmd.needFlags = true
599
600 case "MODSEQ":
601 cmd.needModseq = true
602
603 case "PREVIEW":
604 m := cmd.xensureMessage()
605 preview := m.Preview
606 // We ignore "lazy", generating the preview is fast enough.
607 if preview == nil {
608 // Get the preview. We'll save all generated previews in a single transaction at
609 // the end.
610 _, p := cmd.xensureParsed()
611 s, err := p.Preview(cmd.conn.log)
612 cmd.xcheckf(err, "generating preview")
613 preview = &s
614 cmd.newPreviews[m.UID] = s
615 }
616 var t token = nilt
617 if preview != nil {
618 s := *preview
619
620 // Limit to 200 characters (not bytes). ../rfc/8970:206
621 var n, o int
622 for o = range s {
623 n++
624 if n > 200 {
625 s = s[:o]
626 break
627 }
628 }
629 s = strings.TrimSpace(s)
630 t = string0(s)
631 }
632 return []token{bare(a.field), t}
633
634 default:
635 xserverErrorf("field %q not yet implemented", a.field)
636 }
637 return nil
638}
639
640// ../rfc/9051:6522
641func xenvelope(p *message.Part) token {
642 var env message.Envelope
643 if p.Envelope != nil {
644 env = *p.Envelope
645 }
646 var date token = nilt
647 if !env.Date.IsZero() {
648 // ../rfc/5322:791
649 date = string0(env.Date.Format("Mon, 2 Jan 2006 15:04:05 -0700"))
650 }
651 var subject token = nilt
652 if env.Subject != "" {
653 subject = string0(env.Subject)
654 }
655 var inReplyTo token = nilt
656 if env.InReplyTo != "" {
657 inReplyTo = string0(env.InReplyTo)
658 }
659 var messageID token = nilt
660 if env.MessageID != "" {
661 messageID = string0(env.MessageID)
662 }
663
664 addresses := func(l []message.Address) token {
665 if len(l) == 0 {
666 return nilt
667 }
668 r := listspace{}
669 for _, a := range l {
670 var name token = nilt
671 if a.Name != "" {
672 name = string0(a.Name)
673 }
674 user := string0(a.User)
675 var host token = nilt
676 if a.Host != "" {
677 host = string0(a.Host)
678 }
679 r = append(r, listspace{name, nilt, user, host})
680 }
681 return r
682 }
683
684 // Empty sender or reply-to result in fall-back to from. ../rfc/9051:6140
685 sender := env.Sender
686 if len(sender) == 0 {
687 sender = env.From
688 }
689 replyTo := env.ReplyTo
690 if len(replyTo) == 0 {
691 replyTo = env.From
692 }
693
694 return listspace{
695 date,
696 subject,
697 addresses(env.From),
698 addresses(sender),
699 addresses(replyTo),
700 addresses(env.To),
701 addresses(env.CC),
702 addresses(env.BCC),
703 inReplyTo,
704 messageID,
705 }
706}
707
708func (cmd *fetchCmd) peekOrSeen(peek bool) {
709 if cmd.conn.readonly || peek {
710 return
711 }
712 m := cmd.xensureMessage()
713 if !m.Seen {
714 cmd.markSeen = true
715 cmd.needFlags = true
716 }
717}
718
719// reader that returns the message, but with header Content-Transfer-Encoding left out.
720func (cmd *fetchCmd) xbinaryMessageReader(p *message.Part) io.Reader {
721 hr := cmd.xmodifiedHeader(p, []string{"Content-Transfer-Encoding"}, true)
722 return io.MultiReader(hr, p.Reader())
723}
724
725// return header with only fields, or with everything except fields if "not" is set.
726func (cmd *fetchCmd) xmodifiedHeader(p *message.Part, fields []string, not bool) io.Reader {
727 h, err := io.ReadAll(p.HeaderReader())
728 cmd.xcheckf(err, "reading header")
729
730 matchesFields := func(line []byte) bool {
731 k := bytes.TrimRight(bytes.SplitN(line, []byte(":"), 2)[0], " \t")
732 for _, f := range fields {
733 if bytes.EqualFold(k, []byte(f)) {
734 return true
735 }
736 }
737 return false
738 }
739
740 var match bool
741 hb := &bytes.Buffer{}
742 for len(h) > 0 {
743 line := h
744 i := bytes.Index(line, []byte("\r\n"))
745 if i >= 0 {
746 line = line[:i+2]
747 }
748 h = h[len(line):]
749
750 match = matchesFields(line) || match && (bytes.HasPrefix(line, []byte(" ")) || bytes.HasPrefix(line, []byte("\t")))
751 if match != not || len(line) == 2 {
752 hb.Write(line)
753 }
754 }
755 return hb
756}
757
758func (cmd *fetchCmd) xbinary(a fetchAtt) (string, token) {
759 _, part := cmd.xensureParsed()
760
761 cmd.peekOrSeen(a.peek)
762 if len(a.sectionBinary) == 0 {
763 r := cmd.xbinaryMessageReader(part)
764 if a.partial != nil {
765 r = cmd.xpartialReader(a.partial, r)
766 }
767 return cmd.sectionRespField(a), readerSyncliteral{r}
768 }
769
770 p := part
771 if len(a.sectionBinary) > 0 {
772 p = cmd.xpartnumsDeref(a.sectionBinary, p)
773 }
774 if len(p.Parts) != 0 || p.Message != nil {
775 // ../rfc/9051:4385
776 cmd.xerrorf("binary only allowed on leaf parts, not multipart/* or message/rfc822 or message/global")
777 }
778
779 var cte string
780 if p.ContentTransferEncoding != nil {
781 cte = *p.ContentTransferEncoding
782 }
783 switch cte {
784 case "", "7BIT", "8BIT", "BINARY", "BASE64", "QUOTED-PRINTABLE":
785 default:
786 // ../rfc/9051:5913
787 xusercodeErrorf("UNKNOWN-CTE", "unknown Content-Transfer-Encoding %q", cte)
788 }
789
790 r := p.Reader()
791 if a.partial != nil {
792 r = cmd.xpartialReader(a.partial, r)
793 }
794 return cmd.sectionRespField(a), readerSyncliteral{r}
795}
796
797func (cmd *fetchCmd) xpartialReader(partial *partial, r io.Reader) io.Reader {
798 n, err := io.Copy(io.Discard, io.LimitReader(r, int64(partial.offset)))
799 cmd.xcheckf(err, "skipping to offset for partial")
800 if n != int64(partial.offset) {
801 return strings.NewReader("") // ../rfc/3501:3143 ../rfc/9051:4418
802 }
803 return io.LimitReader(r, int64(partial.count))
804}
805
806func (cmd *fetchCmd) xbody(a fetchAtt) (string, token) {
807 msgr, part := cmd.xensureParsed()
808
809 if a.section == nil {
810 // Non-extensible form of BODYSTRUCTURE.
811 return a.field, xbodystructure(cmd.conn.log, part, false)
812 }
813
814 cmd.peekOrSeen(a.peek)
815
816 respField := cmd.sectionRespField(a)
817
818 if a.section.msgtext == nil && a.section.part == nil {
819 m := cmd.xensureMessage()
820 var offset int64
821 count := m.Size
822 if a.partial != nil {
823 offset = min(int64(a.partial.offset), m.Size)
824 count = int64(a.partial.count)
825 if offset+count > m.Size {
826 count = m.Size - offset
827 }
828 }
829 return respField, readerSizeSyncliteral{&moxio.AtReader{R: msgr, Offset: offset}, count, false}
830 }
831
832 sr := cmd.xsection(a.section, part)
833
834 if a.partial != nil {
835 n, err := io.Copy(io.Discard, io.LimitReader(sr, int64(a.partial.offset)))
836 cmd.xcheckf(err, "skipping to offset for partial")
837 if n != int64(a.partial.offset) {
838 return respField, syncliteral("") // ../rfc/3501:3143 ../rfc/9051:4418
839 }
840 return respField, readerSyncliteral{io.LimitReader(sr, int64(a.partial.count))}
841 }
842 return respField, readerSyncliteral{sr}
843}
844
845func (cmd *fetchCmd) xpartnumsDeref(nums []uint32, p *message.Part) *message.Part {
846 // ../rfc/9051:4481
847 if (len(p.Parts) == 0 && p.Message == nil) && len(nums) == 1 && nums[0] == 1 {
848 return p
849 }
850
851 // ../rfc/9051:4485
852 for i, num := range nums {
853 index := int(num - 1)
854 if p.Message != nil {
855 err := p.SetMessageReaderAt()
856 cmd.xcheckf(err, "preparing submessage")
857 return cmd.xpartnumsDeref(nums[i:], p.Message)
858 }
859 if index < 0 || index >= len(p.Parts) {
860 cmd.xerrorf("requested part does not exist")
861 }
862 p = &p.Parts[index]
863 }
864 return p
865}
866
867func (cmd *fetchCmd) xsection(section *sectionSpec, p *message.Part) io.Reader {
868 // msgtext is not nil, i.e. HEADER* or TEXT (not MIME), for the top-level part (a message).
869 if section.part == nil {
870 return cmd.xsectionMsgtext(section.msgtext, p)
871 }
872
873 p = cmd.xpartnumsDeref(section.part.part, p)
874
875 // If there is no sectionMsgText, then this isn't for HEADER*, TEXT or MIME, i.e. a
876 // part body, e.g. "BODY[1]".
877 if section.part.text == nil {
878 return p.RawReader()
879 }
880
881 // MIME is defined for all parts. Otherwise it's HEADER* or TEXT, which is only
882 // defined for parts that are messages. ../rfc/9051:4500 ../rfc/9051:4517
883 if !section.part.text.mime {
884 if p.Message == nil {
885 cmd.xerrorf("part is not a message, cannot request header* or text")
886 }
887
888 err := p.SetMessageReaderAt()
889 cmd.xcheckf(err, "preparing submessage")
890 p = p.Message
891
892 return cmd.xsectionMsgtext(section.part.text.msgtext, p)
893 }
894
895 // MIME header, see ../rfc/9051:4514 ../rfc/2045:1652
896 h, err := io.ReadAll(p.HeaderReader())
897 cmd.xcheckf(err, "reading header")
898
899 matchesFields := func(line []byte) bool {
900 k := textproto.CanonicalMIMEHeaderKey(string(bytes.TrimRight(bytes.SplitN(line, []byte(":"), 2)[0], " \t")))
901 return strings.HasPrefix(k, "Content-")
902 }
903
904 var match bool
905 hb := &bytes.Buffer{}
906 for len(h) > 0 {
907 line := h
908 i := bytes.Index(line, []byte("\r\n"))
909 if i >= 0 {
910 line = line[:i+2]
911 }
912 h = h[len(line):]
913
914 match = matchesFields(line) || match && (bytes.HasPrefix(line, []byte(" ")) || bytes.HasPrefix(line, []byte("\t")))
915 if match {
916 hb.Write(line)
917 }
918 }
919 return hb
920}
921
922func (cmd *fetchCmd) xsectionMsgtext(smt *sectionMsgtext, p *message.Part) io.Reader {
923 switch smt.s {
924 case "HEADER":
925 return p.HeaderReader()
926
927 case "HEADER.FIELDS":
928 return cmd.xmodifiedHeader(p, smt.headers, false)
929
930 case "HEADER.FIELDS.NOT":
931 return cmd.xmodifiedHeader(p, smt.headers, true)
932
933 case "TEXT":
934 // TEXT the body (excluding headers) of a message, either the top-level message, or
935 // a nested as message/rfc822 or message/global. ../rfc/9051:4517
936 return p.RawReader()
937 }
938 panic(serverError{fmt.Errorf("missing case")})
939}
940
941func (cmd *fetchCmd) sectionRespField(a fetchAtt) string {
942 s := a.field + "["
943 if len(a.sectionBinary) > 0 {
944 s += fmt.Sprintf("%d", a.sectionBinary[0])
945 for _, v := range a.sectionBinary[1:] {
946 s += "." + fmt.Sprintf("%d", v)
947 }
948 } else if a.section != nil {
949 if a.section.part != nil {
950 p := a.section.part
951 s += fmt.Sprintf("%d", p.part[0])
952 for _, v := range p.part[1:] {
953 s += "." + fmt.Sprintf("%d", v)
954 }
955 if p.text != nil {
956 if p.text.mime {
957 s += ".MIME"
958 } else {
959 s += "." + cmd.sectionMsgtextName(p.text.msgtext)
960 }
961 }
962 } else if a.section.msgtext != nil {
963 s += cmd.sectionMsgtextName(a.section.msgtext)
964 }
965 }
966 s += "]"
967 // binary does not have partial in field, unlike BODY ../rfc/9051:6757
968 if a.field != "BINARY" && a.partial != nil {
969 s += fmt.Sprintf("<%d>", a.partial.offset)
970 }
971 return s
972}
973
974func (cmd *fetchCmd) sectionMsgtextName(smt *sectionMsgtext) string {
975 s := smt.s
976 if strings.HasPrefix(smt.s, "HEADER.FIELDS") {
977 l := listspace{}
978 for _, h := range smt.headers {
979 l = append(l, astring(h))
980 }
981 s += " " + l.pack(cmd.conn)
982 }
983 return s
984}
985
986func bodyFldParams(p *message.Part) token {
987 if len(p.ContentTypeParams) == 0 {
988 return nilt
989 }
990 params := make(listspace, 0, 2*len(p.ContentTypeParams))
991 // Ensure same ordering, easier for testing.
992 for _, k := range slices.Sorted(maps.Keys(p.ContentTypeParams)) {
993 v := p.ContentTypeParams[k]
994 params = append(params, string0(strings.ToUpper(k)), string0(v))
995 }
996 return params
997}
998
999func bodyFldEnc(cte *string) token {
1000 var s string
1001 if cte != nil {
1002 s = *cte
1003 }
1004 up := strings.ToUpper(s)
1005 switch up {
1006 case "7BIT", "8BIT", "BINARY", "BASE64", "QUOTED-PRINTABLE":
1007 return dquote(up)
1008 }
1009 return string0(s)
1010}
1011
1012func bodyFldMd5(p *message.Part) token {
1013 if p.ContentMD5 == nil {
1014 return nilt
1015 }
1016 return string0(*p.ContentMD5)
1017}
1018
1019func bodyFldDisp(log mlog.Log, p *message.Part) token {
1020 if p.ContentDisposition == nil {
1021 return nilt
1022 }
1023
1024 // ../rfc/9051:5989
1025 // mime.ParseMediaType recombines parameter value continuations like "title*0" and
1026 // "title*1" into "title". ../rfc/2231:147
1027 // And decodes character sets and removes language tags, like
1028 // "title*0*=us-ascii'en'hello%20world. ../rfc/2231:210
1029
1030 disp, params, err := mime.ParseMediaType(*p.ContentDisposition)
1031 if err != nil {
1032 log.Debugx("parsing content-disposition, ignoring", err, slog.String("header", *p.ContentDisposition))
1033 return nilt
1034 } else if len(params) == 0 {
1035 log.Debug("content-disposition has no parameters, ignoring", slog.String("header", *p.ContentDisposition))
1036 return nilt
1037 }
1038 var fields listspace
1039 for _, k := range slices.Sorted(maps.Keys(params)) {
1040 fields = append(fields, string0(k), string0(params[k]))
1041 }
1042 return listspace{string0(disp), fields}
1043}
1044
1045func bodyFldLang(p *message.Part) token {
1046 // todo: ../rfc/3282:86 ../rfc/5646:218 we currently just split on comma and trim space, should properly parse header.
1047 if p.ContentLanguage == nil {
1048 return nilt
1049 }
1050 var l listspace
1051 for _, s := range strings.Split(*p.ContentLanguage, ",") {
1052 s = strings.TrimSpace(s)
1053 if s == "" {
1054 return string0(*p.ContentLanguage)
1055 }
1056 l = append(l, string0(s))
1057 }
1058 return l
1059}
1060
1061func bodyFldLoc(p *message.Part) token {
1062 if p.ContentLocation == nil {
1063 return nilt
1064 }
1065 return string0(*p.ContentLocation)
1066}
1067
1068// xbodystructure returns a "body".
1069// calls itself for multipart messages and message/{rfc822,global}.
1070func xbodystructure(log mlog.Log, p *message.Part, extensible bool) token {
1071 if p.MediaType == "MULTIPART" {
1072 // Multipart, ../rfc/9051:6355 ../rfc/9051:6411
1073 var bodies concat
1074 for i := range p.Parts {
1075 bodies = append(bodies, xbodystructure(log, &p.Parts[i], extensible))
1076 }
1077 r := listspace{bodies, string0(p.MediaSubType)}
1078 // ../rfc/9051:6371
1079 if extensible {
1080 r = append(r,
1081 bodyFldParams(p),
1082 bodyFldDisp(log, p),
1083 bodyFldLang(p),
1084 bodyFldLoc(p),
1085 )
1086 }
1087 return r
1088 }
1089
1090 // ../rfc/9051:6355
1091 var r listspace
1092 if p.MediaType == "TEXT" {
1093 // ../rfc/9051:6404 ../rfc/9051:6418
1094 r = listspace{
1095 dquote("TEXT"), string0(p.MediaSubType), // ../rfc/9051:6739
1096 // ../rfc/9051:6376
1097 bodyFldParams(p), // ../rfc/9051:6401
1098 nilOrString(p.ContentID),
1099 nilOrString(p.ContentDescription),
1100 bodyFldEnc(p.ContentTransferEncoding),
1101 number(p.EndOffset - p.BodyOffset),
1102 number(p.RawLineCount),
1103 }
1104 } else if p.MediaType == "MESSAGE" && (p.MediaSubType == "RFC822" || p.MediaSubType == "GLOBAL") {
1105 // ../rfc/9051:6415
1106 // note: we don't have to prepare p.Message for reading, because we aren't going to read from it.
1107 r = listspace{
1108 dquote("MESSAGE"), dquote(p.MediaSubType), // ../rfc/9051:6732
1109 // ../rfc/9051:6376
1110 bodyFldParams(p), // ../rfc/9051:6401
1111 nilOrString(p.ContentID),
1112 nilOrString(p.ContentDescription),
1113 bodyFldEnc(p.ContentTransferEncoding),
1114 number(p.EndOffset - p.BodyOffset),
1115 xenvelope(p.Message),
1116 xbodystructure(log, p.Message, extensible),
1117 number(p.RawLineCount), // todo: or mp.RawLineCount?
1118 }
1119 } else {
1120 var media token
1121 switch p.MediaType {
1122 case "APPLICATION", "AUDIO", "IMAGE", "FONT", "MESSAGE", "MODEL", "VIDEO":
1123 media = dquote(p.MediaType)
1124 default:
1125 media = string0(p.MediaType)
1126 }
1127 // ../rfc/9051:6404 ../rfc/9051:6407
1128 r = listspace{
1129 media, string0(p.MediaSubType), // ../rfc/9051:6723
1130 // ../rfc/9051:6376
1131 bodyFldParams(p), // ../rfc/9051:6401
1132 nilOrString(p.ContentID),
1133 nilOrString(p.ContentDescription),
1134 bodyFldEnc(p.ContentTransferEncoding),
1135 number(p.EndOffset - p.BodyOffset),
1136 }
1137 }
1138 if extensible {
1139 // ../rfc/9051:6366
1140 r = append(r,
1141 bodyFldMd5(p),
1142 bodyFldDisp(log, p),
1143 bodyFldLang(p),
1144 bodyFldLoc(p),
1145 )
1146 }
1147 return r
1148}
1149