1package imapserver

2

3// todo: if fetch fails part-way through the command, we wouldn't be storing the messages that were parsed. should we try harder to get parsed form of messages stored in db?

4

5import (

6 "bytes"

7 "context"

8 "fmt"

9 "io"

10 "log/slog"

11 "maps"

12 "net/textproto"

13 "sort"

14 "strings"

15

16 "github.com/mjl-/bstore"

17

18 "github.com/mjl-/mox/message"

19 "github.com/mjl-/mox/mox-"

20 "github.com/mjl-/mox/moxio"

21 "github.com/mjl-/mox/store"

22 "slices"

23)

24

25// functions to handle fetch attribute requests are defined on fetchCmd.

26type fetchCmd struct {

27 conn *conn

28 isUID bool // If this is a UID FETCH command.

29 rtx *bstore.Tx // Read-only transaction, kept open while processing all messages.

30 updateSeen []store.UID // To mark as seen after processing all messages. UID instead of message ID since moved messages keep their ID and insert a new ID in the original mailbox.

31 hasChangedSince bool // Whether CHANGEDSINCE was set. Enables MODSEQ in response.

32 expungeIssued bool // Set if any message has been expunged. Can happen for expunged messages.

33

34 uid store.UID // UID currently processing.

35 markSeen bool

36 needFlags bool

37 needModseq bool // Whether untagged responses needs modseq.

38 newPreviews map[store.UID]string // Save with messages when done.

39

40 // Loaded when first needed, closed when message was processed.

41 m *store.Message // Message currently being processed.

42 msgr *store.MsgReader

43 part *message.Part

44}

45

46// error when processing an attribute. we typically just don't respond with requested attributes that encounter a failure.

47type attrError struct{ err error }

48

49func (e attrError) Error() string {

50 return e.err.Error()

51}

52

53// raise error processing an attribute.

54func (cmd *fetchCmd) xerrorf(format string, args ...any) {

55 panic(attrError{fmt.Errorf(format, args...)})

56}

57

58func (cmd *fetchCmd) xcheckf(err error, format string, args ...any) {

59 if err != nil {

60 msg := fmt.Sprintf(format, args...)

61 cmd.xerrorf("%s: %w", msg, err)

62 }

63}

64

65// Fetch returns information about messages, be it email envelopes, headers,

66// bodies, full messages, flags.

67//

68// State: Selected

69func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {

70 // Command: ../rfc/9051:4330 ../rfc/3501:2992 ../rfc/7162:864

71 // Examples: ../rfc/9051:4463 ../rfc/9051:4520 ../rfc/7162:880

72 // Response syntax: ../rfc/9051:6742 ../rfc/3501:4864 ../rfc/7162:2490

73

74 // Request syntax: ../rfc/9051:6553 ../rfc/3501:4748 ../rfc/4466:535 ../rfc/7162:2475

75 p.xspace()

76 nums := p.xnumSet()

77 p.xspace()

78 atts := p.xfetchAtts(isUID)

79 var changedSince int64

80 var haveChangedSince bool

81 var vanished bool

82 if p.space() {

83 // ../rfc/4466:542

84 // ../rfc/7162:2479

85 p.xtake("(")

86 seen := map[string]bool{}

87 for {

88 var w string

89 if isUID && p.conn.enabled[capQresync] {

90 // Vanished only valid for uid fetch, and only for qresync. ../rfc/7162:1693

91 w = p.xtakelist("CHANGEDSINCE", "VANISHED")

92 } else {

93 w = p.xtakelist("CHANGEDSINCE")

94 }

95 if seen[w] {

96 xsyntaxErrorf("duplicate fetch modifier %s", w)

97 }

98 seen[w] = true

99 switch w {

100 case "CHANGEDSINCE":

101 p.xspace()

102 changedSince = p.xnumber64()

103 // workaround: ios mail (16.5.1) was seen sending changedSince 0 on an existing account that got condstore enabled.

104 if changedSince == 0 && mox.Pedantic {

105 // ../rfc/7162:2551

106 xsyntaxErrorf("changedsince modseq must be > 0")

107 }

108 // CHANGEDSINCE is a CONDSTORE-enabling parameter. ../rfc/7162:380

109 p.conn.xensureCondstore(nil)

110 haveChangedSince = true

111 case "VANISHED":

112 vanished = true

113 }

114 if p.take(")") {

115 break

116 }

117 p.xspace()

118 }

119

120 // ../rfc/7162:1701

121 if vanished && !haveChangedSince {

122 xsyntaxErrorf("VANISHED can only be used with CHANGEDSINCE")

123 }

124 }

125 p.xempty()

126

127 // We only keep a wlock, only for initial checks and listing the uids. Then we

128 // unlock and work without a lock. So changes to the store can happen, and we need

129 // to deal with that. If we need to mark messages as seen, we do so after

130 // processing the fetch for all messages, in a single write transaction. We don't

131 // send untagged changes for those \seen flag changes before finishing this

132 // command, because we have to sequence all changes properly, and since we don't

133 // (want to) hold a wlock while processing messages (can be many!), other changes

134 // may have happened to the store. So instead, we'll silently mark messages as seen

135 // (the client should know this is happening anyway!), then broadcast the changes

136 // to everyone, including ourselves. A noop/idle command that may come next will

137 // return the \seen flag changes, in the correct order, with the correct modseq. We

138 // also cannot just apply pending changes while processing. It is not allowed at

139 // all for non-uid-fetch. It would also make life more complicated, e.g. we would

140 // perhaps have to check if newly added messages also match uid fetch set that was

141 // requested.

142

143 var uids []store.UID

144 var vanishedUIDs []store.UID

145

146 cmd := &fetchCmd{conn: c, isUID: isUID, hasChangedSince: haveChangedSince, newPreviews: map[store.UID]string{}}

147

148 defer func() {

149 if cmd.rtx == nil {

150 return

151 }

152 err := cmd.rtx.Rollback()

153 c.log.Check(err, "rollback rtx")

154 cmd.rtx = nil

155 }()

156

157 c.account.WithRLock(func() {

158 var err error

159 cmd.rtx, err = c.account.DB.Begin(context.TODO(), false)

160 cmd.xcheckf(err, "begin transaction")

161

162 // Ensure the mailbox still exists.

163 c.xmailboxID(cmd.rtx, c.mailboxID)

164

165 // With changedSince, the client is likely asking for a small set of changes. Use a

166 // database query to trim down the uids we need to look at.

167 // ../rfc/7162:871

168 if changedSince > 0 {

169 q := bstore.QueryTx[store.Message](cmd.rtx)

170 q.FilterNonzero(store.Message{MailboxID: c.mailboxID})

171 q.FilterGreater("ModSeq", store.ModSeqFromClient(changedSince))

172 if !vanished {

173 q.FilterEqual("Expunged", false)

174 }

175 err := q.ForEach(func(m store.Message) error {

176 if m.Expunged {

177 vanishedUIDs = append(vanishedUIDs, m.UID)

178 } else if isUID {

179 if nums.containsUID(m.UID, c.uids, c.searchResult) {

180 uids = append(uids, m.UID)

181 }

182 } else {

183 seq := c.sequence(m.UID)

184 if seq > 0 && nums.containsSeq(seq, c.uids, c.searchResult) {

185 uids = append(uids, m.UID)

186 }

187 }

188 return nil

189 })

190 xcheckf(err, "looking up messages with changedsince")

191 } else {

192 uids = c.xnumSetUIDs(isUID, nums)

193 }

194

195 // Send vanished for all missing requested UIDs. ../rfc/7162:1718

196 if !vanished {

197 return

198 }

199

200 delModSeq, err := c.account.HighestDeletedModSeq(cmd.rtx)

201 xcheckf(err, "looking up highest deleted modseq")

202 if changedSince >= delModSeq.Client() {

203 return

204 }

205

206 // First sort the uids we already found, for fast lookup.

207 slices.Sort(vanishedUIDs)

208

209 // We'll be gathering any more vanished uids in more.

210 more := map[store.UID]struct{}{}

211 checkVanished := func(uid store.UID) {

212 if uidSearch(c.uids, uid) <= 0 && uidSearch(vanishedUIDs, uid) <= 0 {

213 more[uid] = struct{}{}

214 }

215 }

216 // Now look through the requested uids. We may have a searchResult, handle it

217 // separately from a numset with potential stars, over which we can more easily

218 // iterate.

219 if nums.searchResult {

220 for _, uid := range c.searchResult {

221 checkVanished(uid)

222 }

223 } else {

224 iter := nums.interpretStar(c.uids).newIter()

225 for {

226 num, ok := iter.Next()

227 if !ok {

228 break

229 }

230 checkVanished(store.UID(num))

231 }

232 }

233 vanishedUIDs = slices.AppendSeq(vanishedUIDs, maps.Keys(more))

234 })

235 // We are continuing without a lock, working off our snapshot of uids to process.

236

237 // First report all vanished UIDs. ../rfc/7162:1714

238 if len(vanishedUIDs) > 0 {

239 // Mention all vanished UIDs in compact numset form.

240 // ../rfc/7162:1985

241 slices.Sort(vanishedUIDs)

242 // No hard limit on response sizes, but clients are recommended to not send more

243 // than 8k. We send a more conservative max 4k.

244 for _, s := range compactUIDSet(vanishedUIDs).Strings(4*1024 - 32) {

245 c.bwritelinef("* VANISHED (EARLIER) %s", s)

246 }

247 }

248

249 for _, cmd.uid = range uids {

250 cmd.conn.log.Debug("processing uid", slog.Any("uid", cmd.uid))

251 cmd.process(atts)

252 }

253

254 // We've returned all data. Now we mark messages as seen in one go, in a new write

255 // transaction. We don't send untagged messages for the changes, since there may be

256 // unprocessed pending changes. Instead, we broadcast them to ourselve too, so a

257 // next noop/idle will return the flags to the client.

258

259 err := cmd.rtx.Rollback()

260 c.log.Check(err, "fetch read tx rollback")

261 cmd.rtx = nil

262

263 // ../rfc/9051:4432 We mark all messages that need it as seen at the end of the

264 // command, in a single transaction.

265 if len(cmd.updateSeen) > 0 || len(cmd.newPreviews) > 0 {

266 c.account.WithWLock(func() {

267 changes := make([]store.Change, 0, len(cmd.updateSeen)+1)

268

269 c.xdbwrite(func(wtx *bstore.Tx) {

270 mb, err := store.MailboxID(wtx, c.mailboxID)

271 if err == store.ErrMailboxExpunged {

272 xusercodeErrorf("NONEXISTENT", "mailbox has been expunged")

273 }

274 xcheckf(err, "get mailbox for updating counts after marking as seen")

275

276 var modseq store.ModSeq

277

278 for _, uid := range cmd.updateSeen {

279 m, err := bstore.QueryTx[store.Message](wtx).FilterNonzero(store.Message{MailboxID: c.mailboxID, UID: uid}).Get()

280 xcheckf(err, "get message")

281 if m.Expunged {

282 // Message has been deleted in the mean time.

283 cmd.expungeIssued = true

284 continue

285 }

286 if m.Seen {

287 // Message already marked as seen by another process.

288 continue

289 }

290

291 if modseq == 0 {

292 modseq, err = c.account.NextModSeq(wtx)

293 xcheckf(err, "get next mod seq")

294 }

295

296 oldFlags := m.Flags

297 mb.Sub(m.MailboxCounts())

298 m.Seen = true

299 mb.Add(m.MailboxCounts())

300 changes = append(changes, m.ChangeFlags(oldFlags))

301

302 m.ModSeq = modseq

303 err = wtx.Update(&m)

304 xcheckf(err, "mark message as seen")

305 }

306

307 changes = append(changes, mb.ChangeCounts())

308

309 for uid, s := range cmd.newPreviews {

310 m, err := bstore.QueryTx[store.Message](wtx).FilterNonzero(store.Message{MailboxID: c.mailboxID, UID: uid}).Get()

311 xcheckf(err, "get message")

312 if m.Expunged {

313 // Message has been deleted in the mean time.

314 cmd.expungeIssued = true

315 continue

316 }

317

318 // note: we are not updating modseq.

319

320 m.Preview = &s

321 err = wtx.Update(&m)

322 xcheckf(err, "saving preview with message")

323 }

324

325 if modseq > 0 {

326 mb.ModSeq = modseq

327 err = wtx.Update(&mb)

328 xcheckf(err, "update mailbox with counts and modseq")

329 }

330 })

331

332 // Broadcast these changes also to ourselves, so we'll send the updated flags, but

333 // in the correct order, after other changes.

334 store.BroadcastChanges(c.account, changes)

335 })

336 }

337

338 if cmd.expungeIssued {

339 // ../rfc/2180:343

340 // ../rfc/9051:5102

341 c.writeresultf("%s OK [EXPUNGEISSUED] at least one message was expunged", tag)

342 } else {

343 c.ok(tag, cmdstr)

344 }

345}

346

347func (cmd *fetchCmd) xensureMessage() *store.Message {

348 if cmd.m != nil {

349 return cmd.m

350 }

351

352 // We do not filter by Expunged, the message may have been deleted in other

353 // sessions, but not in ours.

354 q := bstore.QueryTx[store.Message](cmd.rtx)

355 q.FilterNonzero(store.Message{MailboxID: cmd.conn.mailboxID, UID: cmd.uid})

356 m, err := q.Get()

357 cmd.xcheckf(err, "get message for uid %d", cmd.uid)

358 cmd.m = &m

359 if m.Expunged {

360 cmd.expungeIssued = true

361 }

362 return cmd.m

363}

364

365func (cmd *fetchCmd) xensureParsed() (*store.MsgReader, *message.Part) {

366 if cmd.msgr != nil {

367 return cmd.msgr, cmd.part

368 }

369

370 m := cmd.xensureMessage()

371

372 cmd.msgr = cmd.conn.account.MessageReader(*m)

373 defer func() {

374 if cmd.part == nil {

375 err := cmd.msgr.Close()

376 cmd.conn.xsanity(err, "closing messagereader")

377 cmd.msgr = nil

378 }

379 }()

380

381 p, err := m.LoadPart(cmd.msgr)

382 xcheckf(err, "load parsed message")

383 cmd.part = &p

384 return cmd.msgr, cmd.part

385}

386

387func (cmd *fetchCmd) process(atts []fetchAtt) {

388 defer func() {

389 cmd.m = nil

390 cmd.part = nil

391 if cmd.msgr != nil {

392 err := cmd.msgr.Close()

393 cmd.conn.xsanity(err, "closing messagereader")

394 cmd.msgr = nil

395 }

396

397 x := recover()

398 if x == nil {

399 return

400 }

401 err, ok := x.(attrError)

402 if !ok {

403 panic(x)

404 }

405 cmd.conn.log.Infox("processing fetch attribute", err, slog.Any("uid", cmd.uid))

406 xuserErrorf("processing fetch attribute: %v", err)

407 }()

408

409 data := listspace{bare("UID"), number(cmd.uid)}

410

411 cmd.markSeen = false

412 cmd.needFlags = false

413 cmd.needModseq = false

414

415 for _, a := range atts {

416 data = append(data, cmd.xprocessAtt(a)...)

417 }

418

419 if cmd.markSeen {

420 cmd.updateSeen = append(cmd.updateSeen, cmd.uid)

421 }

422

423 if cmd.needFlags {

424 m := cmd.xensureMessage()

425 data = append(data, bare("FLAGS"), flaglist(m.Flags, m.Keywords))

426 }

427

428 // The wording around when to include the MODSEQ attribute is hard to follow and is

429 // specified and refined in several places.

430 //

431 // An additional rule applies to "QRESYNC servers" (we'll assume it only applies

432 // when QRESYNC is enabled on a connection): setting the \Seen flag also triggers

433 // sending MODSEQ, and so does a UID FETCH command. ../rfc/7162:1421

434 //

435 // For example, ../rfc/7162:389 says the server must include modseq in "all

436 // subsequent untagged fetch responses", then lists cases, but leaves out FETCH/UID

437 // FETCH. That appears intentional, it is not a list of examples, it is the full

438 // list, and the "all subsequent untagged fetch responses" doesn't mean "all", just

439 // those covering the listed cases. That makes sense, because otherwise all the

440 // other mentioning of cases elsewhere in the RFC would be too superfluous.

441 //

442 // ../rfc/7162:877 ../rfc/7162:388 ../rfc/7162:909 ../rfc/7162:1426

443 if cmd.needModseq || cmd.hasChangedSince || cmd.conn.enabled[capQresync] && cmd.isUID {

444 m := cmd.xensureMessage()

445 data = append(data, bare("MODSEQ"), listspace{bare(fmt.Sprintf("%d", m.ModSeq.Client()))})

446 }

447

448 // Write errors are turned into panics because we write through c.

449 fmt.Fprintf(cmd.conn.xbw, "* %d FETCH ", cmd.conn.xsequence(cmd.uid))

450 data.writeTo(cmd.conn, cmd.conn.xbw)

451 cmd.conn.xbw.Write([]byte("\r\n"))

452}

453

454// result for one attribute. if processing fails, e.g. because data was requested

455// that doesn't exist and cannot be represented in imap, the attribute is simply

456// not returned to the user. in this case, the returned value is a nil list.

457func (cmd *fetchCmd) xprocessAtt(a fetchAtt) []token {

458 switch a.field {

459 case "UID":

460 // Always present.

461 return nil

462

463 case "ENVELOPE":

464 _, part := cmd.xensureParsed()

465 envelope := xenvelope(part)

466 return []token{bare("ENVELOPE"), envelope}

467

468 case "INTERNALDATE":

469 // ../rfc/9051:6753 ../rfc/9051:6502

470 m := cmd.xensureMessage()

471 return []token{bare("INTERNALDATE"), dquote(m.Received.Format("_2-Jan-2006 15:04:05 -0700"))}

472

473 case "SAVEDATE":

474 m := cmd.xensureMessage()

475 // For messages in storage from before we implemented this extension, we don't have

476 // a savedate, and we return nil. This is normally meant to be per mailbox, but

477 // returning it per message should be fine. ../rfc/8514:191

478 var savedate token = nilt

479 if m.SaveDate != nil {

480 savedate = dquote(m.SaveDate.Format("_2-Jan-2006 15:04:05 -0700"))

481 }

482 return []token{bare("SAVEDATE"), savedate}

483

484 case "BODYSTRUCTURE":

485 _, part := cmd.xensureParsed()

486 bs := xbodystructure(part, true)

487 return []token{bare("BODYSTRUCTURE"), bs}

488

489 case "BODY":

490 respField, t := cmd.xbody(a)

491 if respField == "" {

492 return nil

493 }

494 return []token{bare(respField), t}

495

496 case "BINARY.SIZE":

497 _, p := cmd.xensureParsed()

498 if len(a.sectionBinary) == 0 {

499 // Must return the size of the entire message but with decoded body.

500 // todo: make this less expensive and/or cache the result?

501 n, err := io.Copy(io.Discard, cmd.xbinaryMessageReader(p))

502 cmd.xcheckf(err, "reading message as binary for its size")

503 return []token{bare(cmd.sectionRespField(a)), number(uint32(n))}

504 }

505 p = cmd.xpartnumsDeref(a.sectionBinary, p)

506 if len(p.Parts) > 0 || p.Message != nil {

507 // ../rfc/9051:4385

508 cmd.xerrorf("binary only allowed on leaf parts, not multipart/* or message/rfc822 or message/global")

509 }

510 return []token{bare(cmd.sectionRespField(a)), number(p.DecodedSize)}

511

512 case "BINARY":

513 respField, t := cmd.xbinary(a)

514 if respField == "" {

515 return nil

516 }

517 return []token{bare(respField), t}

518

519 case "RFC822.SIZE":

520 m := cmd.xensureMessage()

521 return []token{bare("RFC822.SIZE"), number(m.Size)}

522

523 case "RFC822.HEADER":

524 ba := fetchAtt{

525 field: "BODY",

526 peek: true,

527 section: &sectionSpec{

528 msgtext: &sectionMsgtext{s: "HEADER"},

529 },

530 }

531 respField, t := cmd.xbody(ba)

532 if respField == "" {

533 return nil

534 }

535 return []token{bare(a.field), t}

536

537 case "RFC822":

538 ba := fetchAtt{

539 field: "BODY",

540 section: &sectionSpec{},

541 }

542 respField, t := cmd.xbody(ba)

543 if respField == "" {

544 return nil

545 }

546 return []token{bare(a.field), t}

547

548 case "RFC822.TEXT":

549 ba := fetchAtt{

550 field: "BODY",

551 section: &sectionSpec{

552 msgtext: &sectionMsgtext{s: "TEXT"},

553 },

554 }

555 respField, t := cmd.xbody(ba)

556 if respField == "" {

557 return nil

558 }

559 return []token{bare(a.field), t}

560

561 case "FLAGS":

562 cmd.needFlags = true

563

564 case "MODSEQ":

565 cmd.needModseq = true

566

567 case "PREVIEW":

568 m := cmd.xensureMessage()

569 preview := m.Preview

570 // We ignore "lazy", generating the preview is fast enough.

571 if preview == nil {

572 // Get the preview. We'll save all generated previews in a single transaction at

573 // the end.

574 _, p := cmd.xensureParsed()

575 s, err := p.Preview(cmd.conn.log)

576 cmd.xcheckf(err, "generating preview")

577 preview = &s

578 cmd.newPreviews[m.UID] = s

579 }

580 var t token = nilt

581 if preview != nil {

582 s := *preview

583

584 // Limit to 200 characters (not bytes). ../rfc/8970:206

585 var n, o int

586 for o = range s {

587 n++

588 if n > 200 {

589 s = s[:o]

590 break

591 }

592 }

593 s = strings.TrimSpace(s)

594 t = string0(s)

595 }

596 return []token{bare(a.field), t}

597

598 default:

599 xserverErrorf("field %q not yet implemented", a.field)

600 }

601 return nil

602}

603

604// ../rfc/9051:6522

605func xenvelope(p *message.Part) token {

606 var env message.Envelope

607 if p.Envelope != nil {

608 env = *p.Envelope

609 }

610 var date token = nilt

611 if !env.Date.IsZero() {

612 // ../rfc/5322:791

613 date = string0(env.Date.Format("Mon, 2 Jan 2006 15:04:05 -0700"))

614 }

615 var subject token = nilt

616 if env.Subject != "" {

617 subject = string0(env.Subject)

618 }

619 var inReplyTo token = nilt

620 if env.InReplyTo != "" {

621 inReplyTo = string0(env.InReplyTo)

622 }

623 var messageID token = nilt

624 if env.MessageID != "" {

625 messageID = string0(env.MessageID)

626 }

627

628 addresses := func(l []message.Address) token {

629 if len(l) == 0 {

630 return nilt

631 }

632 r := listspace{}

633 for _, a := range l {

634 var name token = nilt

635 if a.Name != "" {

636 name = string0(a.Name)

637 }

638 user := string0(a.User)

639 var host token = nilt

640 if a.Host != "" {

641 host = string0(a.Host)

642 }

643 r = append(r, listspace{name, nilt, user, host})

644 }

645 return r

646 }

647

648 // Empty sender or reply-to result in fall-back to from. ../rfc/9051:6140

649 sender := env.Sender

650 if len(sender) == 0 {

651 sender = env.From

652 }

653 replyTo := env.ReplyTo

654 if len(replyTo) == 0 {

655 replyTo = env.From

656 }

657

658 return listspace{

659 date,

660 subject,

661 addresses(env.From),

662 addresses(sender),

663 addresses(replyTo),

664 addresses(env.To),

665 addresses(env.CC),

666 addresses(env.BCC),

667 inReplyTo,

668 messageID,

669 }

670}

671

672func (cmd *fetchCmd) peekOrSeen(peek bool) {

673 if cmd.conn.readonly || peek {

674 return

675 }

676 m := cmd.xensureMessage()

677 if !m.Seen {

678 cmd.markSeen = true

679 cmd.needFlags = true

680 }

681}

682

683// reader that returns the message, but with header Content-Transfer-Encoding left out.

684func (cmd *fetchCmd) xbinaryMessageReader(p *message.Part) io.Reader {

685 hr := cmd.xmodifiedHeader(p, []string{"Content-Transfer-Encoding"}, true)

686 return io.MultiReader(hr, p.Reader())

687}

688

689// return header with only fields, or with everything except fields if "not" is set.

690func (cmd *fetchCmd) xmodifiedHeader(p *message.Part, fields []string, not bool) io.Reader {

691 h, err := io.ReadAll(p.HeaderReader())

692 cmd.xcheckf(err, "reading header")

693

694 matchesFields := func(line []byte) bool {

695 k := bytes.TrimRight(bytes.SplitN(line, []byte(":"), 2)[0], " \t")

696 for _, f := range fields {

697 if bytes.EqualFold(k, []byte(f)) {

698 return true

699 }

700 }

701 return false

702 }

703

704 var match bool

705 hb := &bytes.Buffer{}

706 for len(h) > 0 {

707 line := h

708 i := bytes.Index(line, []byte("\r\n"))

709 if i >= 0 {

710 line = line[:i+2]

711 }

712 h = h[len(line):]

713

714 match = matchesFields(line) || match && (bytes.HasPrefix(line, []byte(" ")) || bytes.HasPrefix(line, []byte("\t")))

715 if match != not || len(line) == 2 {

716 hb.Write(line)

717 }

718 }

719 return hb

720}

721

722func (cmd *fetchCmd) xbinary(a fetchAtt) (string, token) {

723 _, part := cmd.xensureParsed()

724

725 cmd.peekOrSeen(a.peek)

726 if len(a.sectionBinary) == 0 {

727 r := cmd.xbinaryMessageReader(part)

728 if a.partial != nil {

729 r = cmd.xpartialReader(a.partial, r)

730 }

731 return cmd.sectionRespField(a), readerSyncliteral{r}

732 }

733

734 p := part

735 if len(a.sectionBinary) > 0 {

736 p = cmd.xpartnumsDeref(a.sectionBinary, p)

737 }

738 if len(p.Parts) != 0 || p.Message != nil {

739 // ../rfc/9051:4385

740 cmd.xerrorf("binary only allowed on leaf parts, not multipart/* or message/rfc822 or message/global")

741 }

742

743 switch p.ContentTransferEncoding {

744 case "", "7BIT", "8BIT", "BINARY", "BASE64", "QUOTED-PRINTABLE":

745 default:

746 // ../rfc/9051:5913

747 xusercodeErrorf("UNKNOWN-CTE", "unknown Content-Transfer-Encoding %q", p.ContentTransferEncoding)

748 }

749

750 r := p.Reader()

751 if a.partial != nil {

752 r = cmd.xpartialReader(a.partial, r)

753 }

754 return cmd.sectionRespField(a), readerSyncliteral{r}

755}

756

757func (cmd *fetchCmd) xpartialReader(partial *partial, r io.Reader) io.Reader {

758 n, err := io.Copy(io.Discard, io.LimitReader(r, int64(partial.offset)))

759 cmd.xcheckf(err, "skipping to offset for partial")

760 if n != int64(partial.offset) {

761 return strings.NewReader("") // ../rfc/3501:3143 ../rfc/9051:4418

762 }

763 return io.LimitReader(r, int64(partial.count))

764}

765

766func (cmd *fetchCmd) xbody(a fetchAtt) (string, token) {

767 msgr, part := cmd.xensureParsed()

768

769 if a.section == nil {

770 // Non-extensible form of BODYSTRUCTURE.

771 return a.field, xbodystructure(part, false)

772 }

773

774 cmd.peekOrSeen(a.peek)

775

776 respField := cmd.sectionRespField(a)

777

778 if a.section.msgtext == nil && a.section.part == nil {

779 m := cmd.xensureMessage()

780 var offset int64

781 count := m.Size

782 if a.partial != nil {

783 offset = min(int64(a.partial.offset), m.Size)

784 count = int64(a.partial.count)

785 if offset+count > m.Size {

786 count = m.Size - offset

787 }

788 }

789 return respField, readerSizeSyncliteral{&moxio.AtReader{R: msgr, Offset: offset}, count, false}

790 }

791

792 sr := cmd.xsection(a.section, part)

793

794 if a.partial != nil {

795 n, err := io.Copy(io.Discard, io.LimitReader(sr, int64(a.partial.offset)))

796 cmd.xcheckf(err, "skipping to offset for partial")

797 if n != int64(a.partial.offset) {

798 return respField, syncliteral("") // ../rfc/3501:3143 ../rfc/9051:4418

799 }

800 return respField, readerSyncliteral{io.LimitReader(sr, int64(a.partial.count))}

801 }

802 return respField, readerSyncliteral{sr}

803}

804

805func (cmd *fetchCmd) xpartnumsDeref(nums []uint32, p *message.Part) *message.Part {

806 // ../rfc/9051:4481

807 if (len(p.Parts) == 0 && p.Message == nil) && len(nums) == 1 && nums[0] == 1 {

808 return p

809 }

810

811 // ../rfc/9051:4485

812 for i, num := range nums {

813 index := int(num - 1)

814 if p.Message != nil {

815 err := p.SetMessageReaderAt()

816 cmd.xcheckf(err, "preparing submessage")

817 return cmd.xpartnumsDeref(nums[i:], p.Message)

818 }

819 if index < 0 || index >= len(p.Parts) {

820 cmd.xerrorf("requested part does not exist")

821 }

822 p = &p.Parts[index]

823 }

824 return p

825}

826

827func (cmd *fetchCmd) xsection(section *sectionSpec, p *message.Part) io.Reader {

828 if section.part == nil {

829 return cmd.xsectionMsgtext(section.msgtext, p)

830 }

831

832 p = cmd.xpartnumsDeref(section.part.part, p)

833

834 if section.part.text == nil {

835 return p.RawReader()

836 }

837

838 // ../rfc/9051:4535

839 if p.Message != nil {

840 err := p.SetMessageReaderAt()

841 cmd.xcheckf(err, "preparing submessage")

842 p = p.Message

843 }

844

845 if !section.part.text.mime {

846 return cmd.xsectionMsgtext(section.part.text.msgtext, p)

847 }

848

849 // MIME header, see ../rfc/9051:4534 ../rfc/2045:1645

850 h, err := io.ReadAll(p.HeaderReader())

851 cmd.xcheckf(err, "reading header")

852

853 matchesFields := func(line []byte) bool {

854 k := textproto.CanonicalMIMEHeaderKey(string(bytes.TrimRight(bytes.SplitN(line, []byte(":"), 2)[0], " \t")))

855 // Only add MIME-Version and additional CRLF for messages, not other parts. ../rfc/2045:1645 ../rfc/2045:1652

856 return (p.Envelope != nil && k == "Mime-Version") || strings.HasPrefix(k, "Content-")

857 }

858

859 var match bool

860 hb := &bytes.Buffer{}

861 for len(h) > 0 {

862 line := h

863 i := bytes.Index(line, []byte("\r\n"))

864 if i >= 0 {

865 line = line[:i+2]

866 }

867 h = h[len(line):]

868

869 match = matchesFields(line) || match && (bytes.HasPrefix(line, []byte(" ")) || bytes.HasPrefix(line, []byte("\t")))

870 if match || len(line) == 2 {

871 hb.Write(line)

872 }

873 }

874 return hb

875}

876

877func (cmd *fetchCmd) xsectionMsgtext(smt *sectionMsgtext, p *message.Part) io.Reader {

878 if smt.s == "HEADER" {

879 return p.HeaderReader()

880 }

881

882 switch smt.s {

883 case "HEADER.FIELDS":

884 return cmd.xmodifiedHeader(p, smt.headers, false)

885

886 case "HEADER.FIELDS.NOT":

887 return cmd.xmodifiedHeader(p, smt.headers, true)

888

889 case "TEXT":

890 // It appears imap clients expect to get the body of the message, not a "text body"

891 // which sounds like it means a text/* part of a message. ../rfc/9051:4517

892 return p.RawReader()

893 }

894 panic(serverError{fmt.Errorf("missing case")})

895}

896

897func (cmd *fetchCmd) sectionRespField(a fetchAtt) string {

898 s := a.field + "["

899 if len(a.sectionBinary) > 0 {

900 s += fmt.Sprintf("%d", a.sectionBinary[0])

901 for _, v := range a.sectionBinary[1:] {

902 s += "." + fmt.Sprintf("%d", v)

903 }

904 } else if a.section != nil {

905 if a.section.part != nil {

906 p := a.section.part

907 s += fmt.Sprintf("%d", p.part[0])

908 for _, v := range p.part[1:] {

909 s += "." + fmt.Sprintf("%d", v)

910 }

911 if p.text != nil {

912 if p.text.mime {

913 s += ".MIME"

914 } else {

915 s += "." + cmd.sectionMsgtextName(p.text.msgtext)

916 }

917 }

918 } else if a.section.msgtext != nil {

919 s += cmd.sectionMsgtextName(a.section.msgtext)

920 }

921 }

922 s += "]"

923 // binary does not have partial in field, unlike BODY ../rfc/9051:6757

924 if a.field != "BINARY" && a.partial != nil {

925 s += fmt.Sprintf("<%d>", a.partial.offset)

926 }

927 return s

928}

929

930func (cmd *fetchCmd) sectionMsgtextName(smt *sectionMsgtext) string {

931 s := smt.s

932 if strings.HasPrefix(smt.s, "HEADER.FIELDS") {

933 l := listspace{}

934 for _, h := range smt.headers {

935 l = append(l, astring(h))

936 }

937 s += " " + l.pack(cmd.conn)

938 }

939 return s

940}

941

942func bodyFldParams(params map[string]string) token {

943 if len(params) == 0 {

944 return nilt

945 }

946 // Ensure same ordering, easier for testing.

947 var keys []string

948 for k := range params {

949 keys = append(keys, k)

950 }

951 sort.Strings(keys)

952 l := make(listspace, 2*len(keys))

953 i := 0

954 for _, k := range keys {

955 l[i] = string0(strings.ToUpper(k))

956 l[i+1] = string0(params[k])

957 i += 2

958 }

959 return l

960}

961

962func bodyFldEnc(s string) token {

963 up := strings.ToUpper(s)

964 switch up {

965 case "7BIT", "8BIT", "BINARY", "BASE64", "QUOTED-PRINTABLE":

966 return dquote(up)

967 }

968 return string0(s)

969}

970

971// xbodystructure returns a "body".

972// calls itself for multipart messages and message/{rfc822,global}.

973func xbodystructure(p *message.Part, extensible bool) token {

974 if p.MediaType == "MULTIPART" {

975 // Multipart, ../rfc/9051:6355 ../rfc/9051:6411

976 var bodies concat

977 for i := range p.Parts {

978 bodies = append(bodies, xbodystructure(&p.Parts[i], extensible))

979 }

980 r := listspace{bodies, string0(p.MediaSubType)}

981 if extensible {

982 if len(p.ContentTypeParams) == 0 {

983 r = append(r, nilt)

984 } else {

985 params := make(listspace, 0, 2*len(p.ContentTypeParams))

986 for k, v := range p.ContentTypeParams {

987 params = append(params, string0(k), string0(v))

988 }

989 r = append(r, params)

990 }

991 }

992 return r

993 }

994

995 // ../rfc/9051:6355

996 var r listspace

997 if p.MediaType == "TEXT" {

998 // ../rfc/9051:6404 ../rfc/9051:6418

999 r = listspace{

1000 dquote("TEXT"), string0(p.MediaSubType), // ../rfc/9051:6739

1001 // ../rfc/9051:6376

1002 bodyFldParams(p.ContentTypeParams), // ../rfc/9051:6401

1003 nilOrString(p.ContentID),

1004 nilOrString(p.ContentDescription),

1005 bodyFldEnc(p.ContentTransferEncoding),

1006 number(p.EndOffset - p.BodyOffset),

1007 number(p.RawLineCount),

1008 }

1009 } else if p.MediaType == "MESSAGE" && (p.MediaSubType == "RFC822" || p.MediaSubType == "GLOBAL") {

1010 // ../rfc/9051:6415

1011 // note: we don't have to prepare p.Message for reading, because we aren't going to read from it.

1012 r = listspace{

1013 dquote("MESSAGE"), dquote(p.MediaSubType), // ../rfc/9051:6732

1014 // ../rfc/9051:6376

1015 bodyFldParams(p.ContentTypeParams), // ../rfc/9051:6401

1016 nilOrString(p.ContentID),

1017 nilOrString(p.ContentDescription),

1018 bodyFldEnc(p.ContentTransferEncoding),

1019 number(p.EndOffset - p.BodyOffset),

1020 xenvelope(p.Message),

1021 xbodystructure(p.Message, extensible),

1022 number(p.RawLineCount), // todo: or mp.RawLineCount?

1023 }

1024 } else {

1025 var media token

1026 switch p.MediaType {

1027 case "APPLICATION", "AUDIO", "IMAGE", "FONT", "MESSAGE", "MODEL", "VIDEO":

1028 media = dquote(p.MediaType)

1029 default:

1030 media = string0(p.MediaType)

1031 }

1032 // ../rfc/9051:6404 ../rfc/9051:6407

1033 r = listspace{

1034 media, string0(p.MediaSubType), // ../rfc/9051:6723

1035 // ../rfc/9051:6376

1036 bodyFldParams(p.ContentTypeParams), // ../rfc/9051:6401

1037 nilOrString(p.ContentID),

1038 nilOrString(p.ContentDescription),

1039 bodyFldEnc(p.ContentTransferEncoding),

1040 number(p.EndOffset - p.BodyOffset),

1041 }

1042 }

1043 // todo: if "extensible", we could add the value of the "content-md5" header. we don't have it in our parsed data structure, so we don't add it. likely no one would use it, also not any of the other optional fields. ../rfc/9051:6366

1044 return r

1045}

1046