1package message

2

3// todo: allow more invalid content-type values, we now stop parsing on: empty media type (eg "content-type: ; name=..."), empty value for property (eg "charset=", missing quotes for characters that should be quoted (eg boundary containing "=" but without quotes), duplicate properties (two charsets), empty pairs (eg "text/html;;").

4// todo: should we be forgiving when closing boundary in multipart message is missing? seems like spam messages do this...

5// todo: should we allow base64 messages where a line starts with a space? and possibly more whitespace. is happening in messages. coreutils base64 accepts it, encoding/base64 does not.

6// todo: handle comments in headers?

7// todo: should we just always store messages with \n instead of \r\n? \r\n seems easier for use with imap.

8// todo: can use a cleanup

9

10import (

11 "bufio"

12 "bytes"

13 "encoding/base64"

14 "errors"

15 "fmt"

16 "io"

17 "log/slog"

18 "mime"

19 "mime/quotedprintable"

20 "net/mail"

21 "net/textproto"

22 "strings"

23 "time"

24

25 "golang.org/x/text/encoding/ianaindex"

26

27 "github.com/mjl-/mox/mlog"

28 "github.com/mjl-/mox/smtp"

29)

30

31// Pedantic enables stricter parsing.

32var Pedantic bool

33

34var (

35 ErrBadContentType = errors.New("bad content-type")

36)

37

38var (

39 errNotMultipart = errors.New("not a multipart message")

40 errFirstBoundCloses = errors.New("first boundary cannot be finishing boundary")

41 errLineTooLong = errors.New("line too long")

42 errMissingBoundaryParam = errors.New("missing/empty boundary content-type parameter")

43 errMissingClosingBoundary = errors.New("eof without closing boundary")

44 errBareLF = errors.New("invalid bare line feed")

45 errBareCR = errors.New("invalid bare carriage return")

46 errUnexpectedEOF = errors.New("unexpected eof")

47)

48

49// If set, during tests, attempts to reparse a part will cause an error, because sequentially reading parts should not lead to reparsing.

50var enforceSequential bool

51

52// Part represents a whole mail message, or a part of a multipart message. It

53// is designed to handle IMAP requirements efficiently.

54type Part struct {

55 BoundaryOffset int64 // Offset in message where bound starts. -1 for top-level message.

56 HeaderOffset int64 // Offset in message file where header starts.

57 BodyOffset int64 // Offset in message file where body starts.

58 EndOffset int64 // Where body of part ends. Set when part is fully read.

59 RawLineCount int64 // Number of lines in raw, undecoded, body of part. Set when part is fully read.

60 DecodedSize int64 // Number of octets when decoded. If this is a text mediatype, lines ending only in LF are changed end in CRLF and DecodedSize reflects that.

61

62 MediaType string // From Content-Type, upper case. E.g. "TEXT". Can be empty because content-type may be absent. In this case, the part may be treated as TEXT/PLAIN.

63 MediaSubType string // From Content-Type, upper case. E.g. "PLAIN".

64 ContentTypeParams map[string]string // E.g. holds "boundary" for multipart messages. Has lower-case keys, and original case values.

65 ContentID string

66 ContentDescription string

67 ContentTransferEncoding string // In upper case.

68 Envelope *Envelope // Email message headers. Not for non-message parts.

69

70 Parts []Part // Parts if this is a multipart.

71

72 // Only for message/rfc822 and message/global. This part may have a buffer as

73 // backing io.ReaderAt, because a message/global can have a non-identity

74 // content-transfer-encoding. This part has a nil parent.

75 Message *Part

76

77 r io.ReaderAt

78 header textproto.MIMEHeader // Parsed header.

79 nextBoundOffset int64 // If >= 0, the offset where the next part header starts. We can set this when a user fully reads each part.

80 lastBoundOffset int64 // Start of header of last/previous part. Used to skip a part if ParseNextPart is called and nextBoundOffset is -1.

81 parent *Part // Parent part, for getting bound from, and setting nextBoundOffset when a part has finished reading. Only for subparts, not top-level parts.

82 bound []byte // Only set if valid multipart with boundary, includes leading --, excludes \r\n.

83 strict bool // If set, valid crlf line endings are verified when reading body.

84}

85

86// todo: have all Content* fields in Part?

87// todo: make Address contain a type Localpart and dns.Domain?

88// todo: if we ever make a major change and reparse all parts, switch to lower-case values if not too troublesome.

89

90// Envelope holds the basic/common message headers as used in IMAP4.

91type Envelope struct {

92 Date time.Time

93 Subject string // Q/B-word-decoded.

94 From []Address

95 Sender []Address

96 ReplyTo []Address

97 To []Address

98 CC []Address

99 BCC []Address

100 InReplyTo string // From In-Reply-To header, includes <>.

101 MessageID string // From Message-Id header, includes <>.

102}

103

104// Address as used in From and To headers.

105type Address struct {

106 Name string // Free-form name for display in mail applications.

107 User string // Localpart, encoded as string. Must be parsed before using as Localpart.

108 Host string // Domain in ASCII.

109}

110

111// Parse reads the headers of the mail message and returns a part.

112// A part provides access to decoded and raw contents of a message and its multiple parts.

113//

114// If strict is set, fewer attempts are made to continue parsing when errors are

115// encountered, such as with invalid content-type headers or bare carriage returns.

116func Parse(elog *slog.Logger, strict bool, r io.ReaderAt) (Part, error) {

117 log := mlog.New("message", elog)

118 return newPart(log, strict, r, 0, nil)

119}

120

121// EnsurePart parses a part as with Parse, but ensures a usable part is always

122// returned, even if error is non-nil. If a parse error occurs, the message is

123// returned as application/octet-stream, and headers can still be read if they

124// were valid.

125//

126// If strict is set, fewer attempts are made to continue parsing when errors are

127// encountered, such as with invalid content-type headers or bare carriage returns.

128func EnsurePart(elog *slog.Logger, strict bool, r io.ReaderAt, size int64) (Part, error) {

129 log := mlog.New("message", elog)

130 p, err := Parse(log.Logger, strict, r)

131 if err == nil {

132 err = p.Walk(log.Logger, nil)

133 }

134 if err != nil {

135 np, err2 := fallbackPart(p, r, size)

136 if err2 != nil {

137 err = err2

138 }

139 p = np

140 }

141 return p, err

142}

143

144func fallbackPart(p Part, r io.ReaderAt, size int64) (Part, error) {

145 np := Part{

146 HeaderOffset: p.HeaderOffset,

147 BodyOffset: p.BodyOffset,

148 EndOffset: size,

149 MediaType: "APPLICATION",

150 MediaSubType: "OCTET-STREAM",

151 ContentTypeParams: p.ContentTypeParams,

152 ContentID: p.ContentID,

153 ContentDescription: p.ContentDescription,

154 ContentTransferEncoding: p.ContentTransferEncoding,

155 Envelope: p.Envelope,

156 // We don't keep:

157 // - BoundaryOffset: irrelevant for top-level message.

158 // - RawLineCount and DecodedSize: set below.

159 // - Parts: we are not treating this as a multipart message.

160 }

161 np.SetReaderAt(r)

162 // By reading body, the number of lines and decoded size will be set.

163 _, err := io.Copy(io.Discard, np.Reader())

164 return np, err

165}

166

167// SetReaderAt sets r as reader for this part and all its sub parts, recursively.

168// No reader is set for any Message subpart, see SetMessageReaderAt.

169func (p *Part) SetReaderAt(r io.ReaderAt) {

170 if r == nil {

171 panic("nil reader")

172 }

173 p.r = r

174 for i := range p.Parts {

175 pp := &p.Parts[i]

176 pp.SetReaderAt(r)

177 }

178}

179

180// SetMessageReaderAt sets a reader on p.Message, which must be non-nil.

181func (p *Part) SetMessageReaderAt() error {

182 // todo: if p.Message does not contain any non-identity content-transfer-encoding, we should set an offsetReader of p.Message, recursively.

183 buf, err := io.ReadAll(p.Reader())

184 if err != nil {

185 return err

186 }

187 p.Message.SetReaderAt(bytes.NewReader(buf))

188 return nil

189}

190

191// Walk through message, decoding along the way, and collecting mime part offsets and sizes, and line counts.

192func (p *Part) Walk(elog *slog.Logger, parent *Part) error {

193 log := mlog.New("message", elog)

194

195 if len(p.bound) == 0 {

196 if p.MediaType == "MESSAGE" && (p.MediaSubType == "RFC822" || p.MediaSubType == "GLOBAL") {

197 // todo: don't read whole submessage in memory...

198 buf, err := io.ReadAll(p.Reader())

199 if err != nil {

200 return err

201 }

202 br := bytes.NewReader(buf)

203 mp, err := Parse(log.Logger, p.strict, br)

204 if err != nil {

205 return fmt.Errorf("parsing embedded message: %w", err)

206 }

207 if err := mp.Walk(log.Logger, nil); err != nil {

208 // If this is a DSN and we are not in pedantic mode, accept unexpected end of

209 // message. This is quite common because MTA's sometimes just truncate the original

210 // message in a place that makes the message invalid.

211 if errors.Is(err, errUnexpectedEOF) && !Pedantic && parent != nil && len(parent.Parts) >= 3 && p == &parent.Parts[2] && parent.MediaType == "MULTIPART" && parent.MediaSubType == "REPORT" {

212 mp, err = fallbackPart(mp, br, int64(len(buf)))

213 if err != nil {

214 return fmt.Errorf("parsing invalid embedded message: %w", err)

215 }

216 } else {

217 return fmt.Errorf("parsing parts of embedded message: %w", err)

218 }

219 }

220 // todo: if mp does not contain any non-identity content-transfer-encoding, we should set an offsetReader of p.r on mp, recursively.

221 p.Message = &mp

222 return nil

223 }

224 _, err := io.Copy(io.Discard, p.Reader())

225 return err

226 }

227

228 for {

229 pp, err := p.ParseNextPart(log.Logger)

230 if err == io.EOF {

231 return nil

232 }

233 if err != nil {

234 return err

235 }

236 if err := pp.Walk(log.Logger, p); err != nil {

237 return err

238 }

239 }

240}

241

242// String returns a debugging representation of the part.

243func (p *Part) String() string {

244 return fmt.Sprintf("&Part{%s/%s offsets %d/%d/%d/%d lines %d decodedsize %d next %d last %d bound %q parts %v}", p.MediaType, p.MediaSubType, p.BoundaryOffset, p.HeaderOffset, p.BodyOffset, p.EndOffset, p.RawLineCount, p.DecodedSize, p.nextBoundOffset, p.lastBoundOffset, p.bound, p.Parts)

245}

246

247// newPart parses a new part, which can be the top-level message.

248// offset is the bound offset for parts, and the start of message for top-level messages. parent indicates if this is a top-level message or sub-part.

249// If an error occurs, p's exported values can still be relevant. EnsurePart uses these values.

250func newPart(log mlog.Log, strict bool, r io.ReaderAt, offset int64, parent *Part) (p Part, rerr error) {

251 if r == nil {

252 panic("nil reader")

253 }

254 p = Part{

255 BoundaryOffset: -1,

256 EndOffset: -1,

257 r: r,

258 parent: parent,

259 strict: strict,

260 }

261

262 b := &bufAt{strict: strict, r: r, offset: offset}

263

264 if parent != nil {

265 p.BoundaryOffset = offset

266 if line, _, err := b.ReadLine(true); err != nil {

267 return p, err

268 } else if match, finish := checkBound(line, parent.bound); !match {

269 return p, fmt.Errorf("missing bound")

270 } else if finish {

271 return p, fmt.Errorf("new part for closing boundary")

272 }

273 }

274

275 // Collect header.

276 p.HeaderOffset = b.offset

277 p.BodyOffset = b.offset

278 hb := &bytes.Buffer{}

279 for {

280 line, _, err := b.ReadLine(true)

281 if err == io.EOF {

282 // No body is valid.

283 break

284 }

285 if err != nil {

286 return p, fmt.Errorf("reading header line: %w", err)

287 }

288 hb.Write(line)

289 if len(line) == 2 {

290 break // crlf

291 }

292 }

293 p.BodyOffset = b.offset

294

295 // Don't attempt to parse empty header, mail.ReadMessage doesn't like it.

296 if p.HeaderOffset == p.BodyOffset {

297 p.header = textproto.MIMEHeader{}

298 } else {

299 h, err := parseHeader(hb)

300 if err != nil {

301 return p, fmt.Errorf("parsing header: %w", err)

302 }

303 p.header = h

304 }

305

306 ct := p.header.Get("Content-Type")

307 mt, params, err := mime.ParseMediaType(ct)

308 if err != nil && ct != "" {

309 if Pedantic || strict {

310 return p, fmt.Errorf("%w: %s: %q", ErrBadContentType, err, ct)

311 }

312

313 // Try parsing just a content-type, ignoring parameters.

314 // ../rfc/2045:628

315 ct = strings.TrimSpace(strings.SplitN(ct, ";", 2)[0])

316 t := strings.SplitN(ct, "/", 2)

317 isToken := func(s string) bool {

318 const separators = `()<>@,;:\\"/[]?= ` // ../rfc/2045:663

319 for _, c := range s {

320 if c < 0x20 || c >= 0x80 || strings.ContainsRune(separators, c) {

321 return false

322 }

323 }

324 return len(s) > 0

325 }

326 // We cannot recover content-type of multipart, we won't have a boundary.

327 if len(t) == 2 && isToken(t[0]) && !strings.EqualFold(t[0], "multipart") && isToken(t[1]) {

328 p.MediaType = strings.ToUpper(t[0])

329 p.MediaSubType = strings.ToUpper(t[1])

330 } else {

331 p.MediaType = "APPLICATION"

332 p.MediaSubType = "OCTET-STREAM"

333 }

334 log.Debugx("malformed content-type, attempting to recover and continuing", err,

335 slog.String("contenttype", p.header.Get("Content-Type")),

336 slog.String("mediatype", p.MediaType),

337 slog.String("mediasubtype", p.MediaSubType))

338 } else if mt != "" {

339 t := strings.SplitN(strings.ToUpper(mt), "/", 2)

340 if len(t) != 2 {

341 if Pedantic || strict {

342 return p, fmt.Errorf("bad content-type: %q (content-type %q)", mt, ct)

343 }

344 log.Debug("malformed media-type, ignoring and continuing", slog.String("type", mt))

345 p.MediaType = "APPLICATION"

346 p.MediaSubType = "OCTET-STREAM"

347 } else {

348 p.MediaType = t[0]

349 p.MediaSubType = t[1]

350 p.ContentTypeParams = params

351 }

352 }

353

354 p.ContentID = p.header.Get("Content-Id")

355 p.ContentDescription = p.header.Get("Content-Description")

356 p.ContentTransferEncoding = strings.ToUpper(p.header.Get("Content-Transfer-Encoding"))

357

358 if parent == nil {

359 p.Envelope, err = parseEnvelope(log, mail.Header(p.header))

360 if err != nil {

361 return p, err

362 }

363 }

364

365 if p.MediaType == "MULTIPART" {

366 s := params["boundary"]

367 if s == "" {

368 return p, errMissingBoundaryParam

369 }

370 p.bound = append([]byte("--"), s...)

371

372 // Discard preamble, before first boundary.

373 for {

374 line, _, err := b.PeekLine(true)

375 if err != nil {

376 return p, fmt.Errorf("parsing line for part preamble: %w", err)

377 }

378 // Line only needs boundary prefix, not exact match. ../rfc/2046:1103

379 // Well, for compatibility, we require whitespace after the boundary. Because some

380 // software use the same boundary but with text appended for sub parts.

381 if match, finish := checkBound(line, p.bound); match {

382 if finish {

383 return p, errFirstBoundCloses

384 }

385 break

386 }

387 b.ReadLine(true)

388 }

389 p.nextBoundOffset = b.offset

390 p.lastBoundOffset = b.offset

391 }

392

393 return p, nil

394}

395

396// Header returns the parsed header of this part.

397func (p *Part) Header() (textproto.MIMEHeader, error) {

398 if p.header != nil {

399 return p.header, nil

400 }

401 if p.HeaderOffset == p.BodyOffset {

402 p.header = textproto.MIMEHeader{}

403 return p.header, nil

404 }

405 h, err := parseHeader(p.HeaderReader())

406 p.header = h

407 return h, err

408}

409

410// HeaderReader returns a reader for the header section of this part, including ending bare CRLF.

411func (p *Part) HeaderReader() io.Reader {

412 return io.NewSectionReader(p.r, p.HeaderOffset, p.BodyOffset-p.HeaderOffset)

413}

414

415// parse a header, only call this on non-empty input (even though that is a valid header).

416func parseHeader(r io.Reader) (textproto.MIMEHeader, error) {

417 // We read using mail.ReadMessage instead of textproto.ReadMIMEHeaders because the

418 // first handles email messages properly, while the second only works for HTTP

419 // headers.

420 var zero textproto.MIMEHeader

421

422 // We read the header and add the optional \r\n header/body separator. If the \r\n

423 // is missing, parsing with Go <1.21 results in an EOF error.

424 // todo: directly parse from reader r when Go 1.20 is no longer supported.

425 buf, err := io.ReadAll(r)

426 if err != nil {

427 return zero, err

428 }

429 if bytes.HasSuffix(buf, []byte("\r\n")) && !bytes.HasSuffix(buf, []byte("\r\n\r\n")) {

430 buf = append(buf, "\r\n"...)

431 }

432 msg, err := mail.ReadMessage(bytes.NewReader(buf))

433 if err != nil {

434 return zero, err

435 }

436 return textproto.MIMEHeader(msg.Header), nil

437}

438

439var wordDecoder = mime.WordDecoder{

440 CharsetReader: func(charset string, r io.Reader) (io.Reader, error) {

441 switch strings.ToLower(charset) {

442 case "", "us-ascii", "utf-8":

443 return r, nil

444 }

445 enc, _ := ianaindex.MIME.Encoding(charset)

446 if enc == nil {

447 enc, _ = ianaindex.IANA.Encoding(charset)

448 }

449 if enc == nil {

450 return r, fmt.Errorf("unknown charset %q", charset)

451 }

452 return enc.NewDecoder().Reader(r), nil

453 },

454}

455

456func parseEnvelope(log mlog.Log, h mail.Header) (*Envelope, error) {

457 date, _ := h.Date()

458

459 // We currently marshal this field to JSON. But JSON cannot represent all

460 // time.Time. Time zone of 24:00 was seen in the wild. We won't try for extreme

461 // years, but we can readjust timezones.

462 // todo: remove this once we no longer store using json.

463 _, offset := date.Zone()

464 if date.Year() > 9999 {

465 date = time.Time{}

466 } else if offset <= -24*3600 || offset >= 24*3600 {

467 date = time.Unix(date.Unix(), 0).UTC()

468 }

469

470 subject := h.Get("Subject")

471 if s, err := wordDecoder.DecodeHeader(subject); err == nil {

472 subject = s

473 }

474

475 env := &Envelope{

476 date,

477 subject,

478 parseAddressList(log, h, "from"),

479 parseAddressList(log, h, "sender"),

480 parseAddressList(log, h, "reply-to"),

481 parseAddressList(log, h, "to"),

482 parseAddressList(log, h, "cc"),

483 parseAddressList(log, h, "bcc"),

484 h.Get("In-Reply-To"),

485 h.Get("Message-Id"),

486 }

487 return env, nil

488}

489

490func parseAddressList(log mlog.Log, h mail.Header, k string) []Address {

491 // todo: possibly work around ios mail generating incorrect q-encoded "phrases" with unencoded double quotes? ../rfc/2047:382

492 l, err := h.AddressList(k)

493 if err != nil {

494 return nil

495 }

496 var r []Address

497 for _, a := range l {

498 // todo: parse more fully according to ../rfc/5322:959

499 var user, host string

500 addr, err := smtp.ParseAddress(a.Address)

501 if err != nil {

502 log.Infox("parsing address (continuing)", err, slog.Any("address", a.Address))

503 } else {

504 user = addr.Localpart.String()

505 host = addr.Domain.ASCII

506 }

507 r = append(r, Address{a.Name, user, host})

508 }

509 return r

510}

511

512// ParseNextPart parses the next (sub)part of this multipart message.

513// ParseNextPart returns io.EOF and a nil part when there are no more parts.

514// Only used for initial parsing of message. Once parsed, use p.Parts.

515func (p *Part) ParseNextPart(elog *slog.Logger) (*Part, error) {

516 log := mlog.New("message", elog)

517

518 if len(p.bound) == 0 {

519 return nil, errNotMultipart

520 }

521 if p.nextBoundOffset == -1 {

522 if enforceSequential {

523 panic("access not sequential")

524 }

525 // Set nextBoundOffset by fully reading the last part.

526 last, err := newPart(log, p.strict, p.r, p.lastBoundOffset, p)

527 if err != nil {

528 return nil, err

529 }

530 if _, err := io.Copy(io.Discard, last.RawReader()); err != nil {

531 return nil, err

532 }

533 if p.nextBoundOffset == -1 {

534 return nil, fmt.Errorf("internal error: reading part did not set nextBoundOffset")

535 }

536 }

537 b := &bufAt{strict: p.strict, r: p.r, offset: p.nextBoundOffset}

538 // todo: should we require a crlf on final closing bound? we don't require it because some message/rfc822 don't have a crlf after their closing boundary, so those messages don't end in crlf.

539 line, crlf, err := b.ReadLine(false)

540 if err != nil {

541 return nil, err

542 }

543 if match, finish := checkBound(line, p.bound); !match {

544 return nil, fmt.Errorf("expected bound, got %q", line)

545 } else if finish {

546 // Read any trailing data.

547 if p.parent != nil {

548 for {

549 line, _, err := b.PeekLine(false)

550 if err != nil {

551 break

552 }

553 if match, _ := checkBound(line, p.parent.bound); match {

554 break

555 }

556 b.ReadLine(false)

557 }

558 if p.parent.lastBoundOffset == p.BoundaryOffset {

559 p.parent.nextBoundOffset = b.offset

560 }

561 }

562 p.EndOffset = b.offset

563 return nil, io.EOF

564 } else if !crlf {

565 return nil, fmt.Errorf("non-finishing bound without crlf: %w", errUnexpectedEOF)

566 }

567 boundOffset := p.nextBoundOffset

568 p.lastBoundOffset = boundOffset

569 p.nextBoundOffset = -1

570 np, err := newPart(log, p.strict, p.r, boundOffset, p)

571 if err != nil {

572 return nil, err

573 }

574 p.Parts = append(p.Parts, np)

575 return &p.Parts[len(p.Parts)-1], nil

576}

577

578// IsDSN returns whether the MIME structure of the part is a DSN.

579func (p *Part) IsDSN() bool {

580 return p.MediaType == "MULTIPART" &&

581 p.MediaSubType == "REPORT" &&

582 len(p.Parts) >= 2 &&

583 p.Parts[1].MediaType == "MESSAGE" &&

584 (p.Parts[1].MediaSubType == "DELIVERY-STATUS" || p.Parts[1].MediaSubType == "GLOBAL-DELIVERY-STATUS")

585}

586

587// Reader returns a reader for the decoded body content.

588func (p *Part) Reader() io.Reader {

589 return p.bodyReader(p.RawReader())

590}

591

592// ReaderUTF8OrBinary returns a reader for the decoded body content, transformed to

593// utf-8 for known mime/iana encodings (only if they aren't us-ascii or utf-8

594// already). For unknown or missing character sets/encodings, the original reader

595// is returned.

596func (p *Part) ReaderUTF8OrBinary() io.Reader {

597 return DecodeReader(p.ContentTypeParams["charset"], p.Reader())

598}

599

600func (p *Part) bodyReader(r io.Reader) io.Reader {

601 r = newDecoder(p.ContentTransferEncoding, r)

602 if p.MediaType == "TEXT" {

603 return &textReader{p, bufio.NewReader(r), 0, false}

604 }

605 return &countReader{p, r, 0}

606}

607

608// countReader is an io.Reader that passes Reads to the underlying reader.

609// when eof is read, it sets p.DecodedSize to the number of bytes returned.

610type countReader struct {

611 p *Part

612 r io.Reader

613 count int64

614}

615

616func (cr *countReader) Read(buf []byte) (int, error) {

617 n, err := cr.r.Read(buf)

618 if n >= 0 {

619 cr.count += int64(n)

620 }

621 if err == io.EOF {

622 cr.p.DecodedSize = cr.count

623 }

624 return n, err

625}

626

627// textReader is an io.Reader that ensures all lines return end in CRLF.

628// when eof is read from the underlying reader, it sets p.DecodedSize.

629type textReader struct {

630 p *Part

631 r *bufio.Reader

632 count int64

633 prevcr bool // If previous byte returned was a CR.

634}

635

636func (tr *textReader) Read(buf []byte) (int, error) {

637 o := 0

638 for o < len(buf) {

639 c, err := tr.r.ReadByte()

640 if err != nil {

641 tr.count += int64(o)

642 tr.p.DecodedSize = tr.count

643 return o, err

644 }

645 if c == '\n' && !tr.prevcr {

646 buf[o] = '\r'

647 o++

648 tr.prevcr = true

649 tr.r.UnreadByte()

650 continue

651 }

652 buf[o] = c

653 tr.prevcr = c == '\r'

654 o++

655 }

656 tr.count += int64(o)

657 return o, nil

658}

659

660func newDecoder(cte string, r io.Reader) io.Reader {

661 // ../rfc/2045:775

662 switch cte {

663 case "BASE64":

664 return base64.NewDecoder(base64.StdEncoding, r)

665 case "QUOTED-PRINTABLE":

666 return quotedprintable.NewReader(r)

667 }

668 return r

669}

670

671// RawReader returns a reader for the raw, undecoded body content. E.g. with

672// quoted-printable or base64 content intact.

673// Fully reading a part helps its parent part find its next part efficiently.

674func (p *Part) RawReader() io.Reader {

675 if p.r == nil {

676 panic("missing reader")

677 }

678 if p.EndOffset >= 0 {

679 return &crlfReader{strict: p.strict, r: io.NewSectionReader(p.r, p.BodyOffset, p.EndOffset-p.BodyOffset)}

680 }

681 p.RawLineCount = 0

682 if p.parent == nil {

683 return &offsetReader{p, p.BodyOffset, p.strict, true, false, 0}

684 }

685 return &boundReader{p: p, b: &bufAt{strict: p.strict, r: p.r, offset: p.BodyOffset}, prevlf: true}

686}

687

688// crlfReader verifies there are no bare newlines and optionally no bare carriage returns.

689type crlfReader struct {

690 r io.Reader

691 strict bool

692 prevcr bool

693}

694

695func (r *crlfReader) Read(buf []byte) (int, error) {

696 n, err := r.r.Read(buf)

697 if err == nil || err == io.EOF {

698 for _, b := range buf[:n] {

699 if b == '\n' && !r.prevcr {

700 err = errBareLF

701 break

702 } else if b != '\n' && r.prevcr && (r.strict || Pedantic) {

703 err = errBareCR

704 break

705 }

706 r.prevcr = b == '\r'

707 }

708 }

709 return n, err

710}

711

712// bufAt is a buffered reader on an underlying ReaderAt.

713// bufAt verifies that lines end with crlf.

714type bufAt struct {

715 offset int64 // Offset in r currently consumed, i.e. not including any buffered data.

716

717 strict bool

718 r io.ReaderAt

719 buf []byte // Buffered data.

720 nbuf int // Valid bytes in buf.

721 scratch []byte

722}

723

724// Messages should not have lines longer than 78+2 bytes, and must not have

725// lines longer than 998+2 bytes. But in practice they have longer lines. We

726// have a higher limit, but for when parsing with strict we check for the 1000

727// bytes limit.

728// ../rfc/5321:3512

729const maxLineLength = 8 * 1024

730

731func (b *bufAt) maxLineLength() int {

732 if b.strict || Pedantic {

733 return 1000

734 }

735 return maxLineLength

736}

737

738// ensure makes sure b.nbuf is up to maxLineLength, unless eof is encountered.

739func (b *bufAt) ensure() error {

740 for _, c := range b.buf[:b.nbuf] {

741 if c == '\n' {

742 return nil

743 }

744 }

745 if b.scratch == nil {

746 b.scratch = make([]byte, b.maxLineLength())

747 }

748 if b.buf == nil {

749 b.buf = make([]byte, b.maxLineLength())

750 }

751 for b.nbuf < b.maxLineLength() {

752 n, err := b.r.ReadAt(b.buf[b.nbuf:], b.offset+int64(b.nbuf))

753 if n > 0 {

754 b.nbuf += n

755 }

756 if err != nil && err != io.EOF || err == io.EOF && b.nbuf+n == 0 {

757 return err

758 }

759 if n == 0 || err == io.EOF {

760 break

761 }

762 }

763 return nil

764}

765

766// ReadLine reads a line until \r\n is found, returning the line including \r\n.

767// If not found, or a bare \n is encountered, or a bare \r is enountered in pedantic mode, ReadLine returns an error.

768func (b *bufAt) ReadLine(requirecrlf bool) (buf []byte, crlf bool, err error) {

769 return b.line(true, requirecrlf)

770}

771

772func (b *bufAt) PeekLine(requirecrlf bool) (buf []byte, crlf bool, err error) {

773 return b.line(false, requirecrlf)

774}

775

776func (b *bufAt) line(consume, requirecrlf bool) (buf []byte, crlf bool, err error) {

777 if err := b.ensure(); err != nil {

778 return nil, false, err

779 }

780 for i, c := range b.buf[:b.nbuf] {

781 if c == '\n' {

782 // Should have seen a \r, which should have been handled below.

783 return nil, false, errBareLF

784 }

785 if c != '\r' {

786 continue

787 }

788 i++

789 if i >= b.nbuf || b.buf[i] != '\n' {

790 if b.strict || Pedantic {

791 return nil, false, errBareCR

792 }

793 continue

794 }

795 b.scratch = b.scratch[:i+1]

796 copy(b.scratch, b.buf[:i+1])

797 if consume {

798 copy(b.buf, b.buf[i+1:])

799 b.offset += int64(i + 1)

800 b.nbuf -= i + 1

801 }

802 return b.scratch, true, nil

803 }

804 if b.nbuf >= b.maxLineLength() {

805 return nil, false, errLineTooLong

806 }

807 if requirecrlf {

808 return nil, false, errUnexpectedEOF

809 }

810 b.scratch = b.scratch[:b.nbuf]

811 copy(b.scratch, b.buf[:b.nbuf])

812 if consume {

813 b.offset += int64(b.nbuf)

814 b.nbuf = 0

815 }

816 return b.scratch, false, nil

817}

818

819// PeekByte returns the next unread byte, or an error.

820func (b *bufAt) PeekByte() (byte, error) {

821 if err := b.ensure(); err != nil {

822 return 0, err

823 }

824 if b.nbuf == 0 {

825 return 0, io.EOF

826 }

827 return b.buf[0], nil

828}

829

830// offsetReader reads from p.r starting from offset, and RawLineCount on p.

831// offsetReader validates lines end with \r\n.

832type offsetReader struct {

833 p *Part

834 offset int64

835 strict bool

836 prevlf bool

837 prevcr bool

838 linelength int

839}

840

841func (r *offsetReader) Read(buf []byte) (int, error) {

842 n, err := r.p.r.ReadAt(buf, r.offset)

843 if n > 0 {

844 r.offset += int64(n)

845 max := maxLineLength

846 if r.strict || Pedantic {

847 max = 1000

848 }

849

850 for _, c := range buf[:n] {

851 if r.prevlf {

852 r.p.RawLineCount++

853 }

854 if err == nil || err == io.EOF {

855 if c == '\n' && !r.prevcr {

856 err = errBareLF

857 } else if c != '\n' && r.prevcr && (r.strict || Pedantic) {

858 err = errBareCR

859 }

860 }

861 r.prevlf = c == '\n'

862 r.prevcr = c == '\r'

863 r.linelength++

864 if c == '\n' {

865 r.linelength = 0

866 } else if r.linelength > max && err == nil {

867 err = errLineTooLong

868 }

869 }

870 }

871 if err == io.EOF {

872 r.p.EndOffset = r.offset

873 }

874 return n, err

875}

876

877var crlf = []byte("\r\n")

878

879// boundReader is a reader that stops at a closing multipart boundary.

880// boundReader ensures lines end with crlf through its use of bufAt.

881type boundReader struct {

882 p *Part

883 b *bufAt

884 buf []byte // Data from previous line, to be served first.

885 nbuf int // Number of valid bytes in buf.

886 crlf []byte // Possible crlf, to be returned if we do not yet encounter a boundary.

887 prevlf bool // If last char returned was a newline. For counting lines.

888}

889

890func (b *boundReader) Read(buf []byte) (count int, rerr error) {

891 origBuf := buf

892 defer func() {

893 if count > 0 {

894 for _, c := range origBuf[:count] {

895 if b.prevlf {

896 b.p.RawLineCount++

897 }

898 b.prevlf = c == '\n'

899 }

900 }

901 }()

902

903 for {

904 // Read data from earlier line.

905 if b.nbuf > 0 {

906 n := b.nbuf

907 if n > len(buf) {

908 n = len(buf)

909 }

910 copy(buf, b.buf[:n])

911 copy(b.buf, b.buf[n:])

912 buf = buf[n:]

913 b.nbuf -= n

914 count += n

915 if b.nbuf > 0 {

916 break

917 }

918 }

919

920 // Look at next line. If it is a boundary, we are done and won't serve the crlf from the last line.

921 line, _, err := b.b.PeekLine(false)

922 if match, _ := checkBound(line, b.p.parent.bound); match {

923 b.p.EndOffset = b.b.offset - int64(len(b.crlf))

924 if b.p.parent.lastBoundOffset == b.p.BoundaryOffset {

925 b.p.parent.nextBoundOffset = b.b.offset

926 } else if enforceSequential {

927 panic("access not sequential")

928 }

929 return count, io.EOF

930 }

931 if err == io.EOF {

932 err = errMissingClosingBoundary

933 }

934 if err != nil && err != io.EOF {

935 return count, err

936 }

937 if len(b.crlf) > 0 {

938 n := len(b.crlf)

939 if n > len(buf) {

940 n = len(buf)

941 }

942 copy(buf, b.crlf[:n])

943 count += n

944 buf = buf[n:]

945 b.crlf = b.crlf[n:]

946 }

947 if len(buf) == 0 {

948 break

949 }

950 line, _, err = b.b.ReadLine(true)

951 if err != nil {

952 // Could be an unexpected end of the part.

953 return 0, err

954 }

955 b.crlf = crlf // crlf will be read next time, but not if a boundary follows.

956 n := len(line) - 2

957 line = line[:n]

958 if n > len(buf) {

959 n = len(buf)

960 }

961 copy(buf, line[:n])

962 count += n

963 buf = buf[n:]

964 line = line[n:]

965 if len(line) > 0 {

966 if b.buf == nil {

967 b.buf = make([]byte, b.b.maxLineLength())

968 }

969 copy(b.buf, line)

970 b.nbuf = len(line)

971 }

972 }

973 return count, nil

974}

975

976func checkBound(line, bound []byte) (bool, bool) {

977 if !bytes.HasPrefix(line, bound) {

978 return false, false

979 }

980 line = line[len(bound):]

981 if bytes.HasPrefix(line, []byte("--")) {

982 return true, true

983 }

984 if len(line) == 0 {

985 return true, false

986 }

987 c := line[0]

988 switch c {

989 case ' ', '\t', '\r', '\n':

990 return true, false

991 }

992 return false, false

993}

994