1package message
2
3import (
4 "bytes"
5 "errors"
6 "io"
7 "log"
8 "os"
9 "path/filepath"
10 "reflect"
11 "strings"
12 "testing"
13
14 "github.com/mjl-/mox/mlog"
15)
16
17var pkglog = mlog.New("message", nil)
18
19func tcheck(t *testing.T, err error, msg string) {
20 t.Helper()
21 if err != nil {
22 t.Fatalf("%s: %s", msg, err)
23 }
24}
25
26func tcompare(t *testing.T, got, exp any) {
27 t.Helper()
28 if !reflect.DeepEqual(got, exp) {
29 t.Fatalf("got %q, expected %q", got, exp)
30 }
31}
32
33func tfail(t *testing.T, err, expErr error) {
34 t.Helper()
35 if (err == nil) != (expErr == nil) || expErr != nil && !errors.Is(err, expErr) {
36 t.Fatalf("got err %v, expected %v", err, expErr)
37 }
38}
39
40func TestEmptyHeader(t *testing.T) {
41 s := "\r\nx"
42 p, err := EnsurePart(pkglog.Logger, true, strings.NewReader(s), int64(len(s)))
43 tcheck(t, err, "parse empty headers")
44 buf, err := io.ReadAll(p.Reader())
45 tcheck(t, err, "read")
46 expBody := "x"
47 tcompare(t, string(buf), expBody)
48 tcompare(t, p.MediaType, "")
49 tcompare(t, p.MediaSubType, "")
50}
51
52func TestBadContentType(t *testing.T) {
53 expBody := "test"
54
55 // Pedantic is like strict.
56 Pedantic = true
57 s := "content-type: text/html;;\r\n\r\ntest"
58 p, err := EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
59 tfail(t, err, ErrBadContentType)
60 buf, err := io.ReadAll(p.Reader())
61 tcheck(t, err, "read")
62 tcompare(t, string(buf), expBody)
63 tcompare(t, p.MediaType, "APPLICATION")
64 tcompare(t, p.MediaSubType, "OCTET-STREAM")
65 Pedantic = false
66
67 // Strict
68 s = "content-type: text/html;;\r\n\r\ntest"
69 p, err = EnsurePart(pkglog.Logger, true, strings.NewReader(s), int64(len(s)))
70 tfail(t, err, ErrBadContentType)
71 buf, err = io.ReadAll(p.Reader())
72 tcheck(t, err, "read")
73 tcompare(t, string(buf), expBody)
74 tcompare(t, p.MediaType, "APPLICATION")
75 tcompare(t, p.MediaSubType, "OCTET-STREAM")
76
77 // Non-strict but unrecoverable content-type.
78 s = "content-type: not a content type;;\r\n\r\ntest"
79 p, err = EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
80 tcheck(t, err, "parsing message with bad but recoverable content-type")
81 buf, err = io.ReadAll(p.Reader())
82 tcheck(t, err, "read")
83 tcompare(t, string(buf), expBody)
84 tcompare(t, p.MediaType, "APPLICATION")
85 tcompare(t, p.MediaSubType, "OCTET-STREAM")
86
87 // We try to use only the content-type, typically better than application/octet-stream.
88 s = "content-type: text/html;;\r\n\r\ntest"
89 p, err = EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
90 tcheck(t, err, "parsing message with bad but recoverable content-type")
91 buf, err = io.ReadAll(p.Reader())
92 tcheck(t, err, "read")
93 tcompare(t, string(buf), expBody)
94 tcompare(t, p.MediaType, "TEXT")
95 tcompare(t, p.MediaSubType, "HTML")
96
97 // Not recovering multipart, we won't have a boundary.
98 s = "content-type: multipart/mixed;;\r\n\r\ntest"
99 p, err = EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
100 tcheck(t, err, "parsing message with bad but recoverable content-type")
101 buf, err = io.ReadAll(p.Reader())
102 tcheck(t, err, "read")
103 tcompare(t, string(buf), expBody)
104 tcompare(t, p.MediaType, "APPLICATION")
105 tcompare(t, p.MediaSubType, "OCTET-STREAM")
106}
107
108func TestBareCR(t *testing.T) {
109 s := "content-type: text/html\r\n\r\nbare\rcr\r\n"
110 expBody := "bare\rcr\r\n"
111
112 // Pedantic is like strict.
113 Pedantic = true
114 p, err := EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
115 tfail(t, err, errBareCR)
116 _, err = io.ReadAll(p.Reader())
117 tfail(t, err, errBareCR)
118 Pedantic = false
119
120 // Strict.
121 p, err = EnsurePart(pkglog.Logger, true, strings.NewReader(s), int64(len(s)))
122 tfail(t, err, errBareCR)
123 _, err = io.ReadAll(p.Reader())
124 tcheck(t, err, "read fallback part without error")
125
126 // Non-strict allows bare cr.
127 p, err = EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
128 tcheck(t, err, "parse")
129 buf, err := io.ReadAll(p.Reader())
130 tcheck(t, err, "read")
131 tcompare(t, string(buf), expBody)
132}
133
134var basicMsg = strings.ReplaceAll(`From: <mjl@mox.example>
135Content-Type: text/plain
136Content-Transfer-Encoding: base64
137
138aGkK
139`, "\n", "\r\n")
140
141func TestBasic(t *testing.T) {
142 r := strings.NewReader(basicMsg)
143 p, err := Parse(pkglog.Logger, true, r)
144 tcheck(t, err, "new reader")
145
146 buf, err := io.ReadAll(p.RawReader())
147 tcheck(t, err, "read raw")
148 expBody := "aGkK\r\n"
149 tcompare(t, string(buf), expBody)
150
151 buf, err = io.ReadAll(p.Reader())
152 tcheck(t, err, "read decoded")
153 tcompare(t, string(buf), "hi\r\n")
154
155 if p.RawLineCount != 1 {
156 t.Fatalf("basic message, got %d lines, expected 1", p.RawLineCount)
157 }
158 if size := p.EndOffset - p.BodyOffset; size != int64(len(expBody)) {
159 t.Fatalf("basic message, got size %d, expected %d", size, len(expBody))
160 }
161}
162
163// From ../rfc/3501:2589
164var basicMsg2 = strings.ReplaceAll(`Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST)
165From: Fred Foobar <foobar@Blurdybloop.example>
166Subject: afternoon meeting
167To: mooch@owatagu.siam.edu.example
168Message-Id: <B27397-0100000@Blurdybloop.example>
169MIME-Version: 1.0
170Content-Type: TEXT/PLAIN; CHARSET=US-ASCII
171
172Hello Joe, do you think we can meet at 3:30 tomorrow?
173
174`, "\n", "\r\n")
175
176func TestBasic2(t *testing.T) {
177 r := strings.NewReader(basicMsg2)
178 p, err := Parse(pkglog.Logger, true, r)
179 tcheck(t, err, "new reader")
180
181 buf, err := io.ReadAll(p.RawReader())
182 tcheck(t, err, "read raw")
183 expBody := "Hello Joe, do you think we can meet at 3:30 tomorrow?\r\n\r\n"
184 tcompare(t, string(buf), expBody)
185
186 buf, err = io.ReadAll(p.Reader())
187 tcheck(t, err, "read decoded")
188 tcompare(t, string(buf), expBody)
189
190 if p.RawLineCount != 2 {
191 t.Fatalf("basic message, got %d lines, expected 2", p.RawLineCount)
192 }
193 if size := p.EndOffset - p.BodyOffset; size != int64(len(expBody)) {
194 t.Fatalf("basic message, got size %d, expected %d", size, len(expBody))
195 }
196
197 r = strings.NewReader(basicMsg2)
198 p, err = Parse(pkglog.Logger, true, r)
199 tcheck(t, err, "new reader")
200 err = p.Walk(pkglog.Logger, nil)
201 tcheck(t, err, "walk")
202 if p.RawLineCount != 2 {
203 t.Fatalf("basic message, got %d lines, expected 2", p.RawLineCount)
204 }
205 if size := p.EndOffset - p.BodyOffset; size != int64(len(expBody)) {
206 t.Fatalf("basic message, got size %d, expected %d", size, len(expBody))
207 }
208}
209
210var mimeMsg = strings.ReplaceAll(`From: Nathaniel Borenstein <nsb@bellcore.com>
211To: Ned Freed <ned@innosoft.com>
212Date: Sun, 21 Mar 1993 23:56:48 -0800 (PST)
213Subject: Sample message
214MIME-Version: 1.0
215Content-type: multipart/mixed; boundary="simple boundary"
216
217This is the preamble. It is to be ignored, though it
218is a handy place for composition agents to include an
219explanatory note to non-MIME conformant readers.
220
221--simple boundary
222
223This is implicitly typed plain US-ASCII text.
224It does NOT end with a linebreak.
225--simple boundary
226Content-type: text/plain; charset=us-ascii
227
228This is explicitly typed plain US-ASCII text.
229It DOES end with a linebreak.
230
231--simple boundary--
232
233This is the epilogue. It is also to be ignored.
234`, "\n", "\r\n")
235
236func TestMime(t *testing.T) {
237 // from ../rfc/2046:1148
238 r := strings.NewReader(mimeMsg)
239 p, err := Parse(pkglog.Logger, true, r)
240 tcheck(t, err, "new reader")
241 if len(p.bound) == 0 {
242 t.Fatalf("got no bound, expected bound for mime message")
243 }
244
245 pp, err := p.ParseNextPart(pkglog.Logger)
246 tcheck(t, err, "next part")
247 buf, err := io.ReadAll(pp.Reader())
248 tcheck(t, err, "read all")
249 tcompare(t, string(buf), "This is implicitly typed plain US-ASCII text.\r\nIt does NOT end with a linebreak.")
250
251 pp, err = p.ParseNextPart(pkglog.Logger)
252 tcheck(t, err, "next part")
253 buf, err = io.ReadAll(pp.Reader())
254 tcheck(t, err, "read all")
255 tcompare(t, string(buf), "This is explicitly typed plain US-ASCII text.\r\nIt DOES end with a linebreak.\r\n")
256
257 _, err = p.ParseNextPart(pkglog.Logger)
258 tcompare(t, err, io.EOF)
259
260 if len(p.Parts) != 2 {
261 t.Fatalf("got %d parts, expected 2", len(p.Parts))
262 }
263 if p.Parts[0].RawLineCount != 2 {
264 t.Fatalf("got %d lines for first part, expected 2", p.Parts[0].RawLineCount)
265 }
266 if p.Parts[1].RawLineCount != 2 {
267 t.Fatalf("got %d lines for second part, expected 2", p.Parts[1].RawLineCount)
268 }
269}
270
271func TestLongLine(t *testing.T) {
272 line := make([]byte, maxLineLength+1)
273 for i := range line {
274 line[i] = 'a'
275 }
276 _, err := Parse(pkglog.Logger, true, bytes.NewReader(line))
277 tfail(t, err, errLineTooLong)
278}
279
280func TestBareCrLf(t *testing.T) {
281 parse := func(strict bool, s string) error {
282 p, err := Parse(pkglog.Logger, strict, strings.NewReader(s))
283 if err != nil {
284 return err
285 }
286 return p.Walk(pkglog.Logger, nil)
287 }
288 err := parse(false, "subject: test\ntest\r\n")
289 tfail(t, err, errBareLF)
290 err = parse(false, "\r\ntest\ntest\r\n")
291 tfail(t, err, errBareLF)
292
293 Pedantic = true
294 err = parse(false, "subject: test\rtest\r\n")
295 tfail(t, err, errBareCR)
296 err = parse(false, "\r\ntest\rtest\r\n")
297 tfail(t, err, errBareCR)
298 Pedantic = false
299
300 err = parse(true, "subject: test\rtest\r\n")
301 tfail(t, err, errBareCR)
302 err = parse(true, "\r\ntest\rtest\r\n")
303 tfail(t, err, errBareCR)
304
305 err = parse(false, "subject: test\rtest\r\n")
306 tcheck(t, err, "header with bare cr")
307 err = parse(false, "\r\ntest\rtest\r\n")
308 tcheck(t, err, "body with bare cr")
309}
310
311func TestMissingClosingBoundary(t *testing.T) {
312 message := strings.ReplaceAll(`Content-Type: multipart/mixed; boundary=x
313
314--x
315
316test
317`, "\n", "\r\n")
318 msg, err := Parse(pkglog.Logger, false, strings.NewReader(message))
319 tcheck(t, err, "new reader")
320 err = walkmsg(&msg)
321 tfail(t, err, errMissingClosingBoundary)
322
323 msg, _ = Parse(pkglog.Logger, false, strings.NewReader(message))
324 err = msg.Walk(pkglog.Logger, nil)
325 tfail(t, err, errMissingClosingBoundary)
326}
327
328func TestHeaderEOF(t *testing.T) {
329 message := "header: test"
330 _, err := Parse(pkglog.Logger, false, strings.NewReader(message))
331 tfail(t, err, errUnexpectedEOF)
332}
333
334func TestBodyEOF(t *testing.T) {
335 message := "header: test\r\n\r\ntest"
336 msg, err := Parse(pkglog.Logger, true, strings.NewReader(message))
337 tcheck(t, err, "new reader")
338 buf, err := io.ReadAll(msg.Reader())
339 tcheck(t, err, "read body")
340 tcompare(t, string(buf), "test")
341}
342
343func TestWalk(t *testing.T) {
344 var message = strings.ReplaceAll(`Content-Type: multipart/related; boundary="----=_NextPart_afb3ad6f146b12b709deac3e387a3ad7"
345
346------=_NextPart_afb3ad6f146b12b709deac3e387a3ad7
347Content-Type: multipart/alternative; boundary="----=_NextPart_afb3ad6f146b12b709deac3e387a3ad7_alt"
348
349------=_NextPart_afb3ad6f146b12b709deac3e387a3ad7_alt
350Content-Type: text/plain; charset="utf-8"
351Content-Transfer-Encoding: 8bit
352
353test
354
355
356------=_NextPart_afb3ad6f146b12b709deac3e387a3ad7_alt
357Content-Type: text/html; charset="utf-8"
358Content-Transfer-Encoding: 8bit
359
360test
361
362------=_NextPart_afb3ad6f146b12b709deac3e387a3ad7_alt--
363------=_NextPart_afb3ad6f146b12b709deac3e387a3ad7--
364
365`, "\n", "\r\n")
366
367 msg, err := Parse(pkglog.Logger, false, strings.NewReader(message))
368 tcheck(t, err, "new reader")
369 enforceSequential = true
370 defer func() {
371 enforceSequential = false
372 }()
373 err = walkmsg(&msg)
374 tcheck(t, err, "walkmsg")
375
376 msg, _ = Parse(pkglog.Logger, false, strings.NewReader(message))
377 err = msg.Walk(pkglog.Logger, nil)
378 tcheck(t, err, "msg.Walk")
379}
380
381func TestNested(t *testing.T) {
382 // From ../rfc/2049:801
383 nestedMessage := strings.ReplaceAll(`MIME-Version: 1.0
384From: Nathaniel Borenstein <nsb@nsb.fv.com>
385To: Ned Freed <ned@innosoft.com>
386Date: Fri, 07 Oct 1994 16:15:05 -0700 (PDT)
387Subject: A multipart example
388Content-Type: multipart/mixed;
389 boundary=unique-boundary-1
390
391This is the preamble area of a multipart message.
392Mail readers that understand multipart format
393should ignore this preamble.
394
395If you are reading this text, you might want to
396consider changing to a mail reader that understands
397how to properly display multipart messages.
398
399--unique-boundary-1
400
401 ... Some text appears here ...
402
403[Note that the blank between the boundary and the start
404 of the text in this part means no header fields were
405 given and this is text in the US-ASCII character set.
406 It could have been done with explicit typing as in the
407 next part.]
408
409--unique-boundary-1
410Content-type: text/plain; charset=US-ASCII
411
412This could have been part of the previous part, but
413illustrates explicit versus implicit typing of body
414parts.
415
416--unique-boundary-1
417Content-Type: multipart/parallel; boundary=unique-boundary-2
418
419--unique-boundary-2
420Content-Type: audio/basic
421Content-Transfer-Encoding: base64
422
423
424--unique-boundary-2
425Content-Type: image/jpeg
426Content-Transfer-Encoding: base64
427
428
429--unique-boundary-2--
430
431--unique-boundary-1
432Content-type: text/enriched
433
434This is <bold><italic>enriched.</italic></bold>
435<smaller>as defined in RFC 1896</smaller>
436
437Isn't it
438<bigger><bigger>cool?</bigger></bigger>
439
440--unique-boundary-1
441Content-Type: message/rfc822
442
443From: (mailbox in US-ASCII)
444To: (address in US-ASCII)
445Subject: (subject in US-ASCII)
446Content-Type: Text/plain; charset=ISO-8859-1
447Content-Transfer-Encoding: Quoted-printable
448
449 ... Additional text in ISO-8859-1 goes here ...
450
451--unique-boundary-1--
452`, "\n", "\r\n")
453
454 msg, err := Parse(pkglog.Logger, true, strings.NewReader(nestedMessage))
455 tcheck(t, err, "new reader")
456 enforceSequential = true
457 defer func() {
458 enforceSequential = false
459 }()
460 err = walkmsg(&msg)
461 tcheck(t, err, "walkmsg")
462
463 if len(msg.Parts) != 5 {
464 t.Fatalf("got %d parts, expected 5", len(msg.Parts))
465 }
466 sub := msg.Parts[4].Message
467 if sub == nil {
468 t.Fatalf("missing part.Message")
469 }
470 buf, err := io.ReadAll(sub.Reader())
471 if err != nil {
472 t.Fatalf("read message body: %v", err)
473 }
474 exp := " ... Additional text in ISO-8859-1 goes here ...\r\n"
475 if string(buf) != exp {
476 t.Fatalf("got %q, expected %q", buf, exp)
477 }
478
479 msg, _ = Parse(pkglog.Logger, false, strings.NewReader(nestedMessage))
480 err = msg.Walk(pkglog.Logger, nil)
481 tcheck(t, err, "msg.Walk")
482
483}
484
485func TestWalkdir(t *testing.T) {
486 // Ensure these dirs exist. Developers should bring their own ham/spam example
487 // emails.
488 os.MkdirAll("../testdata/train/ham", 0770)
489 os.MkdirAll("../testdata/train/spam", 0770)
490
491 var n, nfail int
492 twalkdir(t, "../testdata/train/ham", &n, &nfail)
493 twalkdir(t, "../testdata/train/spam", &n, &nfail)
494 log.Printf("parsing messages: %d/%d failed", nfail, n)
495}
496
497func twalkdir(t *testing.T, dir string, n, nfail *int) {
498 names, err := os.ReadDir(dir)
499 tcheck(t, err, "readdir")
500 if len(names) > 1000 {
501 names = names[:1000]
502 }
503 for _, name := range names {
504 p := filepath.Join(dir, name.Name())
505 *n++
506 err := walk(p)
507 if err != nil {
508 *nfail++
509 log.Printf("%s: %v", p, err)
510 }
511 }
512}
513
514func walk(path string) error {
515 r, err := os.Open(path)
516 if err != nil {
517 return err
518 }
519 defer r.Close()
520 msg, err := Parse(pkglog.Logger, false, r)
521 if err != nil {
522 return err
523 }
524 return walkmsg(&msg)
525}
526
527func walkmsg(msg *Part) error {
528 enforceSequential = true
529 defer func() {
530 enforceSequential = false
531 }()
532
533 if len(msg.bound) == 0 {
534 buf, err := io.ReadAll(msg.Reader())
535 if err != nil {
536 return err
537 }
538
539 if msg.MediaType == "MESSAGE" && (msg.MediaSubType == "RFC822" || msg.MediaSubType == "GLOBAL") {
540 mp, err := Parse(pkglog.Logger, false, bytes.NewReader(buf))
541 if err != nil {
542 return err
543 }
544 msg.Message = &mp
545 walkmsg(msg.Message)
546 }
547
548 size := msg.EndOffset - msg.BodyOffset
549 if size < 0 {
550 log.Printf("msg %v", msg)
551 panic("inconsistent body/end offset")
552 }
553 sr := io.NewSectionReader(msg.r, msg.BodyOffset, size)
554 decsr := msg.bodyReader(sr)
555 buf2, err := io.ReadAll(decsr)
556 if err != nil {
557 return err
558 }
559
560 if !bytes.Equal(buf, buf2) {
561 panic("data mismatch reading sequentially vs via offsets")
562 }
563
564 return nil
565 }
566
567 for {
568 pp, err := msg.ParseNextPart(pkglog.Logger)
569 if err == io.EOF {
570 return nil
571 }
572 if err != nil {
573 return err
574 }
575 if err := walkmsg(pp); err != nil {
576 return err
577 }
578 enforceSequential = true
579 }
580}
581
582func TestEmbedded(t *testing.T) {
583 f, err := os.Open("../testdata/message/message-rfc822-multipart.eml")
584 tcheck(t, err, "open")
585 fi, err := f.Stat()
586 tcheck(t, err, "stat")
587 _, err = EnsurePart(pkglog.Logger, false, f, fi.Size())
588 tcheck(t, err, "parse")
589}
590
591func TestEmbedded2(t *testing.T) {
592 buf, err := os.ReadFile("../testdata/message/message-rfc822-multipart2.eml")
593 tcheck(t, err, "readfile")
594 buf = bytes.ReplaceAll(buf, []byte("\n"), []byte("\r\n"))
595
596 _, err = EnsurePart(pkglog.Logger, false, bytes.NewReader(buf), int64(len(buf)))
597 tfail(t, err, nil)
598}
599