1package webmail
2
3import (
4 "bufio"
5 "fmt"
6 "io"
7 "log/slog"
8 "mime"
9 "net/url"
10 "strings"
11
12 "golang.org/x/text/encoding/ianaindex"
13
14 "github.com/mjl-/mox/dns"
15 "github.com/mjl-/mox/message"
16 "github.com/mjl-/mox/mlog"
17 "github.com/mjl-/mox/mox-"
18 "github.com/mjl-/mox/moxio"
19 "github.com/mjl-/mox/smtp"
20 "github.com/mjl-/mox/store"
21)
22
23// todo: we should have all needed information for messageItem in store.Message (perhaps some data in message.Part) for fast access, not having to parse the on-disk message file.
24
25var wordDecoder = mime.WordDecoder{
26 CharsetReader: func(charset string, r io.Reader) (io.Reader, error) {
27 switch strings.ToLower(charset) {
28 case "", "us-ascii", "utf-8":
29 return r, nil
30 }
31 enc, _ := ianaindex.MIME.Encoding(charset)
32 if enc == nil {
33 enc, _ = ianaindex.IANA.Encoding(charset)
34 }
35 if enc == nil {
36 return r, fmt.Errorf("unknown charset %q", charset)
37 }
38 return enc.NewDecoder().Reader(r), nil
39 },
40}
41
42// Attempt q/b-word-decode name, coming from Content-Type "name" field or
43// Content-Disposition "filename" field.
44//
45// RFC 2231 specify an encoding for non-ascii values in mime header parameters. But
46// it appears common practice to instead just q/b-word encode the values.
47// Thunderbird and gmail.com do this for the Content-Type "name" parameter.
48// gmail.com also does that for the Content-Disposition "filename" parameter, where
49// Thunderbird uses the RFC 2231-defined encoding. Go's mime.ParseMediaType parses
50// the mechanism specified in RFC 2231 only. The value for "name" we get here would
51// already be decoded properly for standards-compliant headers, like
52// "filename*0*=UTF-8”%...; filename*1*=%.... We'll look for Q/B-word encoding
53// markers ("=?"-prefix or "?="-suffix) and try to decode if present. This would
54// only cause trouble for filenames having this prefix/suffix.
55func tryDecodeParam(log mlog.Log, name string) string {
56 if name == "" || !strings.HasPrefix(name, "=?") && !strings.HasSuffix(name, "?=") {
57 return name
58 }
59 // todo: find where this is allowed. it seems quite common. perhaps we should remove the pedantic check?
60 if mox.Pedantic {
61 log.Debug("attachment contains rfc2047 q/b-word-encoded mime parameter instead of rfc2231-encoded", slog.String("name", name))
62 return name
63 }
64 s, err := wordDecoder.DecodeHeader(name)
65 if err != nil {
66 log.Debugx("q/b-word decoding mime parameter", err, slog.String("name", name))
67 return name
68 }
69 return s
70}
71
72// todo: mime.FormatMediaType does not wrap long lines. should do it ourselves, and split header into several parts (if commonly supported).
73
74func messageItem(log mlog.Log, m store.Message, state *msgState) (MessageItem, error) {
75 pm, err := parsedMessage(log, m, state, false, true)
76 if err != nil {
77 return MessageItem{}, fmt.Errorf("parsing message %d for item: %v", m.ID, err)
78 }
79 // Clear largish unused data.
80 m.MsgPrefix = nil
81 m.ParsedBuf = nil
82 return MessageItem{m, pm.envelope, pm.attachments, pm.isSigned, pm.isEncrypted, pm.firstLine, true}, nil
83}
84
85// formatFirstLine returns a line the client can display next to the subject line
86// in a mailbox. It will replace quoted text, and any prefixing "On ... write:"
87// line with "[...]" so only new and useful information will be displayed.
88// Trailing signatures are not included.
89func formatFirstLine(r io.Reader) (string, error) {
90 // We look quite a bit of lines ahead for trailing signatures with trailing empty lines.
91 var lines []string
92 scanner := bufio.NewScanner(r)
93 ensureLines := func() {
94 for len(lines) < 10 && scanner.Scan() {
95 lines = append(lines, strings.TrimSpace(scanner.Text()))
96 }
97 }
98 ensureLines()
99
100 isSnipped := func(s string) bool {
101 return s == "[...]" || s == "[…]" || s == "..."
102 }
103
104 nextLineQuoted := func(i int) bool {
105 if i+1 < len(lines) && lines[i+1] == "" {
106 i++
107 }
108 return i+1 < len(lines) && (strings.HasPrefix(lines[i+1], ">") || isSnipped(lines[i+1]))
109 }
110
111 // Remainder is signature if we see a line with only and minimum 2 dashes, and
112 // there are no more empty lines, and there aren't more than 5 lines left.
113 isSignature := func() bool {
114 if len(lines) == 0 || !strings.HasPrefix(lines[0], "--") || strings.Trim(strings.TrimSpace(lines[0]), "-") != "" {
115 return false
116 }
117 l := lines[1:]
118 for len(l) > 0 && l[len(l)-1] == "" {
119 l = l[:len(l)-1]
120 }
121 if len(l) >= 5 {
122 return false
123 }
124 for _, line := range l {
125 if line == "" {
126 return false
127 }
128 }
129 return true
130 }
131
132 result := ""
133
134 resultSnipped := func() bool {
135 return strings.HasSuffix(result, "[...]\n") || strings.HasSuffix(result, "[…]")
136 }
137
138 // Quick check for initial wrapped "On ... wrote:" line.
139 if len(lines) > 3 && strings.HasPrefix(lines[0], "On ") && !strings.HasSuffix(lines[0], "wrote:") && strings.HasSuffix(lines[1], ":") && nextLineQuoted(1) {
140 result = "[...]\n"
141 lines = lines[3:]
142 ensureLines()
143 }
144
145 for ; len(lines) > 0 && !isSignature(); ensureLines() {
146 line := lines[0]
147 if strings.HasPrefix(line, ">") {
148 if !resultSnipped() {
149 result += "[...]\n"
150 }
151 lines = lines[1:]
152 continue
153 }
154 if line == "" {
155 lines = lines[1:]
156 continue
157 }
158 // Check for a "On <date>, <person> wrote:", we require digits before a quoted
159 // line, with an optional empty line in between. If we don't have any text yet, we
160 // don't require the digits.
161 if strings.HasSuffix(line, ":") && (strings.ContainsAny(line, "0123456789") || result == "") && nextLineQuoted(0) {
162 if !resultSnipped() {
163 result += "[...]\n"
164 }
165 lines = lines[1:]
166 continue
167 }
168 // Skip possibly duplicate snipping by author.
169 if !isSnipped(line) || !resultSnipped() {
170 result += line + "\n"
171 }
172 lines = lines[1:]
173 if len(result) > 250 {
174 break
175 }
176 }
177 if len(result) > 250 {
178 result = result[:230] + "..."
179 }
180 return result, scanner.Err()
181}
182
183func parsedMessage(log mlog.Log, m store.Message, state *msgState, full, msgitem bool) (pm ParsedMessage, rerr error) {
184 if full || msgitem {
185 if !state.ensurePart(m, true) {
186 return pm, state.err
187 }
188 if full {
189 pm.Part = *state.part
190 }
191 } else {
192 if !state.ensurePart(m, false) {
193 return pm, state.err
194 }
195 }
196
197 // todo: we should store this form in message.Part, requires a data structure update.
198
199 convertAddrs := func(l []message.Address) []MessageAddress {
200 r := make([]MessageAddress, len(l))
201 for i, a := range l {
202 d, err := dns.ParseDomain(a.Host)
203 log.Check(err, "parsing domain")
204 if err != nil {
205 d = dns.Domain{ASCII: a.Host}
206 }
207 r[i] = MessageAddress{a.Name, a.User, d}
208 }
209 return r
210 }
211
212 if full || msgitem {
213 env := MessageEnvelope{}
214 if state.part.Envelope != nil {
215 e := *state.part.Envelope
216 env.Date = e.Date
217 env.Subject = e.Subject
218 env.InReplyTo = e.InReplyTo
219 env.MessageID = e.MessageID
220 env.From = convertAddrs(e.From)
221 env.Sender = convertAddrs(e.Sender)
222 env.ReplyTo = convertAddrs(e.ReplyTo)
223 env.To = convertAddrs(e.To)
224 env.CC = convertAddrs(e.CC)
225 env.BCC = convertAddrs(e.BCC)
226 }
227 pm.envelope = env
228 }
229
230 if full && state.part.BodyOffset > 0 {
231 hdrs, err := state.part.Header()
232 if err != nil {
233 return ParsedMessage{}, fmt.Errorf("parsing headers: %v", err)
234 }
235 pm.Headers = hdrs
236
237 pm.ListReplyAddress = parseListPostAddress(hdrs.Get("List-Post"))
238 } else {
239 pm.Headers = map[string][]string{}
240 }
241
242 pm.Texts = []string{}
243 pm.attachments = []Attachment{}
244
245 // todo: how should we handle messages where a user prefers html, and we want to show it, but it's a DSN that also has textual-only parts? e.g. gmail's dsn where the first part is multipart/related with multipart/alternative, and second part is the regular message/delivery-status. we want to display both the html and the text.
246
247 var usePart func(p message.Part, index int, parent *message.Part, path []int)
248 usePart = func(p message.Part, index int, parent *message.Part, path []int) {
249 mt := p.MediaType + "/" + p.MediaSubType
250 for i, sp := range p.Parts {
251 if mt == "MULTIPART/SIGNED" && i >= 1 {
252 continue
253 }
254 usePart(sp, i, &p, append(append([]int{}, path...), i))
255 }
256 switch mt {
257 case "TEXT/PLAIN", "/":
258 // Don't include if Content-Disposition attachment.
259 if full || msgitem {
260 // todo: should have this, and perhaps all content-* headers, preparsed in message.Part?
261 h, err := p.Header()
262 log.Check(err, "parsing attachment headers", slog.Int64("msgid", m.ID))
263 cp := h.Get("Content-Disposition")
264 if cp != "" {
265 disp, params, err := mime.ParseMediaType(cp)
266 log.Check(err, "parsing content-disposition", slog.String("cp", cp))
267 if strings.EqualFold(disp, "attachment") {
268 name := tryDecodeParam(log, p.ContentTypeParams["name"])
269 if name == "" {
270 name = tryDecodeParam(log, params["filename"])
271 }
272 pm.attachments = append(pm.attachments, Attachment{path, name, p})
273 return
274 }
275 }
276 }
277
278 if full {
279 buf, err := io.ReadAll(&moxio.LimitReader{R: p.ReaderUTF8OrBinary(), Limit: 2 * 1024 * 1024})
280 if err != nil {
281 rerr = fmt.Errorf("reading text part: %v", err)
282 return
283 }
284 pm.Texts = append(pm.Texts, string(buf))
285 }
286 if msgitem && pm.firstLine == "" {
287 pm.firstLine, rerr = formatFirstLine(p.ReaderUTF8OrBinary())
288 if rerr != nil {
289 rerr = fmt.Errorf("reading text for first line snippet: %v", rerr)
290 return
291 }
292 }
293
294 case "TEXT/HTML":
295 pm.HasHTML = true
296
297 default:
298 // todo: see if there is a common nesting messages that are both signed and encrypted.
299 if parent == nil && mt == "MULTIPART/SIGNED" {
300 pm.isSigned = true
301 }
302 if parent == nil && mt == "MULTIPART/ENCRYPTED" {
303 pm.isEncrypted = true
304 }
305 // todo: possibly do not include anything below multipart/alternative that starts with text/html, they may be cids. perhaps have a separate list of attachments for the text vs html version?
306 if p.MediaType != "MULTIPART" {
307 var parentct string
308 if parent != nil {
309 parentct = parent.MediaType + "/" + parent.MediaSubType
310 }
311
312 // Recognize DSNs.
313 if parentct == "MULTIPART/REPORT" && index == 1 && (mt == "MESSAGE/GLOBAL-DELIVERY-STATUS" || mt == "MESSAGE/DELIVERY-STATUS") {
314 if full {
315 buf, err := io.ReadAll(&moxio.LimitReader{R: p.ReaderUTF8OrBinary(), Limit: 1024 * 1024})
316 if err != nil {
317 rerr = fmt.Errorf("reading text part: %v", err)
318 return
319 }
320 pm.Texts = append(pm.Texts, string(buf))
321 }
322 return
323 }
324 if parentct == "MULTIPART/REPORT" && index == 2 && (mt == "MESSAGE/GLOBAL-HEADERS" || mt == "TEXT/RFC822-HEADERS") {
325 if full {
326 buf, err := io.ReadAll(&moxio.LimitReader{R: p.ReaderUTF8OrBinary(), Limit: 1024 * 1024})
327 if err != nil {
328 rerr = fmt.Errorf("reading text part: %v", err)
329 return
330 }
331 pm.Texts = append(pm.Texts, string(buf))
332 }
333 return
334 }
335 if parentct == "MULTIPART/REPORT" && index == 2 && (mt == "MESSAGE/GLOBAL" || mt == "TEXT/RFC822") {
336 pm.attachments = append(pm.attachments, Attachment{path, "original.eml", p})
337 return
338 }
339
340 name := tryDecodeParam(log, p.ContentTypeParams["name"])
341 if name == "" && (full || msgitem) {
342 // todo: should have this, and perhaps all content-* headers, preparsed in message.Part?
343 h, err := p.Header()
344 log.Check(err, "parsing attachment headers", slog.Int64("msgid", m.ID))
345 cp := h.Get("Content-Disposition")
346 if cp != "" {
347 _, params, err := mime.ParseMediaType(cp)
348 log.Check(err, "parsing content-disposition", slog.String("cp", cp))
349 name = tryDecodeParam(log, params["filename"])
350 }
351 }
352 pm.attachments = append(pm.attachments, Attachment{path, name, p})
353 }
354 }
355 }
356 usePart(*state.part, -1, nil, []int{})
357
358 if rerr == nil {
359 pm.ID = m.ID
360 }
361 return
362}
363
364// parses List-Post header, returning an address if it could be found, and nil otherwise.
365func parseListPostAddress(s string) *MessageAddress {
366 /*
367 Examples:
368 List-Post: <mailto:list@host.com>
369 List-Post: <mailto:moderator@host.com> (Postings are Moderated)
370 List-Post: <mailto:moderator@host.com?subject=list%20posting>
371 List-Post: NO (posting not allowed on this list)
372 List-Post: <https://groups.google.com/group/golang-dev/post>, <mailto:golang-dev@googlegroups.com>
373 */
374 s = strings.TrimSpace(s)
375 for s != "" {
376 if !strings.HasPrefix(s, "<") {
377 return nil
378 }
379 addr, ns, found := strings.Cut(s[1:], ">")
380 if !found {
381 return nil
382 }
383 if strings.HasPrefix(addr, "mailto:") {
384 u, err := url.Parse(addr)
385 if err != nil {
386 return nil
387 }
388 addr, err := smtp.ParseAddress(u.Opaque)
389 if err != nil {
390 return nil
391 }
392 return &MessageAddress{User: addr.Localpart.String(), Domain: addr.Domain}
393 }
394 s = strings.TrimSpace(ns)
395 s = strings.TrimPrefix(s, ",")
396 s = strings.TrimSpace(s)
397 }
398 return nil
399}
400