1package store
2
3import (
4 "bufio"
5 "bytes"
6 "errors"
7 "fmt"
8 "io"
9 "maps"
10 "os"
11 "path/filepath"
12 "slices"
13 "strconv"
14 "strings"
15 "time"
16
17 "github.com/mjl-/mox/mlog"
18)
19
20// MsgSource is implemented by readers for mailbox file formats.
21type MsgSource interface {
22 // Return next message, or io.EOF when there are no more.
23 Next() (*Message, *os.File, string, error)
24}
25
26// MboxReader reads messages from an mbox file, implementing MsgSource.
27type MboxReader struct {
28 log mlog.Log
29 createTemp func(log mlog.Log, pattern string) (*os.File, error)
30 path string
31 line int
32 r *bufio.Reader
33 prevempty bool
34 nonfirst bool
35 eof bool
36 fromLine string // "From "-line for this message.
37 header bool // Now in header section.
38}
39
40func NewMboxReader(log mlog.Log, createTemp func(log mlog.Log, pattern string) (*os.File, error), filename string, r io.Reader) *MboxReader {
41 return &MboxReader{
42 log: log,
43 createTemp: createTemp,
44 path: filename,
45 line: 1,
46 r: bufio.NewReader(r),
47 }
48}
49
50// Position returns "<filename>:<lineno>" for the current position.
51func (mr *MboxReader) Position() string {
52 return fmt.Sprintf("%s:%d", mr.path, mr.line)
53}
54
55// Next returns the next message read from the mbox file. The file is a temporary
56// file and must be removed/consumed. The third return value is the position in the
57// file.
58func (mr *MboxReader) Next() (*Message, *os.File, string, error) {
59 if mr.eof {
60 return nil, nil, "", io.EOF
61 }
62
63 from := []byte("From ")
64
65 if !mr.nonfirst {
66 mr.header = true
67 // First read, we're at the beginning of the file.
68 line, err := mr.r.ReadBytes('\n')
69 if err == io.EOF {
70 return nil, nil, "", io.EOF
71 }
72 mr.line++
73
74 if !bytes.HasPrefix(line, from) {
75 return nil, nil, mr.Position(), fmt.Errorf(`first line does not start with "From "`)
76 }
77 mr.nonfirst = true
78 mr.fromLine = strings.TrimSpace(string(line))
79 }
80
81 f, err := mr.createTemp(mr.log, "mboxreader")
82 if err != nil {
83 return nil, nil, mr.Position(), err
84 }
85 defer func() {
86 if f != nil {
87 CloseRemoveTempFile(mr.log, f, "message after mbox read error")
88 }
89 }()
90
91 fromLine := mr.fromLine
92 bf := bufio.NewWriter(f)
93 var flags Flags
94 keywords := map[string]bool{}
95 var size int64
96 for {
97 line, err := mr.r.ReadBytes('\n')
98 if err != nil && err != io.EOF {
99 return nil, nil, mr.Position(), fmt.Errorf("reading from mbox: %v", err)
100 }
101 if len(line) > 0 {
102 mr.line++
103 // We store data with crlf, adjust any imported messages with bare newlines. ../rfc/4155:354
104 if !bytes.HasSuffix(line, []byte("\r\n")) {
105 line = append(line[:len(line)-1], "\r\n"...)
106 }
107
108 if mr.header {
109 // See https://doc.dovecot.org/admin_manual/mailbox_formats/mbox/
110 if bytes.HasPrefix(line, []byte("Status:")) {
111 s := strings.TrimSpace(strings.SplitN(string(line), ":", 2)[1])
112 for _, c := range s {
113 switch c {
114 case 'R':
115 flags.Seen = true
116 }
117 }
118 } else if bytes.HasPrefix(line, []byte("X-Status:")) {
119 s := strings.TrimSpace(strings.SplitN(string(line), ":", 2)[1])
120 for _, c := range s {
121 switch c {
122 case 'A':
123 flags.Answered = true
124 case 'F':
125 flags.Flagged = true
126 case 'T':
127 flags.Draft = true
128 case 'D':
129 flags.Deleted = true
130 }
131 }
132 } else if bytes.HasPrefix(line, []byte("X-Keywords:")) {
133 s := strings.TrimSpace(strings.SplitN(string(line), ":", 2)[1])
134 for _, t := range strings.Split(s, ",") {
135 word := strings.ToLower(strings.TrimSpace(t))
136 switch word {
137 case "forwarded", "$forwarded":
138 flags.Forwarded = true
139 case "junk", "$junk":
140 flags.Junk = true
141 case "notjunk", "$notjunk", "nonjunk", "$nonjunk":
142 flags.Notjunk = true
143 case "phishing", "$phishing":
144 flags.Phishing = true
145 case "mdnsent", "$mdnsent":
146 flags.MDNSent = true
147 default:
148 if err := CheckKeyword(word); err == nil {
149 keywords[word] = true
150 }
151 }
152 }
153 }
154 }
155 if bytes.Equal(line, []byte("\r\n")) {
156 mr.header = false
157 }
158
159 // Next mail message starts at bare From word. ../rfc/4155:71
160 if mr.prevempty && bytes.HasPrefix(line, from) {
161 mr.fromLine = strings.TrimSpace(string(line))
162 mr.header = true
163 break
164 }
165 // ../rfc/4155:119
166 if bytes.HasPrefix(line, []byte(">")) && bytes.HasPrefix(bytes.TrimLeft(line, ">"), []byte("From ")) {
167 line = line[1:]
168 }
169 n, err := bf.Write(line)
170 if err != nil {
171 return nil, nil, mr.Position(), fmt.Errorf("writing message to file: %v", err)
172 }
173 size += int64(n)
174 mr.prevempty = bytes.Equal(line, []byte("\r\n"))
175 }
176 if err == io.EOF {
177 mr.eof = true
178 break
179 }
180 }
181 if err := bf.Flush(); err != nil {
182 return nil, nil, mr.Position(), fmt.Errorf("flush: %v", err)
183 }
184
185 m := &Message{Flags: flags, Keywords: slices.Sorted(maps.Keys(keywords)), Size: size}
186
187 if t := strings.SplitN(fromLine, " ", 3); len(t) == 3 {
188 layouts := []string{time.ANSIC, time.UnixDate, time.RubyDate}
189 for _, l := range layouts {
190 t, err := time.Parse(l, t[2])
191 if err == nil {
192 m.Received = t
193 break
194 }
195 }
196 }
197
198 // Prevent cleanup by defer.
199 mf := f
200 f = nil
201
202 return m, mf, mr.Position(), nil
203}
204
205type MaildirReader struct {
206 log mlog.Log
207 createTemp func(log mlog.Log, pattern string) (*os.File, error)
208 newf, curf *os.File
209 f *os.File // File we are currently reading from. We first read newf, then curf.
210 dir string // Name of directory for f. Can be empty on first call.
211 entries []os.DirEntry
212 dovecotFlags []string // Lower-case flags/keywords.
213}
214
215func NewMaildirReader(log mlog.Log, createTemp func(log mlog.Log, pattern string) (*os.File, error), newf, curf *os.File) *MaildirReader {
216 mr := &MaildirReader{
217 log: log,
218 createTemp: createTemp,
219 newf: newf,
220 curf: curf,
221 f: newf,
222 }
223
224 // Best-effort parsing of dovecot keywords.
225 kf, err := os.Open(filepath.Join(filepath.Dir(newf.Name()), "dovecot-keywords"))
226 if err == nil {
227 mr.dovecotFlags, err = ParseDovecotKeywordsFlags(kf, log)
228 log.Check(err, "parsing dovecot keywords file")
229 err = kf.Close()
230 log.Check(err, "closing dovecot-keywords file")
231 }
232
233 return mr
234}
235
236func (mr *MaildirReader) Next() (*Message, *os.File, string, error) {
237 if mr.dir == "" {
238 mr.dir = mr.f.Name()
239 }
240
241 if len(mr.entries) == 0 {
242 var err error
243 mr.entries, err = mr.f.ReadDir(100)
244 if err != nil && err != io.EOF {
245 return nil, nil, "", err
246 }
247 if len(mr.entries) == 0 {
248 if mr.f == mr.curf {
249 return nil, nil, "", io.EOF
250 }
251 mr.f = mr.curf
252 mr.dir = ""
253 return mr.Next()
254 }
255 }
256
257 p := filepath.Join(mr.dir, mr.entries[0].Name())
258 mr.entries = mr.entries[1:]
259 sf, err := os.Open(p)
260 if err != nil {
261 return nil, nil, p, fmt.Errorf("open message in maildir: %s", err)
262 }
263 defer func() {
264 err := sf.Close()
265 mr.log.Check(err, "closing message file after error")
266 }()
267 f, err := mr.createTemp(mr.log, "maildirreader")
268 if err != nil {
269 return nil, nil, p, err
270 }
271 defer func() {
272 if f != nil {
273 CloseRemoveTempFile(mr.log, f, "maildir temp message file")
274 }
275 }()
276
277 // Copy data, changing bare \n into \r\n.
278 r := bufio.NewReader(sf)
279 w := bufio.NewWriter(f)
280 var size int64
281 for {
282 line, err := r.ReadBytes('\n')
283 if err != nil && err != io.EOF {
284 return nil, nil, p, fmt.Errorf("reading message: %v", err)
285 }
286 if len(line) > 0 {
287 if !bytes.HasSuffix(line, []byte("\r\n")) {
288 line = append(line[:len(line)-1], "\r\n"...)
289 }
290
291 if n, err := w.Write(line); err != nil {
292 return nil, nil, p, fmt.Errorf("writing message: %v", err)
293 } else {
294 size += int64(n)
295 }
296 }
297 if err == io.EOF {
298 break
299 }
300 }
301 if err := w.Flush(); err != nil {
302 return nil, nil, p, fmt.Errorf("writing message: %v", err)
303 }
304
305 // Take received time from filename, falling back to mtime for maildirs
306 // reconstructed some other sources of message files.
307 var received time.Time
308 t := strings.SplitN(filepath.Base(sf.Name()), ".", 3)
309 if v, err := strconv.ParseInt(t[0], 10, 64); len(t) == 3 && err == nil {
310 received = time.Unix(v, 0)
311 } else if fi, err := sf.Stat(); err == nil {
312 received = fi.ModTime()
313 }
314
315 // Parse flags. See https://cr.yp.to/proto/maildir.html.
316 flags := Flags{}
317 keywords := map[string]bool{}
318 t = strings.SplitN(filepath.Base(sf.Name()), ":2,", 2)
319 if len(t) == 2 {
320 for _, c := range t[1] {
321 switch c {
322 case 'P':
323 // Passed, doesn't map to a common IMAP flag.
324 case 'R':
325 flags.Answered = true
326 case 'S':
327 flags.Seen = true
328 case 'T':
329 flags.Deleted = true
330 case 'D':
331 flags.Draft = true
332 case 'F':
333 flags.Flagged = true
334 default:
335 if c >= 'a' && c <= 'z' {
336 index := int(c - 'a')
337 if index >= len(mr.dovecotFlags) {
338 continue
339 }
340 kw := mr.dovecotFlags[index]
341 switch kw {
342 case "$forwarded", "forwarded":
343 flags.Forwarded = true
344 case "$junk", "junk":
345 flags.Junk = true
346 case "$notjunk", "notjunk", "nonjunk":
347 flags.Notjunk = true
348 case "$mdnsent", "mdnsent":
349 flags.MDNSent = true
350 case "$phishing", "phishing":
351 flags.Phishing = true
352 default:
353 keywords[kw] = true
354 }
355 }
356 }
357 }
358 }
359
360 m := &Message{Received: received, Flags: flags, Keywords: slices.Sorted(maps.Keys(keywords)), Size: size}
361
362 // Prevent cleanup by defer.
363 mf := f
364 f = nil
365
366 return m, mf, p, nil
367}
368
369// ParseDovecotKeywordsFlags attempts to parse a dovecot-keywords file. It only
370// returns valid flags/keywords, as lower-case. If an error is encountered and
371// returned, any keywords that were found are still returned. The returned list has
372// both system/well-known flags and custom keywords.
373func ParseDovecotKeywordsFlags(r io.Reader, log mlog.Log) ([]string, error) {
374 /*
375 If the dovecot-keywords file is present, we parse its additional flags, see
376 https://doc.dovecot.org/admin_manual/mailbox_formats/maildir/
377
378 0 Old
379 1 Junk
380 2 NonJunk
381 3 $Forwarded
382 4 $Junk
383 */
384 keywords := make([]string, 26)
385 end := 0
386 scanner := bufio.NewScanner(r)
387 var errs []string
388 for scanner.Scan() {
389 s := scanner.Text()
390 t := strings.SplitN(s, " ", 2)
391 if len(t) != 2 {
392 errs = append(errs, fmt.Sprintf("unexpected dovecot keyword line: %q", s))
393 continue
394 }
395 v, err := strconv.ParseInt(t[0], 10, 32)
396 if err != nil {
397 errs = append(errs, fmt.Sprintf("unexpected dovecot keyword index: %q", s))
398 continue
399 }
400 if v < 0 || v >= int64(len(keywords)) {
401 errs = append(errs, fmt.Sprintf("dovecot keyword index too big: %q", s))
402 continue
403 }
404 index := int(v)
405 if keywords[index] != "" {
406 errs = append(errs, fmt.Sprintf("duplicate dovecot keyword: %q", s))
407 continue
408 }
409 kw := strings.ToLower(t[1])
410 if !systemWellKnownFlags[kw] {
411 if err := CheckKeyword(kw); err != nil {
412 errs = append(errs, fmt.Sprintf("invalid keyword %q", kw))
413 continue
414 }
415 }
416 keywords[index] = kw
417 if index >= end {
418 end = index + 1
419 }
420 }
421 if err := scanner.Err(); err != nil {
422 errs = append(errs, fmt.Sprintf("reading dovecot keywords file: %v", err))
423 }
424 var err error
425 if len(errs) > 0 {
426 err = errors.New(strings.Join(errs, "; "))
427 }
428 return keywords[:end], err
429}
430