1package store

2

3import (

4 "archive/tar"

5 "archive/zip"

6 "bufio"

7 "bytes"

8 "context"

9 "fmt"

10 "io"

11 "log/slog"

12 "os"

13 "path"

14 "path/filepath"

15 "strings"

16 "time"

17

18 "github.com/mjl-/bstore"

19

20 "github.com/mjl-/mox/mlog"

21)

22

23// Archiver can archive multiple mailboxes and their messages.

24type Archiver interface {

25 // Add file to archive. If name ends with a slash, it is created as a directory and

26 // the returned io.WriteCloser can be ignored.

27 Create(name string, size int64, mtime time.Time) (io.WriteCloser, error)

28 Close() error

29}

30

31// TarArchiver is an Archiver that writes to a tar file.

32type TarArchiver struct {

33 *tar.Writer

34}

35

36// Create adds a file header to the tar file.

37func (a TarArchiver) Create(name string, size int64, mtime time.Time) (io.WriteCloser, error) {

38 hdr := tar.Header{

39 Name: name,

40 Size: size,

41 Mode: 0660,

42 ModTime: mtime,

43 Format: tar.FormatPAX,

44 }

45 if err := a.WriteHeader(&hdr); err != nil {

46 return nil, err

47 }

48 return nopCloser{a}, nil

49}

50

51// ZipArchiver is an Archiver that writes to a zip file.

52type ZipArchiver struct {

53 *zip.Writer

54}

55

56// Create adds a file header to the zip file.

57func (a ZipArchiver) Create(name string, size int64, mtime time.Time) (io.WriteCloser, error) {

58 hdr := zip.FileHeader{

59 Name: name,

60 Method: zip.Deflate,

61 Modified: mtime,

62 UncompressedSize64: uint64(size),

63 }

64 w, err := a.CreateHeader(&hdr)

65 if err != nil {

66 return nil, err

67 }

68 return nopCloser{w}, nil

69}

70

71type nopCloser struct {

72 io.Writer

73}

74

75// Close does nothing.

76func (nopCloser) Close() error {

77 return nil

78}

79

80// DirArchiver is an Archiver that writes to a directory.

81type DirArchiver struct {

82 Dir string

83}

84

85// Create creates name in the file system, in dir.

86// name must always use forwarded slashes.

87func (a DirArchiver) Create(name string, size int64, mtime time.Time) (io.WriteCloser, error) {

88 isdir := strings.HasSuffix(name, "/")

89 name = strings.TrimSuffix(name, "/")

90 p := filepath.Join(a.Dir, filepath.FromSlash(name))

91 os.MkdirAll(filepath.Dir(p), 0770)

92 if isdir {

93 return nil, os.Mkdir(p, 0770)

94 }

95 return os.OpenFile(p, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0660)

96}

97

98// Close on a dir does nothing.

99func (a DirArchiver) Close() error {

100 return nil

101}

102

103// MboxArchive fakes being an archiver to which a single mbox file can be written.

104// It returns an error when a second file is added. It returns its writer for the

105// first file to be written, leaving parameters unused.

106type MboxArchiver struct {

107 Writer io.Writer

108 have bool

109}

110

111// Create returns the underlying writer for the first call, and an error on later calls.

112func (a *MboxArchiver) Create(name string, size int64, mtime time.Time) (io.WriteCloser, error) {

113 if a.have {

114 return nil, fmt.Errorf("cannot export multiple files with mbox")

115 }

116 a.have = true

117 return nopCloser{a.Writer}, nil

118}

119

120// Close on an mbox archiver does nothing.

121func (a *MboxArchiver) Close() error {

122 return nil

123}

124

125// ExportMessages writes messages to archiver. Either in maildir format, or otherwise in

126// mbox. If mailboxOpt is empty, all mailboxes are exported, otherwise only the

127// named mailbox.

128//

129// Some errors are not fatal and result in skipped messages. In that happens, a

130// file "errors.txt" is added to the archive describing the errors. The goal is to

131// let users export (hopefully) most messages even in the face of errors.

132func ExportMessages(ctx context.Context, log mlog.Log, db *bstore.DB, accountDir string, archiver Archiver, maildir bool, mailboxOpt string, recursive bool) error {

133 // todo optimize: should prepare next file to add to archive (can be an mbox with many messages) while writing a file to the archive (which typically compresses, which takes time).

134

135 // Start transaction without closure, we are going to close it early, but don't

136 // want to deal with declaring many variables now to be able to assign them in a

137 // closure and use them afterwards.

138 tx, err := db.Begin(ctx, false)

139 if err != nil {

140 return fmt.Errorf("transaction: %v", err)

141 }

142 defer func() {

143 err := tx.Rollback()

144 log.Check(err, "transaction rollback")

145 }()

146

147 start := time.Now()

148

149 // We keep track of errors reading message files. We continue exporting and add an

150 // errors.txt file to the archive. In case of errors, the user can get (hopefully)

151 // most of their emails, and see something went wrong. For other errors, like

152 // writing to the archiver (e.g. a browser), we abort, because we don't want to

153 // continue with useless work.

154 var errors string

155

156 // Process mailboxes sorted by name, so submaildirs come after their parent.

157 prefix := mailboxOpt + "/"

158 var trimPrefix string

159 if mailboxOpt != "" {

160 // If exporting a specific mailbox, trim its parent path from stored file names.

161 trimPrefix = path.Dir(mailboxOpt) + "/"

162 }

163 q := bstore.QueryTx[Mailbox](tx)

164 q.FilterFn(func(mb Mailbox) bool {

165 return mailboxOpt == "" || mb.Name == mailboxOpt || recursive && strings.HasPrefix(mb.Name, prefix)

166 })

167 q.SortAsc("Name")

168 err = q.ForEach(func(mb Mailbox) error {

169 mailboxName := mb.Name

170 if trimPrefix != "" {

171 mailboxName = strings.TrimPrefix(mailboxName, trimPrefix)

172 }

173 errmsgs, err := exportMailbox(log, tx, accountDir, mb.ID, mailboxName, archiver, maildir, start)

174 if err != nil {

175 return err

176 }

177 errors += errmsgs

178 return nil

179 })

180 if err != nil {

181 return fmt.Errorf("query mailboxes: %w", err)

182 }

183

184 if errors != "" {

185 w, err := archiver.Create("errors.txt", int64(len(errors)), time.Now())

186 if err != nil {

187 log.Errorx("adding errors.txt to archive", err)

188 return err

189 }

190 if _, err := w.Write([]byte(errors)); err != nil {

191 log.Errorx("writing errors.txt to archive", err)

192 xerr := w.Close()

193 log.Check(xerr, "closing errors.txt after error")

194 return err

195 }

196 if err := w.Close(); err != nil {

197 return err

198 }

199 }

200 return nil

201}

202

203func exportMailbox(log mlog.Log, tx *bstore.Tx, accountDir string, mailboxID int64, mailboxName string, archiver Archiver, maildir bool, start time.Time) (string, error) {

204 var errors string

205

206 var mboxtmp *os.File

207 var mboxwriter *bufio.Writer

208 defer func() {

209 if mboxtmp != nil {

210 CloseRemoveTempFile(log, mboxtmp, "mbox")

211 }

212 }()

213

214 // For dovecot-keyword-style flags not in standard maildir.

215 maildirFlags := map[string]int{}

216 var maildirFlaglist []string

217 maildirFlag := func(flag string) string {

218 i, ok := maildirFlags[flag]

219 if !ok {

220 if len(maildirFlags) >= 26 {

221 // Max 26 flag characters.

222 return ""

223 }

224 i = len(maildirFlags)

225 maildirFlags[flag] = i

226 maildirFlaglist = append(maildirFlaglist, flag)

227 }

228 return string(rune('a' + i))

229 }

230

231 finishMailbox := func() error {

232 if maildir {

233 if len(maildirFlags) == 0 {

234 return nil

235 }

236

237 var b bytes.Buffer

238 for i, flag := range maildirFlaglist {

239 if _, err := fmt.Fprintf(&b, "%d %s\n", i, flag); err != nil {

240 return err

241 }

242 }

243 w, err := archiver.Create(mailboxName+"/dovecot-keywords", int64(b.Len()), start)

244 if err != nil {

245 return fmt.Errorf("adding dovecot-keywords: %v", err)

246 }

247 if _, err := w.Write(b.Bytes()); err != nil {

248 xerr := w.Close()

249 log.Check(xerr, "closing dovecot-keywords file after closing")

250 return fmt.Errorf("writing dovecot-keywords: %v", err)

251 }

252 maildirFlags = map[string]int{}

253 maildirFlaglist = nil

254 return w.Close()

255 }

256

257 if err := mboxwriter.Flush(); err != nil {

258 return fmt.Errorf("flush mbox writer: %v", err)

259 }

260 fi, err := mboxtmp.Stat()

261 if err != nil {

262 return fmt.Errorf("stat temporary mbox file: %v", err)

263 }

264 if _, err := mboxtmp.Seek(0, 0); err != nil {

265 return fmt.Errorf("seek to start of temporary mbox file")

266 }

267 w, err := archiver.Create(mailboxName+".mbox", fi.Size(), fi.ModTime())

268 if err != nil {

269 return fmt.Errorf("add mbox to archive: %v", err)

270 }

271 if _, err := io.Copy(w, mboxtmp); err != nil {

272 xerr := w.Close()

273 log.Check(xerr, "closing mbox message file after error")

274 return fmt.Errorf("copying temp mbox file to archive: %v", err)

275 }

276 if err := w.Close(); err != nil {

277 return fmt.Errorf("closing message file: %v", err)

278 }

279 name := mboxtmp.Name()

280 err = mboxtmp.Close()

281 log.Check(err, "closing temporary mbox file")

282 err = os.Remove(name)

283 log.Check(err, "removing temporary mbox file", slog.String("path", name))

284 mboxwriter = nil

285 mboxtmp = nil

286 return nil

287 }

288

289 exportMessage := func(m Message) error {

290 mp := filepath.Join(accountDir, "msg", MessagePath(m.ID))

291 var mr io.ReadCloser

292 if m.Size == int64(len(m.MsgPrefix)) {

293 mr = io.NopCloser(bytes.NewReader(m.MsgPrefix))

294 } else {

295 mf, err := os.Open(mp)

296 if err != nil {

297 errors += fmt.Sprintf("open message file for id %d, path %s: %v (message skipped)\n", m.ID, mp, err)

298 return nil

299 }

300 defer func() {

301 err := mf.Close()

302 log.Check(err, "closing message file after export")

303 }()

304 st, err := mf.Stat()

305 if err != nil {

306 errors += fmt.Sprintf("stat message file for id %d, path %s: %v (message skipped)\n", m.ID, mp, err)

307 return nil

308 }

309 size := st.Size() + int64(len(m.MsgPrefix))

310 if size != m.Size {

311 errors += fmt.Sprintf("message size mismatch for message id %d, database has %d, size is %d+%d=%d, using calculated size\n", m.ID, m.Size, len(m.MsgPrefix), st.Size(), size)

312 }

313 mr = FileMsgReader(m.MsgPrefix, mf)

314 }

315

316 if maildir {

317 p := mailboxName

318 if m.Flags.Seen {

319 p = filepath.Join(p, "cur")

320 } else {

321 p = filepath.Join(p, "new")

322 }

323 name := fmt.Sprintf("%d.%d.mox:2,", m.Received.Unix(), m.ID)

324

325 // Standard flags. May need to be sorted.

326 if m.Flags.Draft {

327 name += "D"

328 }

329 if m.Flags.Flagged {

330 name += "F"

331 }

332 if m.Flags.Answered {

333 name += "R"

334 }

335 if m.Flags.Seen {

336 name += "S"

337 }

338 if m.Flags.Deleted {

339 name += "T"

340 }

341

342 // Non-standard flag. We set them with a dovecot-keywords file.

343 if m.Flags.Forwarded {

344 name += maildirFlag("$Forwarded")

345 }

346 if m.Flags.Junk {

347 name += maildirFlag("$Junk")

348 }

349 if m.Flags.Notjunk {

350 name += maildirFlag("$NotJunk")

351 }

352 if m.Flags.Phishing {

353 name += maildirFlag("$Phishing")

354 }

355 if m.Flags.MDNSent {

356 name += maildirFlag("$MDNSent")

357 }

358

359 p = filepath.Join(p, name)

360

361 // We store messages with \r\n, maildir needs without. But we need to know the

362 // final size. So first convert, then create file with size, and write from buffer.

363 // todo: for large messages, we should go through a temporary file instead of memory.

364 var dst bytes.Buffer

365 r := bufio.NewReader(mr)

366 for {

367 line, rerr := r.ReadBytes('\n')

368 if rerr != io.EOF && rerr != nil {

369 errors += fmt.Sprintf("reading from message for id %d: %v (message skipped)\n", m.ID, rerr)

370 return nil

371 }

372 if len(line) > 0 {

373 if bytes.HasSuffix(line, []byte("\r\n")) {

374 line = line[:len(line)-1]

375 line[len(line)-1] = '\n'

376 }

377 if _, err := dst.Write(line); err != nil {

378 return fmt.Errorf("writing message: %v", err)

379 }

380 }

381 if rerr == io.EOF {

382 break

383 }

384 }

385 size := int64(dst.Len())

386 w, err := archiver.Create(p, size, m.Received)

387 if err != nil {

388 return fmt.Errorf("adding message to archive: %v", err)

389 }

390 if _, err := io.Copy(w, &dst); err != nil {

391 xerr := w.Close()

392 log.Check(xerr, "closing message")

393 return fmt.Errorf("copying message to archive: %v", err)

394 }

395 return w.Close()

396 }

397

398 mailfrom := "mox"

399 if m.MailFrom != "" {

400 mailfrom = m.MailFrom

401 }

402 if _, err := fmt.Fprintf(mboxwriter, "From %s %s\n", mailfrom, m.Received.Format(time.ANSIC)); err != nil {

403 return fmt.Errorf("write message line to mbox temp file: %v", err)

404 }

405

406 // Write message flags in the three headers that mbox consumers may (or may not) understand.

407 if m.Seen {

408 if _, err := fmt.Fprintf(mboxwriter, "Status: R\n"); err != nil {

409 return fmt.Errorf("writing status header: %v", err)

410 }

411 }

412 xstatus := ""

413 if m.Answered {

414 xstatus += "A"

415 }

416 if m.Flagged {

417 xstatus += "F"

418 }

419 if m.Draft {

420 xstatus += "T"

421 }

422 if m.Deleted {

423 xstatus += "D"

424 }

425 if xstatus != "" {

426 if _, err := fmt.Fprintf(mboxwriter, "X-Status: %s\n", xstatus); err != nil {

427 return fmt.Errorf("writing x-status header: %v", err)

428 }

429 }

430 var xkeywords []string

431 if m.Forwarded {

432 xkeywords = append(xkeywords, "$Forwarded")

433 }

434 if m.Junk && !m.Notjunk {

435 xkeywords = append(xkeywords, "$Junk")

436 }

437 if m.Notjunk && !m.Junk {

438 xkeywords = append(xkeywords, "$NotJunk")

439 }

440 if m.Phishing {

441 xkeywords = append(xkeywords, "$Phishing")

442 }

443 if m.MDNSent {

444 xkeywords = append(xkeywords, "$MDNSent")

445 }

446 if len(xkeywords) > 0 {

447 if _, err := fmt.Fprintf(mboxwriter, "X-Keywords: %s\n", strings.Join(xkeywords, ",")); err != nil {

448 return fmt.Errorf("writing x-keywords header: %v", err)

449 }

450 }

451

452 header := true

453 r := bufio.NewReader(mr)

454 for {

455 line, rerr := r.ReadBytes('\n')

456 if rerr != io.EOF && rerr != nil {

457 return fmt.Errorf("reading message: %v", rerr)

458 }

459 if len(line) > 0 {

460 if bytes.HasSuffix(line, []byte("\r\n")) {

461 line = line[:len(line)-1]

462 line[len(line)-1] = '\n'

463 }

464 if header && len(line) == 1 {

465 header = false

466 }

467 if header {

468 // Skip any previously stored flag-holding or now incorrect content-length headers.

469 // This assumes these headers are just a single line.

470 switch strings.ToLower(string(bytes.SplitN(line, []byte(":"), 2)[0])) {

471 case "status", "x-status", "x-keywords", "content-length":

472 continue

473 }

474 }

475 if bytes.HasPrefix(bytes.TrimLeft(line, ">"), []byte("From ")) {

476 if _, err := fmt.Fprint(mboxwriter, ">"); err != nil {

477 return fmt.Errorf("writing escaping >: %v", err)

478 }

479 }

480 if _, err := mboxwriter.Write(line); err != nil {

481 return fmt.Errorf("writing line: %v", err)

482 }

483 }

484 if rerr == io.EOF {

485 break

486 }

487 }

488 if _, err := fmt.Fprint(mboxwriter, "\n"); err != nil {

489 return fmt.Errorf("writing end of message newline: %v", err)

490 }

491 return nil

492 }

493

494 if maildir {

495 // Create the directories that show this is a maildir.

496 if _, err := archiver.Create(mailboxName+"/new/", 0, start); err != nil {

497 return errors, fmt.Errorf("adding maildir new directory: %v", err)

498 }

499 if _, err := archiver.Create(mailboxName+"/cur/", 0, start); err != nil {

500 return errors, fmt.Errorf("adding maildir cur directory: %v", err)

501 }

502 if _, err := archiver.Create(mailboxName+"/tmp/", 0, start); err != nil {

503 return errors, fmt.Errorf("adding maildir tmp directory: %v", err)

504 }

505 } else {

506 var err error

507 mboxtmp, err = os.CreateTemp("", "mox-mail-export-mbox")

508 if err != nil {

509 return errors, fmt.Errorf("creating temp mbox file: %v", err)

510 }

511 mboxwriter = bufio.NewWriter(mboxtmp)

512 }

513

514 // Fetch all messages for mailbox.

515 q := bstore.QueryTx[Message](tx)

516 q.FilterNonzero(Message{MailboxID: mailboxID})

517 q.FilterEqual("Expunged", false)

518 q.SortAsc("Received", "ID")

519 err := q.ForEach(func(m Message) error {

520 return exportMessage(m)

521 })

522 if err != nil {

523 return errors, err

524 }

525 if err := finishMailbox(); err != nil {

526 return errors, err

527 }

528

529 return errors, nil

530}

531