19 "github.com/mjl-/bstore"
21 "github.com/mjl-/mox/dmarcdb"
22 "github.com/mjl-/mox/mox-"
23 "github.com/mjl-/mox/moxvar"
24 "github.com/mjl-/mox/mtastsdb"
25 "github.com/mjl-/mox/queue"
26 "github.com/mjl-/mox/store"
27 "github.com/mjl-/mox/tlsrptdb"
30func backupctl(ctx context.Context, ctl *ctl) {
39 // Convention in this function: variables containing "src" or "dst" are file system
40 // paths that can be passed to os.Open and such. Variables with dirs/paths without
41 // "src" or "dst" are incomplete paths relative to the source or destination data
45 verbose := ctl.xread() == "verbose"
47 // Set when an error is encountered. At the end, we warn if set.
50 // We'll be writing output, and logging both to mox and the ctl stream.
51 writer := ctl.writer()
53 // Format easily readable output for the user.
54 formatLog := func(prefix, text string, err error, attrs ...slog.Attr) []byte {
56 fmt.Fprint(&b, prefix)
59 fmt.Fprint(&b, ": "+err.Error())
61 for _, a := range attrs {
62 fmt.Fprintf(&b, "; %s=%v", a.Key, a.Value)
68 // Log an error to both the mox service as the user running "mox backup".
69 pkglogx := func(prefix, text string, err error, attrs ...slog.Attr) {
70 ctl.log.Errorx(text, err, attrs...)
72 _, werr := writer.Write(formatLog(prefix, text, err, attrs...))
73 ctl.xcheck(werr, "write to ctl")
76 // Log an error but don't mark backup as failed.
77 xwarnx := func(text string, err error, attrs ...slog.Attr) {
78 pkglogx("warning: ", text, err, attrs...)
81 // Log an error that causes the backup to be marked as failed. We typically
82 // continue processing though.
83 xerrx := func(text string, err error, attrs ...slog.Attr) {
85 pkglogx("error: ", text, err, attrs...)
88 // If verbose is enabled, log to the cli command. Always log as info level.
89 xvlog := func(text string, attrs ...slog.Attr) {
90 ctl.log.Info(text, attrs...)
92 _, werr := writer.Write(formatLog("", text, nil, attrs...))
93 ctl.xcheck(werr, "write to ctl")
97 dstConfigDir := filepath.Join(dstDir, "config")
98 dstDataDir := filepath.Join(dstDir, "data")
100 // Warn if directories already exist, will likely cause failures when trying to
101 // write files that already exist.
102 if _, err := os.Stat(dstConfigDir); err == nil {
103 xwarnx("destination config directory already exists", nil, slog.String("configdir", dstConfigDir))
105 if _, err := os.Stat(dstDataDir); err == nil {
106 xwarnx("destination data directory already exists", nil, slog.String("datadir", dstDataDir))
109 os.MkdirAll(dstDir, 0770)
110 os.MkdirAll(dstConfigDir, 0770)
111 os.MkdirAll(dstDataDir, 0770)
113 // Copy all files in the config dir.
114 srcConfigDir := filepath.Clean(mox.ConfigDirPath("."))
115 err := filepath.WalkDir(srcConfigDir, func(srcPath string, d fs.DirEntry, err error) error {
120 if srcConfigDir == srcPath {
124 // Trim directory and separator.
125 relPath := srcPath[len(srcConfigDir)+1:]
127 destPath := filepath.Join(dstConfigDir, relPath)
130 if info, err := os.Stat(srcPath); err != nil {
131 return fmt.Errorf("stat config dir %s: %v", srcPath, err)
132 } else if err := os.Mkdir(destPath, info.Mode()&0777); err != nil {
133 return fmt.Errorf("mkdir %s: %v", destPath, err)
137 if d.Type()&fs.ModeSymlink != 0 {
138 linkDest, err := os.Readlink(srcPath)
140 return fmt.Errorf("reading symlink %s: %v", srcPath, err)
142 if err := os.Symlink(linkDest, destPath); err != nil {
143 return fmt.Errorf("creating symlink %s: %v", destPath, err)
147 if !d.Type().IsRegular() {
148 xwarnx("skipping non-regular/dir/symlink file in config dir", nil, slog.String("path", srcPath))
152 sf, err := os.Open(srcPath)
154 return fmt.Errorf("open config file %s: %v", srcPath, err)
156 info, err := sf.Stat()
158 return fmt.Errorf("stat config file %s: %v", srcPath, err)
160 df, err := os.OpenFile(destPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0777&info.Mode())
162 return fmt.Errorf("create destination config file %s: %v", destPath, err)
167 ctl.log.Check(err, "closing file")
172 ctl.log.Check(err, "closing file")
174 if _, err := io.Copy(df, sf); err != nil {
175 return fmt.Errorf("copying config file %s to %s: %v", srcPath, destPath, err)
177 if err := df.Close(); err != nil {
178 return fmt.Errorf("closing destination config file %s: %v", srcPath, err)
184 xerrx("storing config directory", err)
187 srcDataDir := filepath.Clean(mox.DataDirPath("."))
189 // When creating a file in the destination, we first ensure its directory exists.
190 // We track which directories we created, to prevent needless syscalls.
191 createdDirs := map[string]struct{}{}
192 ensureDestDir := func(dstpath string) {
193 dstdir := filepath.Dir(dstpath)
194 if _, ok := createdDirs[dstdir]; !ok {
195 err := os.MkdirAll(dstdir, 0770)
197 xerrx("creating directory", err)
199 createdDirs[dstdir] = struct{}{}
203 // Backup a single file by copying (never hardlinking, the file may change).
204 backupFile := func(path string) {
206 srcpath := filepath.Join(srcDataDir, path)
207 dstpath := filepath.Join(dstDataDir, path)
209 sf, err := os.Open(srcpath)
211 xerrx("open source file (not backed up)", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
216 ensureDestDir(dstpath)
217 df, err := os.OpenFile(dstpath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660)
219 xerrx("creating destination file (not backed up)", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
227 if _, err := io.Copy(df, sf); err != nil {
228 xerrx("copying file (not backed up properly)", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
234 xerrx("closing destination file (not backed up properly)", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
237 xvlog("backed up file", slog.String("path", path), slog.Duration("duration", time.Since(tmFile)))
240 // Back up the files in a directory (by copying).
241 backupDir := func(dir string) {
243 srcdir := filepath.Join(srcDataDir, dir)
244 dstdir := filepath.Join(dstDataDir, dir)
245 err := filepath.WalkDir(srcdir, func(srcpath string, d fs.DirEntry, err error) error {
247 xerrx("walking file (not backed up)", err, slog.String("srcpath", srcpath))
253 backupFile(srcpath[len(srcDataDir)+1:])
257 xerrx("copying directory (not backed up properly)", err,
258 slog.String("srcdir", srcdir),
259 slog.String("dstdir", dstdir),
260 slog.Duration("duration", time.Since(tmDir)))
263 xvlog("backed up directory", slog.String("dir", dir), slog.Duration("duration", time.Since(tmDir)))
266 // Backup a database by copying it in a readonly transaction.
267 // Always logs on error, so caller doesn't have to, but also returns the error so
268 // callers can see result.
269 backupDB := func(db *bstore.DB, path string) (rerr error) {
272 xerrx("backing up database", rerr, slog.String("path", path))
278 dstpath := filepath.Join(dstDataDir, path)
279 ensureDestDir(dstpath)
280 df, err := os.OpenFile(dstpath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660)
282 return fmt.Errorf("creating destination file: %v", err)
289 err = db.Read(ctx, func(tx *bstore.Tx) error {
290 // Using regular WriteTo seems fine, and fast. It just copies pages.
292 // bolt.Compact is slower, it writes all key/value pairs, building up new data
293 // structures. My compacted test database was ~60% of original size. Lz4 on the
294 // uncompacted database got it to 14%. Lz4 on the compacted database got it to 13%.
295 // Backups are likely archived somewhere with compression, so we don't compact.
297 // Tests with WriteTo and os.O_DIRECT were slower than without O_DIRECT, but
298 // probably because everything fit in the page cache. It may be better to use
299 // O_DIRECT when copying many large or inactive databases.
300 _, err := tx.WriteTo(df)
304 return fmt.Errorf("copying database: %v", err)
309 return fmt.Errorf("closing destination database after copy: %v", err)
311 xvlog("backed up database file", slog.String("path", path), slog.Duration("duration", time.Since(tmDB)))
315 // Try to create a hardlink. Fall back to copying the file (e.g. when on different file system).
316 warnedHardlink := false // We warn once about failing to hardlink.
317 linkOrCopy := func(srcpath, dstpath string) (bool, error) {
318 ensureDestDir(dstpath)
320 if err := os.Link(srcpath, dstpath); err == nil {
322 } else if os.IsNotExist(err) {
323 // No point in trying with regular copy, we would warn twice.
325 } else if !warnedHardlink {
326 var hardlinkHint string
327 if runtime.GOOS == "linux" && errors.Is(err, syscall.EXDEV) {
328 hardlinkHint = " (hint: if running under systemd, ReadWritePaths in mox.service may cause multiple mountpoints; consider merging paths into a single parent directory to prevent cross-device/mountpoint hardlinks)"
330 xwarnx("creating hardlink to message failed, will be doing regular file copies and not warn again"+hardlinkHint, err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
331 warnedHardlink = true
334 // Fall back to copying.
335 sf, err := os.Open(srcpath)
337 return false, fmt.Errorf("open source path %s: %v", srcpath, err)
341 ctl.log.Check(err, "closing copied source file")
344 df, err := os.OpenFile(dstpath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660)
346 return false, fmt.Errorf("create destination path %s: %v", dstpath, err)
351 ctl.log.Check(err, "closing partial destination file")
354 if _, err := io.Copy(df, sf); err != nil {
355 return false, fmt.Errorf("coping: %v", err)
360 return false, fmt.Errorf("closing destination file: %v", err)
365 // Start making the backup.
366 tmStart := time.Now()
368 ctl.log.Print("making backup", slog.String("destdir", dstDataDir))
370 if err := os.MkdirAll(dstDataDir, 0770); err != nil {
371 xerrx("creating destination data directory", err)
374 if err := os.WriteFile(filepath.Join(dstDataDir, "moxversion"), []byte(moxvar.Version), 0660); err != nil {
375 xerrx("writing moxversion", err)
377 backupDB(store.AuthDB, "auth.db")
378 backupDB(dmarcdb.ReportsDB, "dmarcrpt.db")
379 backupDB(dmarcdb.EvalDB, "dmarceval.db")
380 backupDB(mtastsdb.DB, "mtasts.db")
381 backupDB(tlsrptdb.ReportDB, "tlsrpt.db")
382 backupDB(tlsrptdb.ResultDB, "tlsrptresult.db")
383 backupFile("receivedid.key")
385 // Acme directory is optional.
386 srcAcmeDir := filepath.Join(srcDataDir, "acme")
387 if _, err := os.Stat(srcAcmeDir); err == nil {
389 } else if err != nil && !os.IsNotExist(err) {
390 xerrx("copying acme/", err)
393 // Copy the queue database and all message files.
394 backupQueue := func(path string) {
395 tmQueue := time.Now()
397 if err := backupDB(queue.DB, path); err != nil {
398 xerrx("queue not backed up", err, slog.String("path", path), slog.Duration("duration", time.Since(tmQueue)))
402 dstdbpath := filepath.Join(dstDataDir, path)
403 opts := bstore.Options{MustExist: true, RegisterLogger: ctl.log.Logger}
404 db, err := bstore.Open(ctx, dstdbpath, &opts, queue.DBTypes...)
406 xerrx("open copied queue database", err, slog.String("dstpath", dstdbpath), slog.Duration("duration", time.Since(tmQueue)))
413 ctl.log.Check(err, "closing new queue db")
417 // Link/copy known message files. If a message has been removed while we read the
418 // database, our backup is not consistent and the backup will be marked failed.
420 seen := map[string]struct{}{}
421 var nlinked, ncopied int
423 err = bstore.QueryDB[queue.Msg](ctx, db).ForEach(func(m queue.Msg) error {
427 mp := store.MessagePath(m.ID)
428 seen[mp] = struct{}{}
429 srcpath := filepath.Join(srcDataDir, "queue", mp)
430 dstpath := filepath.Join(dstDataDir, "queue", mp)
431 if linked, err := linkOrCopy(srcpath, dstpath); err != nil {
432 xerrx("linking/copying queue message", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
441 xerrx("processing queue messages (not backed up properly)", err, slog.Duration("duration", time.Since(tmMsgs)))
443 xvlog("queue message files linked/copied",
444 slog.Int("linked", nlinked),
445 slog.Int("copied", ncopied),
446 slog.Duration("duration", time.Since(tmMsgs)))
449 // Read through all files in queue directory and warn about anything we haven't
450 // handled yet. Message files that are newer than we expect from our consistent
451 // database snapshot are ignored.
453 srcqdir := filepath.Join(srcDataDir, "queue")
454 err = filepath.WalkDir(srcqdir, func(srcqpath string, d fs.DirEntry, err error) error {
456 xerrx("walking files in queue", err, slog.String("srcpath", srcqpath))
462 p := srcqpath[len(srcqdir)+1:]
463 if _, ok := seen[p]; ok {
469 // Skip any messages that were added since we started on our consistent snapshot.
470 // We don't want to cause spurious backup warnings.
471 if id, err := strconv.ParseInt(filepath.Base(p), 10, 64); err == nil && maxID > 0 && id > maxID && p == store.MessagePath(id) {
475 qp := filepath.Join("queue", p)
476 xwarnx("backing up unrecognized file in queue directory", nil, slog.String("path", qp))
481 xerrx("walking queue directory (not backed up properly)", err, slog.String("dir", "queue"), slog.Duration("duration", time.Since(tmWalk)))
483 xvlog("walked queue directory", slog.Duration("duration", time.Since(tmWalk)))
486 xvlog("queue backed finished", slog.Duration("duration", time.Since(tmQueue)))
488 backupQueue(filepath.FromSlash("queue/index.db"))
490 backupAccount := func(acc *store.Account) {
493 tmAccount := time.Now()
495 // Copy database file.
496 dbpath := filepath.Join("accounts", acc.Name, "index.db")
497 err := backupDB(acc.DB, dbpath)
499 xerrx("copying account database", err, slog.String("path", dbpath), slog.Duration("duration", time.Since(tmAccount)))
502 // todo: should document/check not taking a rlock on account.
504 // Copy junkfilter files, if configured.
505 if jf, _, err := acc.OpenJunkFilter(ctx, ctl.log); err != nil {
506 if !errors.Is(err, store.ErrNoJunkFilter) {
507 xerrx("opening junk filter for account (not backed up)", err)
511 jfpath := filepath.Join("accounts", acc.Name, "junkfilter.db")
513 bloompath := filepath.Join("accounts", acc.Name, "junkfilter.bloom")
514 backupFile(bloompath)
517 ctl.log.Check(err, "closing junkfilter")
520 dstdbpath := filepath.Join(dstDataDir, dbpath)
521 opts := bstore.Options{MustExist: true, RegisterLogger: ctl.log.Logger}
522 db, err := bstore.Open(ctx, dstdbpath, &opts, store.DBTypes...)
524 xerrx("open copied account database", err, slog.String("dstpath", dstdbpath), slog.Duration("duration", time.Since(tmAccount)))
531 ctl.log.Check(err, "close account database")
535 // Link/copy known message files. If a message has been removed while we read the
536 // database, our backup is not consistent and the backup will be marked failed.
538 seen := map[string]struct{}{}
540 var nlinked, ncopied int
541 err = bstore.QueryDB[store.Message](ctx, db).FilterEqual("Expunged", false).ForEach(func(m store.Message) error {
545 mp := store.MessagePath(m.ID)
546 seen[mp] = struct{}{}
547 amp := filepath.Join("accounts", acc.Name, "msg", mp)
548 srcpath := filepath.Join(srcDataDir, amp)
549 dstpath := filepath.Join(dstDataDir, amp)
550 if linked, err := linkOrCopy(srcpath, dstpath); err != nil {
551 xerrx("linking/copying account message", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
560 xerrx("processing account messages (not backed up properly)", err, slog.Duration("duration", time.Since(tmMsgs)))
562 xvlog("account message files linked/copied",
563 slog.Int("linked", nlinked),
564 slog.Int("copied", ncopied),
565 slog.Duration("duration", time.Since(tmMsgs)))
568 // Read through all files in queue directory and warn about anything we haven't
569 // handled yet. Message files that are newer than we expect from our consistent
570 // database snapshot are ignored.
572 srcadir := filepath.Join(srcDataDir, "accounts", acc.Name)
573 err = filepath.WalkDir(srcadir, func(srcapath string, d fs.DirEntry, err error) error {
575 xerrx("walking files in account", err, slog.String("srcpath", srcapath))
581 p := srcapath[len(srcadir)+1:]
582 l := strings.Split(p, string(filepath.Separator))
584 mp := filepath.Join(l[1:]...)
585 if _, ok := seen[mp]; ok {
589 // Skip any messages that were added since we started on our consistent snapshot.
590 // We don't want to cause spurious backup warnings.
591 if id, err := strconv.ParseInt(l[len(l)-1], 10, 64); err == nil && id > maxID && mp == store.MessagePath(id) {
596 case "index.db", "junkfilter.db", "junkfilter.bloom":
599 ap := filepath.Join("accounts", acc.Name, p)
600 if strings.HasPrefix(p, "msg"+string(filepath.Separator)) {
601 xwarnx("backing up unrecognized file in account message directory (should be moved away)", nil, slog.String("path", ap))
603 xwarnx("backing up unrecognized file in account directory", nil, slog.String("path", ap))
609 xerrx("walking account directory (not backed up properly)", err, slog.String("srcdir", srcadir), slog.Duration("duration", time.Since(tmWalk)))
611 xvlog("walked account directory", slog.Duration("duration", time.Since(tmWalk)))
614 xvlog("account backup finished", slog.String("dir", filepath.Join("accounts", acc.Name)), slog.Duration("duration", time.Since(tmAccount)))
617 // For each configured account, open it, make a copy of the database and
618 // hardlink/copy the messages. We track the accounts we handled, and skip the
619 // account directories when handling "all other files" below.
620 accounts := map[string]struct{}{}
621 for _, accName := range mox.Conf.Accounts() {
622 acc, err := store.OpenAccount(ctl.log, accName, false)
624 xerrx("opening account for copying (will try to copy as regular files later)", err, slog.String("account", accName))
627 accounts[accName] = struct{}{}
631 // Copy all other files, that aren't part of the known files, databases, queue or accounts.
633 err = filepath.WalkDir(srcDataDir, func(srcpath string, d fs.DirEntry, err error) error {
635 xerrx("walking path", err, slog.String("path", srcpath))
639 if srcpath == srcDataDir {
642 p := srcpath[len(srcDataDir)+1:]
643 if p == "queue" || p == "acme" || p == "tmp" {
646 l := strings.Split(p, string(filepath.Separator))
647 if len(l) >= 2 && l[0] == "accounts" {
649 if _, ok := accounts[name]; ok {
654 // Only files are explicitly backed up.
660 case "auth.db", "dmarcrpt.db", "dmarceval.db", "mtasts.db", "tlsrpt.db", "tlsrptresult.db", "receivedid.key", "ctl":
663 case "lastknownversion": // Optional file, not yet handled.
665 xwarnx("backing up unrecognized file", nil, slog.String("path", p))
671 xerrx("walking other files (not backed up properly)", err, slog.Duration("duration", time.Since(tmWalk)))
673 xvlog("walking other files finished", slog.Duration("duration", time.Since(tmWalk)))
676 xvlog("backup finished", slog.Duration("duration", time.Since(tmStart)))
681 ctl.xwrite("errors were encountered during backup")