1package main

2

3import (

4 "context"

5 "errors"

6 "fmt"

7 "io/fs"

8 "log"

9 "os"

10 "path/filepath"

11 "slices"

12 "strconv"

13 "strings"

14

15 bolt "go.etcd.io/bbolt"

16

17 "github.com/mjl-/bstore"

18

19 "github.com/mjl-/mox/dmarcdb"

20 "github.com/mjl-/mox/junk"

21 "github.com/mjl-/mox/moxvar"

22 "github.com/mjl-/mox/mtastsdb"

23 "github.com/mjl-/mox/queue"

24 "github.com/mjl-/mox/store"

25 "github.com/mjl-/mox/tlsrptdb"

26)

27

28func cmdVerifydata(c *cmd) {

29 c.params = "data-dir"

30 c.help = `Verify the contents of a data directory, typically of a backup.

31

32Verifydata checks all database files to see if they are valid BoltDB/bstore

33databases. It checks that all messages in the database have a corresponding

34on-disk message file and there are no unrecognized files. If option -fix is

35specified, unrecognized message files are moved away. This may be needed after

36a restore, because messages enqueued or delivered in the future may get those

37message sequence numbers assigned and writing the message file would fail.

38Consistency of message/mailbox UID, UIDNEXT and UIDVALIDITY is verified as

39well.

40

41Because verifydata opens the database files, schema upgrades may automatically

42be applied. This can happen if you use a new mox release. It is useful to run

43"mox verifydata" with a new binary before attempting an upgrade, but only on a

44copy of the database files, as made with "mox backup". Before upgrading, make a

45new backup again since "mox verifydata" may have upgraded the database files,

46possibly making them potentially no longer readable by the previous version.

47`

48 var fix bool

49 c.flag.BoolVar(&fix, "fix", false, "fix fixable problems, such as moving away message files not referenced by their database")

50

51 // To prevent aborting the upgrade test with v0.0.[45] that had a message with

52 // incorrect Size.

53 var skipSizeCheck bool

54 c.flag.BoolVar(&skipSizeCheck, "skip-size-check", false, "skip the check for message size")

55

56 args := c.Parse()

57 if len(args) != 1 {

58 c.Usage()

59 }

60

61 dataDir := filepath.Clean(args[0])

62

63 ctxbg := context.Background()

64

65 // Check whether file exists, or rather, that it doesn't not exist. Other errors

66 // will return true as well, so the triggered check can give the details.

67 exists := func(path string) bool {

68 _, err := os.Stat(path)

69 return err == nil || !os.IsNotExist(err)

70 }

71

72 // Check for error. If so, write a log line, including the path, and set fail so we

73 // can warn at the end.

74 var fail bool

75 checkf := func(err error, path, format string, args ...any) {

76 if err == nil {

77 return

78 }

79 fail = true

80 log.Printf("error: %s: %s: %v", path, fmt.Sprintf(format, args...), err)

81 }

82

83 // When we fix problems, we may have to move files/dirs. We need to ensure the

84 // directory of the destination path exists before we move. We keep track of

85 // created dirs so we don't try to create the same directory all the time.

86 createdDirs := map[string]struct{}{}

87 ensureDir := func(path string) {

88 dir := filepath.Dir(path)

89 if _, ok := createdDirs[dir]; ok {

90 return

91 }

92 err := os.MkdirAll(dir, 0770)

93 checkf(err, dir, "creating directory")

94 createdDirs[dir] = struct{}{}

95 }

96

97 // Check a database file by opening it with BoltDB and bstore and lightly checking

98 // its contents.

99 checkDB := func(required bool, path string, types []any) {

100 _, err := os.Stat(path)

101 if !required && err != nil && errors.Is(err, fs.ErrNotExist) {

102 return

103 }

104 checkf(err, path, "checking if database file exists")

105 if err != nil {

106 return

107 }

108 bdb, err := bolt.Open(path, 0600, nil)

109 checkf(err, path, "open database with bolt")

110 if err != nil {

111 return

112 }

113 // Check BoltDB consistency.

114 err = bdb.View(func(tx *bolt.Tx) error {

115 for err := range tx.Check() {

116 checkf(err, path, "bolt database problem")

117 }

118 return nil

119 })

120 checkf(err, path, "reading bolt database")

121 if err := bdb.Close(); err != nil {

122 log.Printf("closing database file: %v", err)

123 }

124

125 opts := bstore.Options{RegisterLogger: c.log.Logger}

126 db, err := bstore.Open(ctxbg, path, &opts, types...)

127 checkf(err, path, "open database with bstore")

128 if err != nil {

129 return

130 }

131 defer func() {

132 if err := db.Close(); err != nil {

133 log.Printf("closing database file: %v", err)

134 }

135 }()

136

137 err = db.Read(ctxbg, func(tx *bstore.Tx) error {

138 // Check bstore consistency, if it can export all records for all types. This is a

139 // quick way to get bstore to parse all records.

140 types, err := tx.Types()

141 checkf(err, path, "getting bstore types from database")

142 if err != nil {

143 return nil

144 }

145 for _, t := range types {

146 var fields []string

147 err := tx.Records(t, &fields, func(m map[string]any) error {

148 return nil

149 })

150 checkf(err, path, "parsing record for type %q", t)

151 }

152 return nil

153 })

154 checkf(err, path, "checking database file")

155 }

156

157 checkFile := func(dbpath, path string, prefixSize int, size int64) {

158 st, err := os.Stat(path)

159 checkf(err, path, "checking if file exists")

160 if !skipSizeCheck && err == nil && int64(prefixSize)+st.Size() != size {

161 filesize := st.Size()

162 checkf(fmt.Errorf("%s: message size is %d, should be %d (length of MsgPrefix %d + file size %d), see \"mox fixmsgsize\"", path, size, int64(prefixSize)+st.Size(), prefixSize, filesize), dbpath, "checking message size")

163 }

164 }

165

166 checkQueue := func() {

167 dbpath := filepath.Join(dataDir, "queue/index.db")

168 checkDB(true, dbpath, queue.DBTypes)

169

170 // Check that all messages present in the database also exist on disk.

171 seen := map[string]struct{}{}

172 opts := bstore.Options{MustExist: true, RegisterLogger: c.log.Logger}

173 db, err := bstore.Open(ctxbg, dbpath, &opts, queue.DBTypes...)

174 checkf(err, dbpath, "opening queue database to check messages")

175 if err == nil {

176 err := bstore.QueryDB[queue.Msg](ctxbg, db).ForEach(func(m queue.Msg) error {

177 mp := store.MessagePath(m.ID)

178 seen[mp] = struct{}{}

179 p := filepath.Join(dataDir, "queue", mp)

180 checkFile(dbpath, p, len(m.MsgPrefix), m.Size)

181 return nil

182 })

183 checkf(err, dbpath, "reading messages in queue database to check files")

184 }

185

186 // Check that there are no files that could be treated as a message.

187 qdir := filepath.Join(dataDir, "queue")

188 err = filepath.WalkDir(qdir, func(qpath string, d fs.DirEntry, err error) error {

189 checkf(err, qpath, "walk")

190 if err != nil {

191 return nil

192 }

193 if d.IsDir() {

194 return nil

195 }

196 p := qpath[len(qdir)+1:]

197 if p == "index.db" {

198 return nil

199 }

200 if _, ok := seen[p]; ok {

201 return nil

202 }

203 l := strings.Split(p, string(filepath.Separator))

204 if len(l) == 1 {

205 log.Printf("warning: %s: unrecognized file in queue directory, ignoring", qpath)

206 return nil

207 }

208 // If it doesn't look like a message number, there is no risk of it being the name

209 // of a message enqueued in the future.

210 if len(l) >= 3 {

211 if _, err := strconv.ParseInt(l[1], 10, 64); err != nil {

212 log.Printf("warning: %s: unrecognized file in queue directory, ignoring", qpath)

213 return nil

214 }

215 }

216 if !fix {

217 checkf(errors.New("may interfere with messages enqueued in the future"), qpath, "unrecognized file in queue directory (use the -fix flag to move it away)")

218 return nil

219 }

220 npath := filepath.Join(dataDir, "moved", "queue", p)

221 ensureDir(npath)

222 err = os.Rename(qpath, npath)

223 checkf(err, qpath, "moving queue message file away")

224 if err == nil {

225 log.Printf("warning: moved %s to %s", qpath, npath)

226 }

227 return nil

228 })

229 checkf(err, qdir, "walking queue directory")

230 }

231

232 // Check an account, with its database file and messages.

233 checkAccount := func(name string) {

234 accdir := filepath.Join(dataDir, "accounts", name)

235 checkDB(true, filepath.Join(accdir, "index.db"), store.DBTypes)

236

237 jfdbpath := filepath.Join(accdir, "junkfilter.db")

238 jfbloompath := filepath.Join(accdir, "junkfilter.bloom")

239 if exists(jfdbpath) || exists(jfbloompath) {

240 checkDB(true, jfdbpath, junk.DBTypes)

241 }

242 // todo: add some kind of check for the bloom filter?

243

244 // Check that all messages in the database have a message file on disk.

245 // And check consistency of UIDs with the mailbox UIDNext, and check UIDValidity.

246 seen := map[string]struct{}{}

247 dbpath := filepath.Join(accdir, "index.db")

248 opts := bstore.Options{MustExist: true, RegisterLogger: c.log.Logger}

249 db, err := bstore.Open(ctxbg, dbpath, &opts, store.DBTypes...)

250 checkf(err, dbpath, "opening account database to check messages")

251 if err == nil {

252 uidvalidity := store.NextUIDValidity{ID: 1}

253 if err := db.Get(ctxbg, &uidvalidity); err != nil {

254 checkf(err, dbpath, "missing nextuidvalidity")

255 }

256

257 up := store.Upgrade{ID: 1}

258 if err := db.Get(ctxbg, &up); err != nil {

259 log.Printf("warning: %s: getting upgrade record (continuing, but not checking message threading): %v", dbpath, err)

260 } else if up.Threads != 2 {

261 log.Printf("warning: %s: no message threading in database, skipping checks for threading consistency", dbpath)

262 }

263

264 mailboxes := map[int64]store.Mailbox{}

265 err := bstore.QueryDB[store.Mailbox](ctxbg, db).ForEach(func(mb store.Mailbox) error {

266 mailboxes[mb.ID] = mb

267

268 if mb.UIDValidity >= uidvalidity.Next {

269 checkf(errors.New(`inconsistent uidvalidity for mailbox/account, see "mox fixuidmeta"`), dbpath, "mailbox %q (id %d) has uidvalidity %d >= account nextuidvalidity %d", mb.Name, mb.ID, mb.UIDValidity, uidvalidity.Next)

270 }

271 return nil

272 })

273 checkf(err, dbpath, "reading mailboxes to check uidnext consistency")

274

275 mbCounts := map[int64]store.MailboxCounts{}

276 var totalSize int64

277 err = bstore.QueryDB[store.Message](ctxbg, db).ForEach(func(m store.Message) error {

278 mb := mailboxes[m.MailboxID]

279 if m.UID >= mb.UIDNext {

280 checkf(errors.New(`inconsistent uidnext for message/mailbox, see "mox fixuidmeta"`), dbpath, "message id %d in mailbox %q (id %d) has uid %d >= mailbox uidnext %d", m.ID, mb.Name, mb.ID, m.UID, mb.UIDNext)

281 }

282

283 if m.ModSeq < m.CreateSeq {

284 checkf(errors.New(`inconsistent modseq/createseq for message`), dbpath, "message id %d in mailbox %q (id %d) has modseq %d < createseq %d", m.ID, mb.Name, mb.ID, m.ModSeq, m.CreateSeq)

285 }

286

287 mc := mbCounts[mb.ID]

288 mc.Add(m.MailboxCounts())

289 mbCounts[mb.ID] = mc

290

291 if m.Expunged {

292 return nil

293 }

294 if mb.Expunged {

295 checkf(errors.New("mailbox is expunged but message is not"), dbpath, "message id %d is in expunged mailbox %q (id %d)", m.ID, mb.Name, mb.ID)

296 }

297 totalSize += m.Size

298

299 mp := store.MessagePath(m.ID)

300 seen[mp] = struct{}{}

301 p := filepath.Join(accdir, "msg", mp)

302 checkFile(dbpath, p, len(m.MsgPrefix), m.Size)

303

304 if up.Threads != 2 {

305 return nil

306 }

307

308 if m.ThreadID <= 0 {

309 checkf(errors.New(`see "mox reassignthreads"`), dbpath, "message id %d, thread %d in mailbox %q (id %d) has bad threadid", m.ID, m.ThreadID, mb.Name, mb.ID)

310 }

311 if len(m.ThreadParentIDs) == 0 {

312 return nil

313 }

314 if slices.Contains(m.ThreadParentIDs, m.ID) {

315 checkf(errors.New(`see "mox reassignthreads"`), dbpath, "message id %d, thread %d in mailbox %q (id %d) has itself as thread parent", m.ID, m.ThreadID, mb.Name, mb.ID)

316 }

317 for i, pid := range m.ThreadParentIDs {

318 am := store.Message{ID: pid}

319 if err := db.Get(ctxbg, &am); err == bstore.ErrAbsent || err == nil && am.Expunged {

320 continue

321 } else if err != nil {

322 return fmt.Errorf("get ancestor message: %v", err)

323 } else if !slices.Equal(m.ThreadParentIDs[i+1:], am.ThreadParentIDs) {

324 checkf(errors.New(`see "mox reassignthreads"`), dbpath, "message %d, thread %d has ancestor ids %v, and ancestor at index %d with id %d should have the same tail but has %v", m.ID, m.ThreadID, m.ThreadParentIDs, i, am.ID, am.ThreadParentIDs)

325 } else {

326 break

327 }

328 }

329 return nil

330 })

331 checkf(err, dbpath, "reading messages in account database to check files")

332

333 haveCounts := true

334 for _, mb := range mailboxes {

335 // We only check if database doesn't have zero values, i.e. not yet set.

336 if !mb.HaveCounts {

337 haveCounts = false

338 }

339 if mb.HaveCounts && mb.MailboxCounts != mbCounts[mb.ID] {

340 checkf(errors.New(`wrong mailbox counts, see "mox recalculatemailboxcounts"`), dbpath, "mailbox %q (id %d) has wrong counts %s, should be %s", mb.Name, mb.ID, mb.MailboxCounts, mbCounts[mb.ID])

341 }

342 }

343

344 if haveCounts {

345 du := store.DiskUsage{ID: 1}

346 err := db.Get(ctxbg, &du)

347 if err == nil {

348 if du.MessageSize != totalSize {

349 checkf(errors.New(`wrong total message size, see mox recalculatemailboxcounts"`), dbpath, "account has wrong total message size %d, should be %d", du.MessageSize, totalSize)

350 }

351 } else if err != nil && !errors.Is(err, bstore.ErrAbsent) {

352 checkf(err, dbpath, "get disk usage")

353 }

354 }

355 }

356

357 // Walk through all files in the msg directory. Warn about files that weren't in

358 // the database as message file. Possibly move away files that could cause trouble.

359 msgdir := filepath.Join(accdir, "msg")

360 if !exists(msgdir) {

361 // New accounts with messages don't have a msg directory.

362 return

363 }

364 err = filepath.WalkDir(msgdir, func(msgpath string, d fs.DirEntry, err error) error {

365 checkf(err, msgpath, "walk")

366 if err != nil {

367 return nil

368 }

369 if d.IsDir() {

370 return nil

371 }

372 p := msgpath[len(msgdir)+1:]

373 if _, ok := seen[p]; ok {

374 return nil

375 }

376 l := strings.Split(p, string(filepath.Separator))

377 if len(l) == 1 {

378 log.Printf("warning: %s: unrecognized file in message directory, ignoring", msgpath)

379 return nil

380 }

381 if !fix {

382 checkf(errors.New("may interfere with future account messages"), msgpath, "unrecognized file in account message directory (use the -fix flag to move it away)")

383 return nil

384 }

385 npath := filepath.Join(dataDir, "moved", "accounts", name, "msg", p)

386 ensureDir(npath)

387 err = os.Rename(msgpath, npath)

388 checkf(err, msgpath, "moving account message file away")

389 if err == nil {

390 log.Printf("warning: moved %s to %s", msgpath, npath)

391 }

392 return nil

393 })

394 checkf(err, msgdir, "walking account message directory")

395 }

396

397 // Check everything in the "accounts" directory.

398 checkAccounts := func() {

399 accountsDir := filepath.Join(dataDir, "accounts")

400 entries, err := os.ReadDir(accountsDir)

401 checkf(err, accountsDir, "reading accounts directory")

402 for _, e := range entries {

403 // We treat all directories as accounts. When we were backing up, we only verified

404 // accounts from the config and made regular file copies of all other files

405 // (perhaps an old account, but at least not with an open database file). It may

406 // turn out that that account was/is not valid, generating warnings. Better safe

407 // than sorry. It should hopefully get the admin to move away such an old account.

408 if e.IsDir() {

409 checkAccount(e.Name())

410 } else {

411 log.Printf("warning: %s: unrecognized file in accounts directory, ignoring", filepath.Join("accounts", e.Name()))

412 }

413 }

414 }

415

416 // Check all files, skipping the known files, queue and accounts directories. Warn

417 // about unknown files. Skip a "tmp" directory. And a "moved" directory, we

418 // probably created it ourselves.

419 backupmoxversion := "(unknown)"

420 checkOther := func() {

421 err := filepath.WalkDir(dataDir, func(dpath string, d fs.DirEntry, err error) error {

422 checkf(err, dpath, "walk")

423 if err != nil {

424 return nil

425 }

426 if dpath == dataDir {

427 return nil

428 }

429 p := dpath

430 if dataDir != "." {

431 p = p[len(dataDir)+1:]

432 }

433 switch p {

434 case "auth.db", "dmarcrpt.db", "dmarceval.db", "mtasts.db", "tlsrpt.db", "tlsrptresult.db", "receivedid.key", "lastknownversion":

435 return nil

436 case "acme", "queue", "accounts", "tmp", "moved":

437 return fs.SkipDir

438 case "moxversion":

439 buf, err := os.ReadFile(dpath)

440 checkf(err, dpath, "reading moxversion")

441 if err == nil {

442 backupmoxversion = string(buf)

443 }

444 return nil

445 }

446 log.Printf("warning: %s: unrecognized other file, ignoring", dpath)

447 return nil

448 })

449 checkf(err, dataDir, "walking data directory")

450 }

451

452 checkDB(false, filepath.Join(dataDir, "auth.db"), store.AuthDBTypes) // Since v0.0.14.

453 checkDB(true, filepath.Join(dataDir, "dmarcrpt.db"), dmarcdb.ReportsDBTypes)

454 checkDB(false, filepath.Join(dataDir, "dmarceval.db"), dmarcdb.EvalDBTypes) // After v0.0.7.

455 checkDB(true, filepath.Join(dataDir, "mtasts.db"), mtastsdb.DBTypes)

456 checkDB(true, filepath.Join(dataDir, "tlsrpt.db"), tlsrptdb.ReportDBTypes)

457 checkDB(false, filepath.Join(dataDir, "tlsrptresult.db"), tlsrptdb.ResultDBTypes) // After v0.0.7.

458 checkQueue()

459 checkAccounts()

460 checkOther()

461

462 if backupmoxversion != moxvar.Version {

463 log.Printf("NOTE: The backup was made with mox version %q, while verifydata was run with mox version %q. Database files have probably been modified by running mox verifydata. Make a fresh backup before upgrading.", backupmoxversion, moxvar.Version)

464 }

465

466 if fail {

467 log.Fatalf("errors were found")

468 } else {

469 fmt.Printf("%s: OK\n", dataDir)

470 }

471}

472