1package main

2

3import (

4 "context"

5 "errors"

6 "fmt"

7 "io/fs"

8 "log"

9 "os"

10 "path/filepath"

11 "slices"

12 "strconv"

13 "strings"

14

15 bolt "go.etcd.io/bbolt"

16

17 "github.com/mjl-/bstore"

18

19 "github.com/mjl-/mox/dmarcdb"

20 "github.com/mjl-/mox/junk"

21 "github.com/mjl-/mox/moxvar"

22 "github.com/mjl-/mox/mtastsdb"

23 "github.com/mjl-/mox/queue"

24 "github.com/mjl-/mox/store"

25 "github.com/mjl-/mox/tlsrptdb"

26)

27

28func cmdVerifydata(c *cmd) {

29 c.params = "data-dir"

30 c.help = `Verify the contents of a data directory, typically of a backup.

31

32Verifydata checks all database files to see if they are valid BoltDB/bstore

33databases. It checks that all messages in the database have a corresponding

34on-disk message file and there are no unrecognized files. If option -fix is

35specified, unrecognized message files are moved away. This may be needed after

36a restore, because messages enqueued or delivered in the future may get those

37message sequence numbers assigned and writing the message file would fail.

38Consistency of message/mailbox UID, UIDNEXT and UIDVALIDITY is verified as

39well.

40

41Because verifydata opens the database files, schema upgrades may automatically

42be applied. This can happen if you use a new mox release. It is useful to run

43"mox verifydata" with a new binary before attempting an upgrade, but only on a

44copy of the database files, as made with "mox backup". Before upgrading, make a

45new backup again since "mox verifydata" may have upgraded the database files,

46possibly making them potentially no longer readable by the previous version.

47`

48 var fix bool

49 c.flag.BoolVar(&fix, "fix", false, "fix fixable problems, such as moving away message files not referenced by their database")

50

51 // To prevent aborting the upgrade test with v0.0.[45] that had a message with

52 // incorrect Size.

53 var skipSizeCheck bool

54 c.flag.BoolVar(&skipSizeCheck, "skip-size-check", false, "skip the check for message size")

55

56 args := c.Parse()

57 if len(args) != 1 {

58 c.Usage()

59 }

60

61 dataDir := filepath.Clean(args[0])

62

63 ctxbg := context.Background()

64

65 // Check whether file exists, or rather, that it doesn't not exist. Other errors

66 // will return true as well, so the triggered check can give the details.

67 exists := func(path string) bool {

68 _, err := os.Stat(path)

69 return err == nil || !os.IsNotExist(err)

70 }

71

72 // Check for error. If so, write a log line, including the path, and set fail so we

73 // can warn at the end.

74 var fail bool

75 checkf := func(err error, path, format string, args ...any) {

76 if err == nil {

77 return

78 }

79 fail = true

80 log.Printf("error: %s: %s: %v", path, fmt.Sprintf(format, args...), err)

81 }

82

83 // When we fix problems, we may have to move files/dirs. We need to ensure the

84 // directory of the destination path exists before we move. We keep track of

85 // created dirs so we don't try to create the same directory all the time.

86 createdDirs := map[string]struct{}{}

87 ensureDir := func(path string) {

88 dir := filepath.Dir(path)

89 if _, ok := createdDirs[dir]; ok {

90 return

91 }

92 err := os.MkdirAll(dir, 0770)

93 checkf(err, dir, "creating directory")

94 createdDirs[dir] = struct{}{}

95 }

96

97 // Check a database file by opening it with BoltDB and bstore and lightly checking

98 // its contents.

99 checkDB := func(required bool, path string, types []any) {

100 _, err := os.Stat(path)

101 if !required && err != nil && errors.Is(err, fs.ErrNotExist) {

102 return

103 }

104 checkf(err, path, "checking if database file exists")

105 if err != nil {

106 return

107 }

108 bdb, err := bolt.Open(path, 0600, nil)

109 checkf(err, path, "open database with bolt")

110 if err != nil {

111 return

112 }

113 // Check BoltDB consistency.

114 err = bdb.View(func(tx *bolt.Tx) error {

115 for err := range tx.Check() {

116 checkf(err, path, "bolt database problem")

117 }

118 return nil

119 })

120 checkf(err, path, "reading bolt database")

121 bdb.Close()

122

123 db, err := bstore.Open(ctxbg, path, nil, types...)

124 checkf(err, path, "open database with bstore")

125 if err != nil {

126 return

127 }

128 defer db.Close()

129

130 err = db.Read(ctxbg, func(tx *bstore.Tx) error {

131 // Check bstore consistency, if it can export all records for all types. This is a

132 // quick way to get bstore to parse all records.

133 types, err := tx.Types()

134 checkf(err, path, "getting bstore types from database")

135 if err != nil {

136 return nil

137 }

138 for _, t := range types {

139 var fields []string

140 err := tx.Records(t, &fields, func(m map[string]any) error {

141 return nil

142 })

143 checkf(err, path, "parsing record for type %q", t)

144 }

145 return nil

146 })

147 checkf(err, path, "checking database file")

148 }

149

150 checkFile := func(dbpath, path string, prefixSize int, size int64) {

151 st, err := os.Stat(path)

152 checkf(err, path, "checking if file exists")

153 if !skipSizeCheck && err == nil && int64(prefixSize)+st.Size() != size {

154 filesize := st.Size()

155 checkf(fmt.Errorf("%s: message size is %d, should be %d (length of MsgPrefix %d + file size %d), see \"mox fixmsgsize\"", path, size, int64(prefixSize)+st.Size(), prefixSize, filesize), dbpath, "checking message size")

156 }

157 }

158

159 checkQueue := func() {

160 dbpath := filepath.Join(dataDir, "queue/index.db")

161 checkDB(true, dbpath, queue.DBTypes)

162

163 // Check that all messages present in the database also exist on disk.

164 seen := map[string]struct{}{}

165 db, err := bstore.Open(ctxbg, dbpath, &bstore.Options{MustExist: true}, queue.DBTypes...)

166 checkf(err, dbpath, "opening queue database to check messages")

167 if err == nil {

168 err := bstore.QueryDB[queue.Msg](ctxbg, db).ForEach(func(m queue.Msg) error {

169 mp := store.MessagePath(m.ID)

170 seen[mp] = struct{}{}

171 p := filepath.Join(dataDir, "queue", mp)

172 checkFile(dbpath, p, len(m.MsgPrefix), m.Size)

173 return nil

174 })

175 checkf(err, dbpath, "reading messages in queue database to check files")

176 }

177

178 // Check that there are no files that could be treated as a message.

179 qdir := filepath.Join(dataDir, "queue")

180 err = filepath.WalkDir(qdir, func(qpath string, d fs.DirEntry, err error) error {

181 checkf(err, qpath, "walk")

182 if err != nil {

183 return nil

184 }

185 if d.IsDir() {

186 return nil

187 }

188 p := qpath[len(qdir)+1:]

189 if p == "index.db" {

190 return nil

191 }

192 if _, ok := seen[p]; ok {

193 return nil

194 }

195 l := strings.Split(p, string(filepath.Separator))

196 if len(l) == 1 {

197 log.Printf("warning: %s: unrecognized file in queue directory, ignoring", qpath)

198 return nil

199 }

200 // If it doesn't look like a message number, there is no risk of it being the name

201 // of a message enqueued in the future.

202 if len(l) >= 3 {

203 if _, err := strconv.ParseInt(l[1], 10, 64); err != nil {

204 log.Printf("warning: %s: unrecognized file in queue directory, ignoring", qpath)

205 return nil

206 }

207 }

208 if !fix {

209 checkf(errors.New("may interfere with messages enqueued in the future"), qpath, "unrecognized file in queue directory (use the -fix flag to move it away)")

210 return nil

211 }

212 npath := filepath.Join(dataDir, "moved", "queue", p)

213 ensureDir(npath)

214 err = os.Rename(qpath, npath)

215 checkf(err, qpath, "moving queue message file away")

216 if err == nil {

217 log.Printf("warning: moved %s to %s", qpath, npath)

218 }

219 return nil

220 })

221 checkf(err, qdir, "walking queue directory")

222 }

223

224 // Check an account, with its database file and messages.

225 checkAccount := func(name string) {

226 accdir := filepath.Join(dataDir, "accounts", name)

227 checkDB(true, filepath.Join(accdir, "index.db"), store.DBTypes)

228

229 jfdbpath := filepath.Join(accdir, "junkfilter.db")

230 jfbloompath := filepath.Join(accdir, "junkfilter.bloom")

231 if exists(jfdbpath) || exists(jfbloompath) {

232 checkDB(true, jfdbpath, junk.DBTypes)

233 }

234 // todo: add some kind of check for the bloom filter?

235

236 // Check that all messages in the database have a message file on disk.

237 // And check consistency of UIDs with the mailbox UIDNext, and check UIDValidity.

238 seen := map[string]struct{}{}

239 dbpath := filepath.Join(accdir, "index.db")

240 db, err := bstore.Open(ctxbg, dbpath, &bstore.Options{MustExist: true}, store.DBTypes...)

241 checkf(err, dbpath, "opening account database to check messages")

242 if err == nil {

243 uidvalidity := store.NextUIDValidity{ID: 1}

244 if err := db.Get(ctxbg, &uidvalidity); err != nil {

245 checkf(err, dbpath, "missing nextuidvalidity")

246 }

247

248 up := store.Upgrade{ID: 1}

249 if err := db.Get(ctxbg, &up); err != nil {

250 log.Printf("warning: %s: getting upgrade record (continuing, but not checking message threading): %v", dbpath, err)

251 } else if up.Threads != 2 {

252 log.Printf("warning: %s: no message threading in database, skipping checks for threading consistency", dbpath)

253 }

254

255 mailboxes := map[int64]store.Mailbox{}

256 err := bstore.QueryDB[store.Mailbox](ctxbg, db).ForEach(func(mb store.Mailbox) error {

257 mailboxes[mb.ID] = mb

258

259 if mb.UIDValidity >= uidvalidity.Next {

260 checkf(errors.New(`inconsistent uidvalidity for mailbox/account, see "mox fixuidmeta"`), dbpath, "mailbox %q (id %d) has uidvalidity %d >= account nextuidvalidity %d", mb.Name, mb.ID, mb.UIDValidity, uidvalidity.Next)

261 }

262 return nil

263 })

264 checkf(err, dbpath, "reading mailboxes to check uidnext consistency")

265

266 mbCounts := map[int64]store.MailboxCounts{}

267 var totalSize int64

268 err = bstore.QueryDB[store.Message](ctxbg, db).ForEach(func(m store.Message) error {

269 mb := mailboxes[m.MailboxID]

270 if m.UID >= mb.UIDNext {

271 checkf(errors.New(`inconsistent uidnext for message/mailbox, see "mox fixuidmeta"`), dbpath, "message id %d in mailbox %q (id %d) has uid %d >= mailbox uidnext %d", m.ID, mb.Name, mb.ID, m.UID, mb.UIDNext)

272 }

273

274 if m.ModSeq < m.CreateSeq {

275 checkf(errors.New(`inconsistent modseq/createseq for message`), dbpath, "message id %d in mailbox %q (id %d) has modseq %d < createseq %d", m.ID, mb.Name, mb.ID, m.ModSeq, m.CreateSeq)

276 }

277

278 mc := mbCounts[mb.ID]

279 mc.Add(m.MailboxCounts())

280 mbCounts[mb.ID] = mc

281

282 if m.Expunged {

283 return nil

284 }

285 totalSize += m.Size

286

287 mp := store.MessagePath(m.ID)

288 seen[mp] = struct{}{}

289 p := filepath.Join(accdir, "msg", mp)

290 checkFile(dbpath, p, len(m.MsgPrefix), m.Size)

291

292 if up.Threads != 2 {

293 return nil

294 }

295

296 if m.ThreadID <= 0 {

297 checkf(errors.New(`see "mox reassignthreads"`), dbpath, "message id %d, thread %d in mailbox %q (id %d) has bad threadid", m.ID, m.ThreadID, mb.Name, mb.ID)

298 }

299 if len(m.ThreadParentIDs) == 0 {

300 return nil

301 }

302 if slices.Contains(m.ThreadParentIDs, m.ID) {

303 checkf(errors.New(`see "mox reassignthreads"`), dbpath, "message id %d, thread %d in mailbox %q (id %d) has itself as thread parent", m.ID, m.ThreadID, mb.Name, mb.ID)

304 }

305 for i, pid := range m.ThreadParentIDs {

306 am := store.Message{ID: pid}

307 if err := db.Get(ctxbg, &am); err == bstore.ErrAbsent {

308 continue

309 } else if err != nil {

310 return fmt.Errorf("get ancestor message: %v", err)

311 } else if !slices.Equal(m.ThreadParentIDs[i+1:], am.ThreadParentIDs) {

312 checkf(errors.New(`see "mox reassignthreads"`), dbpath, "message %d, thread %d has ancestor ids %v, and ancestor at index %d with id %d should have the same tail but has %v", m.ID, m.ThreadID, m.ThreadParentIDs, i, am.ID, am.ThreadParentIDs)

313 } else {

314 break

315 }

316 }

317 return nil

318 })

319 checkf(err, dbpath, "reading messages in account database to check files")

320

321 haveCounts := true

322 for _, mb := range mailboxes {

323 // We only check if database doesn't have zero values, i.e. not yet set.

324 if !mb.HaveCounts {

325 haveCounts = false

326 }

327 if mb.HaveCounts && mb.MailboxCounts != mbCounts[mb.ID] {

328 checkf(errors.New(`wrong mailbox counts, see "mox recalculatemailboxcounts"`), dbpath, "mailbox %q (id %d) has wrong counts %s, should be %s", mb.Name, mb.ID, mb.MailboxCounts, mbCounts[mb.ID])

329 }

330 }

331

332 if haveCounts {

333 du := store.DiskUsage{ID: 1}

334 err := db.Get(ctxbg, &du)

335 if err == nil {

336 if du.MessageSize != totalSize {

337 checkf(errors.New(`wrong total message size, see mox recalculatemailboxcounts"`), dbpath, "account has wrong total message size %d, should be %d", du.MessageSize, totalSize)

338 }

339 } else if err != nil && !errors.Is(err, bstore.ErrAbsent) {

340 checkf(err, dbpath, "get disk usage")

341 }

342 }

343 }

344

345 // Walk through all files in the msg directory. Warn about files that weren't in

346 // the database as message file. Possibly move away files that could cause trouble.

347 msgdir := filepath.Join(accdir, "msg")

348 if !exists(msgdir) {

349 // New accounts with messages don't have a msg directory.

350 return

351 }

352 err = filepath.WalkDir(msgdir, func(msgpath string, d fs.DirEntry, err error) error {

353 checkf(err, msgpath, "walk")

354 if err != nil {

355 return nil

356 }

357 if d.IsDir() {

358 return nil

359 }

360 p := msgpath[len(msgdir)+1:]

361 if _, ok := seen[p]; ok {

362 return nil

363 }

364 l := strings.Split(p, string(filepath.Separator))

365 if len(l) == 1 {

366 log.Printf("warning: %s: unrecognized file in message directory, ignoring", msgpath)

367 return nil

368 }

369 if !fix {

370 checkf(errors.New("may interfere with future account messages"), msgpath, "unrecognized file in account message directory (use the -fix flag to move it away)")

371 return nil

372 }

373 npath := filepath.Join(dataDir, "moved", "accounts", name, "msg", p)

374 ensureDir(npath)

375 err = os.Rename(msgpath, npath)

376 checkf(err, msgpath, "moving account message file away")

377 if err == nil {

378 log.Printf("warning: moved %s to %s", msgpath, npath)

379 }

380 return nil

381 })

382 checkf(err, msgdir, "walking account message directory")

383 }

384

385 // Check everything in the "accounts" directory.

386 checkAccounts := func() {

387 accountsDir := filepath.Join(dataDir, "accounts")

388 entries, err := os.ReadDir(accountsDir)

389 checkf(err, accountsDir, "reading accounts directory")

390 for _, e := range entries {

391 // We treat all directories as accounts. When we were backing up, we only verified

392 // accounts from the config and made regular file copies of all other files

393 // (perhaps an old account, but at least not with an open database file). It may

394 // turn out that that account was/is not valid, generating warnings. Better safe

395 // than sorry. It should hopefully get the admin to move away such an old account.

396 if e.IsDir() {

397 checkAccount(e.Name())

398 } else {

399 log.Printf("warning: %s: unrecognized file in accounts directory, ignoring", filepath.Join("accounts", e.Name()))

400 }

401 }

402 }

403

404 // Check all files, skipping the known files, queue and accounts directories. Warn

405 // about unknown files. Skip a "tmp" directory. And a "moved" directory, we

406 // probably created it ourselves.

407 backupmoxversion := "(unknown)"

408 checkOther := func() {

409 err := filepath.WalkDir(dataDir, func(dpath string, d fs.DirEntry, err error) error {

410 checkf(err, dpath, "walk")

411 if err != nil {

412 return nil

413 }

414 if dpath == dataDir {

415 return nil

416 }

417 p := dpath

418 if dataDir != "." {

419 p = p[len(dataDir)+1:]

420 }

421 switch p {

422 case "dmarcrpt.db", "dmarceval.db", "mtasts.db", "tlsrpt.db", "tlsrptresult.db", "receivedid.key", "lastknownversion":

423 return nil

424 case "acme", "queue", "accounts", "tmp", "moved":

425 return fs.SkipDir

426 case "moxversion":

427 buf, err := os.ReadFile(dpath)

428 checkf(err, dpath, "reading moxversion")

429 if err == nil {

430 backupmoxversion = string(buf)

431 }

432 return nil

433 }

434 log.Printf("warning: %s: unrecognized other file, ignoring", dpath)

435 return nil

436 })

437 checkf(err, dataDir, "walking data directory")

438 }

439

440 checkDB(true, filepath.Join(dataDir, "dmarcrpt.db"), dmarcdb.ReportsDBTypes)

441 checkDB(false, filepath.Join(dataDir, "dmarceval.db"), dmarcdb.EvalDBTypes) // After v0.0.7.

442 checkDB(true, filepath.Join(dataDir, "mtasts.db"), mtastsdb.DBTypes)

443 checkDB(true, filepath.Join(dataDir, "tlsrpt.db"), tlsrptdb.ReportDBTypes)

444 checkDB(false, filepath.Join(dataDir, "tlsrptresult.db"), tlsrptdb.ResultDBTypes) // After v0.0.7.

445 checkQueue()

446 checkAccounts()

447 checkOther()

448

449 if backupmoxversion != moxvar.Version {

450 log.Printf("NOTE: The backup was made with mox version %q, while verifydata was run with mox version %q. Database files have probably been modified by running mox verifydata. Make a fresh backup before upgrading.", backupmoxversion, moxvar.Version)

451 }

452

453 if fail {

454 log.Fatalf("errors were found")

455 } else {

456 fmt.Printf("%s: OK\n", dataDir)

457 }

458}

459