1package main

2

3import (

4 "context"

5 "errors"

6 "fmt"

7 "io/fs"

8 "log"

9 "os"

10 "path/filepath"

11 "slices"

12 "strconv"

13 "strings"

14

15 bolt "go.etcd.io/bbolt"

16

17 "github.com/mjl-/bstore"

18

19 "github.com/mjl-/mox/dmarcdb"

20 "github.com/mjl-/mox/junk"

21 "github.com/mjl-/mox/moxvar"

22 "github.com/mjl-/mox/mtastsdb"

23 "github.com/mjl-/mox/queue"

24 "github.com/mjl-/mox/store"

25 "github.com/mjl-/mox/tlsrptdb"

26)

27

28func cmdVerifydata(c *cmd) {

29 c.params = "data-dir"

30 c.help = `Verify the contents of a data directory, typically of a backup.

31

32Verifydata checks all database files to see if they are valid BoltDB/bstore

33databases. It checks that all messages in the database have a corresponding

34on-disk message file and there are no unrecognized files. If option -fix is

35specified, unrecognized message files are moved away. This may be needed after

36a restore, because messages enqueued or delivered in the future may get those

37message sequence numbers assigned and writing the message file would fail.

38Consistency of message/mailbox UID, UIDNEXT and UIDVALIDITY is verified as

39well.

40

41Because verifydata opens the database files, schema upgrades may automatically

42be applied. This can happen if you use a new mox release. It is useful to run

43"mox verifydata" with a new binary before attempting an upgrade, but only on a

44copy of the database files, as made with "mox backup". Before upgrading, make a

45new backup again since "mox verifydata" may have upgraded the database files,

46possibly making them potentially no longer readable by the previous version.

47`

48 var fix bool

49 c.flag.BoolVar(&fix, "fix", false, "fix fixable problems, such as moving away message files not referenced by their database")

50

51 // To prevent aborting the upgrade test with v0.0.[45] that had a message with

52 // incorrect Size.

53 var skipSizeCheck bool

54 c.flag.BoolVar(&skipSizeCheck, "skip-size-check", false, "skip the check for message size")

55

56 args := c.Parse()

57 if len(args) != 1 {

58 c.Usage()

59 }

60

61 dataDir := filepath.Clean(args[0])

62

63 ctxbg := context.Background()

64

65 // Check whether file exists, or rather, that it doesn't not exist. Other errors

66 // will return true as well, so the triggered check can give the details.

67 exists := func(path string) bool {

68 _, err := os.Stat(path)

69 return err == nil || !os.IsNotExist(err)

70 }

71

72 // Check for error. If so, write a log line, including the path, and set fail so we

73 // can warn at the end.

74 var fail bool

75 checkf := func(err error, path, format string, args ...any) {

76 if err == nil {

77 return

78 }

79 fail = true

80 log.Printf("error: %s: %s: %v", path, fmt.Sprintf(format, args...), err)

81 }

82

83 // When we fix problems, we may have to move files/dirs. We need to ensure the

84 // directory of the destination path exists before we move. We keep track of

85 // created dirs so we don't try to create the same directory all the time.

86 createdDirs := map[string]struct{}{}

87 ensureDir := func(path string) {

88 dir := filepath.Dir(path)

89 if _, ok := createdDirs[dir]; ok {

90 return

91 }

92 err := os.MkdirAll(dir, 0770)

93 checkf(err, dir, "creating directory")

94 createdDirs[dir] = struct{}{}

95 }

96

97 // Check a database file by opening it with BoltDB and bstore and lightly checking

98 // its contents.

99 checkDB := func(required bool, path string, types []any) {

100 _, err := os.Stat(path)

101 if !required && err != nil && errors.Is(err, fs.ErrNotExist) {

102 return

103 }

104 checkf(err, path, "checking if database file exists")

105 if err != nil {

106 return

107 }

108 bdb, err := bolt.Open(path, 0600, nil)

109 checkf(err, path, "open database with bolt")

110 if err != nil {

111 return

112 }

113 // Check BoltDB consistency.

114 err = bdb.View(func(tx *bolt.Tx) error {

115 for err := range tx.Check() {

116 checkf(err, path, "bolt database problem")

117 }

118 return nil

119 })

120 checkf(err, path, "reading bolt database")

121 bdb.Close()

122

123 opts := bstore.Options{RegisterLogger: c.log.Logger}

124 db, err := bstore.Open(ctxbg, path, &opts, types...)

125 checkf(err, path, "open database with bstore")

126 if err != nil {

127 return

128 }

129 defer db.Close()

130

131 err = db.Read(ctxbg, func(tx *bstore.Tx) error {

132 // Check bstore consistency, if it can export all records for all types. This is a

133 // quick way to get bstore to parse all records.

134 types, err := tx.Types()

135 checkf(err, path, "getting bstore types from database")

136 if err != nil {

137 return nil

138 }

139 for _, t := range types {

140 var fields []string

141 err := tx.Records(t, &fields, func(m map[string]any) error {

142 return nil

143 })

144 checkf(err, path, "parsing record for type %q", t)

145 }

146 return nil

147 })

148 checkf(err, path, "checking database file")

149 }

150

151 checkFile := func(dbpath, path string, prefixSize int, size int64) {

152 st, err := os.Stat(path)

153 checkf(err, path, "checking if file exists")

154 if !skipSizeCheck && err == nil && int64(prefixSize)+st.Size() != size {

155 filesize := st.Size()

156 checkf(fmt.Errorf("%s: message size is %d, should be %d (length of MsgPrefix %d + file size %d), see \"mox fixmsgsize\"", path, size, int64(prefixSize)+st.Size(), prefixSize, filesize), dbpath, "checking message size")

157 }

158 }

159

160 checkQueue := func() {

161 dbpath := filepath.Join(dataDir, "queue/index.db")

162 checkDB(true, dbpath, queue.DBTypes)

163

164 // Check that all messages present in the database also exist on disk.

165 seen := map[string]struct{}{}

166 opts := bstore.Options{MustExist: true, RegisterLogger: c.log.Logger}

167 db, err := bstore.Open(ctxbg, dbpath, &opts, queue.DBTypes...)

168 checkf(err, dbpath, "opening queue database to check messages")

169 if err == nil {

170 err := bstore.QueryDB[queue.Msg](ctxbg, db).ForEach(func(m queue.Msg) error {

171 mp := store.MessagePath(m.ID)

172 seen[mp] = struct{}{}

173 p := filepath.Join(dataDir, "queue", mp)

174 checkFile(dbpath, p, len(m.MsgPrefix), m.Size)

175 return nil

176 })

177 checkf(err, dbpath, "reading messages in queue database to check files")

178 }

179

180 // Check that there are no files that could be treated as a message.

181 qdir := filepath.Join(dataDir, "queue")

182 err = filepath.WalkDir(qdir, func(qpath string, d fs.DirEntry, err error) error {

183 checkf(err, qpath, "walk")

184 if err != nil {

185 return nil

186 }

187 if d.IsDir() {

188 return nil

189 }

190 p := qpath[len(qdir)+1:]

191 if p == "index.db" {

192 return nil

193 }

194 if _, ok := seen[p]; ok {

195 return nil

196 }

197 l := strings.Split(p, string(filepath.Separator))

198 if len(l) == 1 {

199 log.Printf("warning: %s: unrecognized file in queue directory, ignoring", qpath)

200 return nil

201 }

202 // If it doesn't look like a message number, there is no risk of it being the name

203 // of a message enqueued in the future.

204 if len(l) >= 3 {

205 if _, err := strconv.ParseInt(l[1], 10, 64); err != nil {

206 log.Printf("warning: %s: unrecognized file in queue directory, ignoring", qpath)

207 return nil

208 }

209 }

210 if !fix {

211 checkf(errors.New("may interfere with messages enqueued in the future"), qpath, "unrecognized file in queue directory (use the -fix flag to move it away)")

212 return nil

213 }

214 npath := filepath.Join(dataDir, "moved", "queue", p)

215 ensureDir(npath)

216 err = os.Rename(qpath, npath)

217 checkf(err, qpath, "moving queue message file away")

218 if err == nil {

219 log.Printf("warning: moved %s to %s", qpath, npath)

220 }

221 return nil

222 })

223 checkf(err, qdir, "walking queue directory")

224 }

225

226 // Check an account, with its database file and messages.

227 checkAccount := func(name string) {

228 accdir := filepath.Join(dataDir, "accounts", name)

229 checkDB(true, filepath.Join(accdir, "index.db"), store.DBTypes)

230

231 jfdbpath := filepath.Join(accdir, "junkfilter.db")

232 jfbloompath := filepath.Join(accdir, "junkfilter.bloom")

233 if exists(jfdbpath) || exists(jfbloompath) {

234 checkDB(true, jfdbpath, junk.DBTypes)

235 }

236 // todo: add some kind of check for the bloom filter?

237

238 // Check that all messages in the database have a message file on disk.

239 // And check consistency of UIDs with the mailbox UIDNext, and check UIDValidity.

240 seen := map[string]struct{}{}

241 dbpath := filepath.Join(accdir, "index.db")

242 opts := bstore.Options{MustExist: true, RegisterLogger: c.log.Logger}

243 db, err := bstore.Open(ctxbg, dbpath, &opts, store.DBTypes...)

244 checkf(err, dbpath, "opening account database to check messages")

245 if err == nil {

246 uidvalidity := store.NextUIDValidity{ID: 1}

247 if err := db.Get(ctxbg, &uidvalidity); err != nil {

248 checkf(err, dbpath, "missing nextuidvalidity")

249 }

250

251 up := store.Upgrade{ID: 1}

252 if err := db.Get(ctxbg, &up); err != nil {

253 log.Printf("warning: %s: getting upgrade record (continuing, but not checking message threading): %v", dbpath, err)

254 } else if up.Threads != 2 {

255 log.Printf("warning: %s: no message threading in database, skipping checks for threading consistency", dbpath)

256 }

257

258 mailboxes := map[int64]store.Mailbox{}

259 err := bstore.QueryDB[store.Mailbox](ctxbg, db).ForEach(func(mb store.Mailbox) error {

260 mailboxes[mb.ID] = mb

261

262 if mb.UIDValidity >= uidvalidity.Next {

263 checkf(errors.New(`inconsistent uidvalidity for mailbox/account, see "mox fixuidmeta"`), dbpath, "mailbox %q (id %d) has uidvalidity %d >= account nextuidvalidity %d", mb.Name, mb.ID, mb.UIDValidity, uidvalidity.Next)

264 }

265 return nil

266 })

267 checkf(err, dbpath, "reading mailboxes to check uidnext consistency")

268

269 mbCounts := map[int64]store.MailboxCounts{}

270 var totalSize int64

271 err = bstore.QueryDB[store.Message](ctxbg, db).ForEach(func(m store.Message) error {

272 mb := mailboxes[m.MailboxID]

273 if m.UID >= mb.UIDNext {

274 checkf(errors.New(`inconsistent uidnext for message/mailbox, see "mox fixuidmeta"`), dbpath, "message id %d in mailbox %q (id %d) has uid %d >= mailbox uidnext %d", m.ID, mb.Name, mb.ID, m.UID, mb.UIDNext)

275 }

276

277 if m.ModSeq < m.CreateSeq {

278 checkf(errors.New(`inconsistent modseq/createseq for message`), dbpath, "message id %d in mailbox %q (id %d) has modseq %d < createseq %d", m.ID, mb.Name, mb.ID, m.ModSeq, m.CreateSeq)

279 }

280

281 mc := mbCounts[mb.ID]

282 mc.Add(m.MailboxCounts())

283 mbCounts[mb.ID] = mc

284

285 if m.Expunged {

286 return nil

287 }

288 totalSize += m.Size

289

290 mp := store.MessagePath(m.ID)

291 seen[mp] = struct{}{}

292 p := filepath.Join(accdir, "msg", mp)

293 checkFile(dbpath, p, len(m.MsgPrefix), m.Size)

294

295 if up.Threads != 2 {

296 return nil

297 }

298

299 if m.ThreadID <= 0 {

300 checkf(errors.New(`see "mox reassignthreads"`), dbpath, "message id %d, thread %d in mailbox %q (id %d) has bad threadid", m.ID, m.ThreadID, mb.Name, mb.ID)

301 }

302 if len(m.ThreadParentIDs) == 0 {

303 return nil

304 }

305 if slices.Contains(m.ThreadParentIDs, m.ID) {

306 checkf(errors.New(`see "mox reassignthreads"`), dbpath, "message id %d, thread %d in mailbox %q (id %d) has itself as thread parent", m.ID, m.ThreadID, mb.Name, mb.ID)

307 }

308 for i, pid := range m.ThreadParentIDs {

309 am := store.Message{ID: pid}

310 if err := db.Get(ctxbg, &am); err == bstore.ErrAbsent {

311 continue

312 } else if err != nil {

313 return fmt.Errorf("get ancestor message: %v", err)

314 } else if !slices.Equal(m.ThreadParentIDs[i+1:], am.ThreadParentIDs) {

315 checkf(errors.New(`see "mox reassignthreads"`), dbpath, "message %d, thread %d has ancestor ids %v, and ancestor at index %d with id %d should have the same tail but has %v", m.ID, m.ThreadID, m.ThreadParentIDs, i, am.ID, am.ThreadParentIDs)

316 } else {

317 break

318 }

319 }

320 return nil

321 })

322 checkf(err, dbpath, "reading messages in account database to check files")

323

324 haveCounts := true

325 for _, mb := range mailboxes {

326 // We only check if database doesn't have zero values, i.e. not yet set.

327 if !mb.HaveCounts {

328 haveCounts = false

329 }

330 if mb.HaveCounts && mb.MailboxCounts != mbCounts[mb.ID] {

331 checkf(errors.New(`wrong mailbox counts, see "mox recalculatemailboxcounts"`), dbpath, "mailbox %q (id %d) has wrong counts %s, should be %s", mb.Name, mb.ID, mb.MailboxCounts, mbCounts[mb.ID])

332 }

333 }

334

335 if haveCounts {

336 du := store.DiskUsage{ID: 1}

337 err := db.Get(ctxbg, &du)

338 if err == nil {

339 if du.MessageSize != totalSize {

340 checkf(errors.New(`wrong total message size, see mox recalculatemailboxcounts"`), dbpath, "account has wrong total message size %d, should be %d", du.MessageSize, totalSize)

341 }

342 } else if err != nil && !errors.Is(err, bstore.ErrAbsent) {

343 checkf(err, dbpath, "get disk usage")

344 }

345 }

346 }

347

348 // Walk through all files in the msg directory. Warn about files that weren't in

349 // the database as message file. Possibly move away files that could cause trouble.

350 msgdir := filepath.Join(accdir, "msg")

351 if !exists(msgdir) {

352 // New accounts with messages don't have a msg directory.

353 return

354 }

355 err = filepath.WalkDir(msgdir, func(msgpath string, d fs.DirEntry, err error) error {

356 checkf(err, msgpath, "walk")

357 if err != nil {

358 return nil

359 }

360 if d.IsDir() {

361 return nil

362 }

363 p := msgpath[len(msgdir)+1:]

364 if _, ok := seen[p]; ok {

365 return nil

366 }

367 l := strings.Split(p, string(filepath.Separator))

368 if len(l) == 1 {

369 log.Printf("warning: %s: unrecognized file in message directory, ignoring", msgpath)

370 return nil

371 }

372 if !fix {

373 checkf(errors.New("may interfere with future account messages"), msgpath, "unrecognized file in account message directory (use the -fix flag to move it away)")

374 return nil

375 }

376 npath := filepath.Join(dataDir, "moved", "accounts", name, "msg", p)

377 ensureDir(npath)

378 err = os.Rename(msgpath, npath)

379 checkf(err, msgpath, "moving account message file away")

380 if err == nil {

381 log.Printf("warning: moved %s to %s", msgpath, npath)

382 }

383 return nil

384 })

385 checkf(err, msgdir, "walking account message directory")

386 }

387

388 // Check everything in the "accounts" directory.

389 checkAccounts := func() {

390 accountsDir := filepath.Join(dataDir, "accounts")

391 entries, err := os.ReadDir(accountsDir)

392 checkf(err, accountsDir, "reading accounts directory")

393 for _, e := range entries {

394 // We treat all directories as accounts. When we were backing up, we only verified

395 // accounts from the config and made regular file copies of all other files

396 // (perhaps an old account, but at least not with an open database file). It may

397 // turn out that that account was/is not valid, generating warnings. Better safe

398 // than sorry. It should hopefully get the admin to move away such an old account.

399 if e.IsDir() {

400 checkAccount(e.Name())

401 } else {

402 log.Printf("warning: %s: unrecognized file in accounts directory, ignoring", filepath.Join("accounts", e.Name()))

403 }

404 }

405 }

406

407 // Check all files, skipping the known files, queue and accounts directories. Warn

408 // about unknown files. Skip a "tmp" directory. And a "moved" directory, we

409 // probably created it ourselves.

410 backupmoxversion := "(unknown)"

411 checkOther := func() {

412 err := filepath.WalkDir(dataDir, func(dpath string, d fs.DirEntry, err error) error {

413 checkf(err, dpath, "walk")

414 if err != nil {

415 return nil

416 }

417 if dpath == dataDir {

418 return nil

419 }

420 p := dpath

421 if dataDir != "." {

422 p = p[len(dataDir)+1:]

423 }

424 switch p {

425 case "dmarcrpt.db", "dmarceval.db", "mtasts.db", "tlsrpt.db", "tlsrptresult.db", "receivedid.key", "lastknownversion":

426 return nil

427 case "acme", "queue", "accounts", "tmp", "moved":

428 return fs.SkipDir

429 case "moxversion":

430 buf, err := os.ReadFile(dpath)

431 checkf(err, dpath, "reading moxversion")

432 if err == nil {

433 backupmoxversion = string(buf)

434 }

435 return nil

436 }

437 log.Printf("warning: %s: unrecognized other file, ignoring", dpath)

438 return nil

439 })

440 checkf(err, dataDir, "walking data directory")

441 }

442

443 checkDB(true, filepath.Join(dataDir, "dmarcrpt.db"), dmarcdb.ReportsDBTypes)

444 checkDB(false, filepath.Join(dataDir, "dmarceval.db"), dmarcdb.EvalDBTypes) // After v0.0.7.

445 checkDB(true, filepath.Join(dataDir, "mtasts.db"), mtastsdb.DBTypes)

446 checkDB(true, filepath.Join(dataDir, "tlsrpt.db"), tlsrptdb.ReportDBTypes)

447 checkDB(false, filepath.Join(dataDir, "tlsrptresult.db"), tlsrptdb.ResultDBTypes) // After v0.0.7.

448 checkQueue()

449 checkAccounts()

450 checkOther()

451

452 if backupmoxversion != moxvar.Version {

453 log.Printf("NOTE: The backup was made with mox version %q, while verifydata was run with mox version %q. Database files have probably been modified by running mox verifydata. Make a fresh backup before upgrading.", backupmoxversion, moxvar.Version)

454 }

455

456 if fail {

457 log.Fatalf("errors were found")

458 } else {

459 fmt.Printf("%s: OK\n", dataDir)

460 }

461}

462