1package store

2

3import (

4 "context"

5 "errors"

6 "io/fs"

7 "os"

8 "path/filepath"

9 "reflect"

10 "regexp"

11 "strings"

12 "testing"

13 "time"

14

15 "github.com/mjl-/bstore"

16 "github.com/mjl-/sconf"

17

18 "github.com/mjl-/mox/config"

19 "github.com/mjl-/mox/message"

20 "github.com/mjl-/mox/mlog"

21 "github.com/mjl-/mox/mox-"

22)

23

24var ctxbg = context.Background()

25var pkglog = mlog.New("store", nil)

26

27func tcheck(t *testing.T, err error, msg string) {

28 t.Helper()

29 if err != nil {

30 t.Fatalf("%s: %s", msg, err)

31 }

32}

33

34func tcompare(t *testing.T, got, expect any) {

35 t.Helper()

36 if !reflect.DeepEqual(got, expect) {

37 t.Fatalf("got:\n%#v\nexpected:\n%#v", got, expect)

38 }

39}

40

41func TestMailbox(t *testing.T) {

42 log := mlog.New("store", nil)

43 os.RemoveAll("../testdata/store/data")

44 mox.ConfigStaticPath = filepath.FromSlash("../testdata/store/mox.conf")

45 mox.MustLoadConfig(true, false)

46 err := Init(ctxbg)

47 tcheck(t, err, "init")

48 defer func() {

49 err := Close()

50 tcheck(t, err, "close")

51 }()

52 defer Switchboard()()

53 acc, err := OpenAccount(log, "mjl", false)

54 tcheck(t, err, "open account")

55 defer func() {

56 err = acc.Close()

57 tcheck(t, err, "closing account")

58 acc.WaitClosed()

59 }()

60

61 msgFile, err := CreateMessageTemp(log, "account-test")

62 tcheck(t, err, "create temp message file")

63 defer CloseRemoveTempFile(log, msgFile, "temp message file")

64 msgWriter := message.NewWriter(msgFile)

65 _, err = msgWriter.Write([]byte(" message"))

66 tcheck(t, err, "write message")

67

68 msgPrefix := []byte("From: <mjl@mox.example\r\nTo: <mjl@mox.example>\r\nCc: <mjl@mox.example>Subject: test\r\nMessage-Id: <m01@mox.example>\r\n\r\n")

69 msgPrefixCatchall := []byte("Subject: catchall\r\n\r\n")

70 m := Message{

71 Received: time.Now(),

72 Size: int64(len(msgPrefix)) + msgWriter.Size,

73 MsgPrefix: msgPrefix,

74 }

75 msent := m

76 m.ThreadMuted = true

77 m.ThreadCollapsed = true

78 var mbsent Mailbox

79 mreject := m

80 mconsumed := Message{

81 Received: m.Received,

82 Size: int64(len(msgPrefixCatchall)) + msgWriter.Size,

83 MsgPrefix: msgPrefixCatchall,

84 }

85 acc.WithWLock(func() {

86 conf, _ := acc.Conf()

87 err := acc.DeliverDestination(log, conf.Destinations["mjl"], &m, msgFile)

88 tcheck(t, err, "deliver without consume")

89

90 err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {

91 var err error

92 mbsent, err = bstore.QueryTx[Mailbox](tx).FilterNonzero(Mailbox{Name: "Sent"}).Get()

93 tcheck(t, err, "sent mailbox")

94 msent.MailboxID = mbsent.ID

95 msent.MailboxOrigID = mbsent.ID

96 err = acc.MessageAdd(log, tx, &mbsent, &msent, msgFile, AddOpts{SkipSourceFileSync: true, SkipDirSync: true})

97 tcheck(t, err, "deliver message")

98 if !msent.ThreadMuted || !msent.ThreadCollapsed {

99 t.Fatalf("thread muted & collapsed should have been copied from parent (duplicate message-id) m")

100 }

101 err = tx.Update(&mbsent)

102 tcheck(t, err, "update mbsent")

103

104 modseq, err := acc.NextModSeq(tx)

105 tcheck(t, err, "get next modseq")

106 mbrejects := Mailbox{Name: "Rejects", UIDValidity: 1, UIDNext: 1, ModSeq: modseq, CreateSeq: modseq, HaveCounts: true}

107 err = tx.Insert(&mbrejects)

108 tcheck(t, err, "insert rejects mailbox")

109 mreject.MailboxID = mbrejects.ID

110 mreject.MailboxOrigID = mbrejects.ID

111 err = acc.MessageAdd(log, tx, &mbrejects, &mreject, msgFile, AddOpts{SkipSourceFileSync: true, SkipDirSync: true})

112 tcheck(t, err, "deliver message")

113 err = tx.Update(&mbrejects)

114 tcheck(t, err, "update mbrejects")

115

116 return nil

117 })

118 tcheck(t, err, "deliver as sent and rejects")

119

120 err = acc.DeliverDestination(log, conf.Destinations["mjl"], &mconsumed, msgFile)

121 tcheck(t, err, "deliver with consume")

122

123 err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {

124 m.Junk = true

125 l := []Message{m}

126 err = acc.RetrainMessages(ctxbg, log, tx, l)

127 tcheck(t, err, "train as junk")

128 m = l[0]

129 return nil

130 })

131 tcheck(t, err, "train messages")

132 })

133

134 m.Junk = false

135 m.Notjunk = true

136 jf, _, err := acc.OpenJunkFilter(ctxbg, log)

137 tcheck(t, err, "open junk filter")

138 err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {

139 return acc.RetrainMessage(ctxbg, log, tx, jf, &m)

140 })

141 tcheck(t, err, "retraining as non-junk")

142 err = jf.Close()

143 tcheck(t, err, "close junk filter")

144

145 m.Notjunk = false

146 err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {

147 return acc.RetrainMessages(ctxbg, log, tx, []Message{m})

148 })

149 tcheck(t, err, "untraining non-junk")

150

151 err = acc.SetPassword(log, "testtest")

152 tcheck(t, err, "set password")

153

154 key0, err := acc.Subjectpass("test@localhost")

155 tcheck(t, err, "subjectpass")

156 key1, err := acc.Subjectpass("test@localhost")

157 tcheck(t, err, "subjectpass")

158 if key0 != key1 {

159 t.Fatalf("different keys for same address")

160 }

161 key2, err := acc.Subjectpass("test2@localhost")

162 tcheck(t, err, "subjectpass")

163 if key2 == key0 {

164 t.Fatalf("same key for different address")

165 }

166

167 var modseq ModSeq

168 acc.WithWLock(func() {

169 var changes []Change

170

171 err := acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {

172 _, _, err := acc.MailboxEnsure(tx, "Testbox", true, SpecialUse{}, &modseq)

173 return err

174 })

175 tcheck(t, err, "ensure mailbox exists")

176 err = acc.DB.Read(ctxbg, func(tx *bstore.Tx) error {

177 _, _, err := acc.MailboxEnsure(tx, "Testbox", true, SpecialUse{}, &modseq)

178 return err

179 })

180 tcheck(t, err, "ensure mailbox exists")

181

182 err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {

183 _, _, err := acc.MailboxEnsure(tx, "Testbox2", false, SpecialUse{}, &modseq)

184 tcheck(t, err, "create mailbox")

185

186 exists, err := acc.MailboxExists(tx, "Testbox2")

187 tcheck(t, err, "checking that mailbox exists")

188 if !exists {

189 t.Fatalf("mailbox does not exist")

190 }

191

192 exists, err = acc.MailboxExists(tx, "Testbox3")

193 tcheck(t, err, "checking that mailbox does not exist")

194 if exists {

195 t.Fatalf("mailbox does exist")

196 }

197

198 xmb, err := acc.MailboxFind(tx, "Testbox3")

199 tcheck(t, err, "finding non-existing mailbox")

200 if xmb != nil {

201 t.Fatalf("did find Testbox3: %v", xmb)

202 }

203 xmb, err = acc.MailboxFind(tx, "Testbox2")

204 tcheck(t, err, "finding existing mailbox")

205 if xmb == nil {

206 t.Fatalf("did not find Testbox2")

207 }

208

209 nchanges, err := acc.SubscriptionEnsure(tx, "Testbox2")

210 tcheck(t, err, "ensuring new subscription")

211 if len(nchanges) == 0 {

212 t.Fatalf("new subscription did not result in changes")

213 }

214 changes = append(changes, nchanges...)

215 nchanges, err = acc.SubscriptionEnsure(tx, "Testbox2")

216 tcheck(t, err, "ensuring already present subscription")

217 if len(nchanges) != 0 {

218 t.Fatalf("already present subscription resulted in changes")

219 }

220

221 // todo: check that messages are removed.

222 mbRej, err := bstore.QueryTx[Mailbox](tx).FilterNonzero(Mailbox{Name: "Rejects"}).Get()

223 tcheck(t, err, "get rejects mailbox")

224 nchanges, hasSpace, err := acc.TidyRejectsMailbox(log, tx, &mbRej)

225 tcheck(t, err, "tidy rejects mailbox")

226 changes = append(changes, nchanges...)

227 if !hasSpace {

228 t.Fatalf("no space for more rejects")

229 }

230

231 return nil

232 })

233 tcheck(t, err, "write tx")

234

235 BroadcastChanges(acc, changes)

236

237 acc.RejectsRemove(log, "Rejects", "m01@mox.example")

238 })

239

240 // Run the auth tests twice for possible cache effects.

241 for range 2 {

242 _, _, err := OpenEmailAuth(log, "mjl@mox.example", "bogus", false)

243 if err != ErrUnknownCredentials {

244 t.Fatalf("got %v, expected ErrUnknownCredentials", err)

245 }

246 }

247

248 for range 2 {

249 acc2, _, err := OpenEmailAuth(log, "mjl@mox.example", "testtest", false)

250 tcheck(t, err, "open for email with auth")

251 err = acc2.Close()

252 tcheck(t, err, "close account")

253 }

254

255 acc2, _, err := OpenEmailAuth(log, "other@mox.example", "testtest", false)

256 tcheck(t, err, "open for email with auth")

257 err = acc2.Close()

258 tcheck(t, err, "close account")

259

260 _, _, err = OpenEmailAuth(log, "bogus@mox.example", "testtest", false)

261 if err != ErrUnknownCredentials {

262 t.Fatalf("got %v, expected ErrUnknownCredentials", err)

263 }

264

265 _, _, err = OpenEmailAuth(log, "mjl@test.example", "testtest", false)

266 if err != ErrUnknownCredentials {

267 t.Fatalf("got %v, expected ErrUnknownCredentials", err)

268 }

269}

270

271func TestMessageRuleset(t *testing.T) {

272 log := mlog.New("store", nil)

273 f, err := CreateMessageTemp(log, "msgruleset")

274 tcheck(t, err, "creating temp msg file")

275 defer CloseRemoveTempFile(log, f, "temp message file")

276

277 msgBuf := []byte(strings.ReplaceAll(`List-ID: <test.mox.example>

278

279test

280`, "\n", "\r\n"))

281

282 const destConf = `

283Rulesets:

284 -

285 HeadersRegexp:

286 list-id: <test\.mox\.example>

287 Mailbox: test

288`

289 var dest config.Destination

290 err = sconf.Parse(strings.NewReader(destConf), &dest)

291 tcheck(t, err, "parse config")

292 // todo: should use regular config initialization functions for this.

293 var hdrs [][2]*regexp.Regexp

294 for k, v := range dest.Rulesets[0].HeadersRegexp {

295 rk, err := regexp.Compile(k)

296 tcheck(t, err, "compile key")

297 rv, err := regexp.Compile(v)

298 tcheck(t, err, "compile value")

299 hdrs = append(hdrs, [...]*regexp.Regexp{rk, rv})

300 }

301 dest.Rulesets[0].HeadersRegexpCompiled = hdrs

302

303 c := MessageRuleset(log, dest, &Message{}, msgBuf, f)

304 if c == nil {

305 t.Fatalf("expected ruleset match")

306 }

307

308 msg2Buf := []byte(strings.ReplaceAll(`From: <mjl@mox.example>

309

310test

311`, "\n", "\r\n"))

312 c = MessageRuleset(log, dest, &Message{}, msg2Buf, f)

313 if c != nil {

314 t.Fatalf("expected no ruleset match")

315 }

316

317 // todo: test the SMTPMailFrom and VerifiedDomains rule.

318}

319

320// Check that opening an account forwards the Message.ID used for new additions if

321// message files already exist in the file system.

322func TestNextMessageID(t *testing.T) {

323 log := mlog.New("store", nil)

324 os.RemoveAll("../testdata/store/data")

325 mox.ConfigStaticPath = filepath.FromSlash("../testdata/store/mox.conf")

326 mox.MustLoadConfig(true, false)

327 err := Init(ctxbg)

328 tcheck(t, err, "init")

329 defer func() {

330 err := Close()

331 tcheck(t, err, "close")

332 }()

333 defer Switchboard()()

334

335 // Ensure account exists.

336 acc, err := OpenAccount(log, "mjl", false)

337 tcheck(t, err, "open account")

338 err = acc.Close()

339 tcheck(t, err, "closing account")

340 acc.WaitClosed()

341 acc = nil

342

343 // Create file on disk to occupy the first Message.ID that would otherwise be used for deliveries..

344 msgData := []byte("a: b\r\n\r\ntest\r\n")

345 msgDir := filepath.FromSlash("../testdata/store/data/accounts/mjl/msg")

346 os.MkdirAll(filepath.Join(msgDir, "a"), 0700)

347 msgPath := filepath.Join(msgDir, "a", "1")

348 err = os.WriteFile(msgPath, msgData, 0700)

349 tcheck(t, err, "write message file")

350

351 msgPathBogus := filepath.Join(msgDir, "a", "bogus")

352 err = os.WriteFile(msgPathBogus, []byte("test"), 0700)

353 tcheck(t, err, "create message file")

354 msgPathBadID := filepath.Join(msgDir, "a", "10000") // Out of range.

355 err = os.WriteFile(msgPathBadID, []byte("test"), 0700)

356 tcheck(t, err, "create message file")

357

358 // Open account. This should increase the next message ID.

359 acc, err = OpenAccount(log, "mjl", false)

360 tcheck(t, err, "open account")

361

362 // Deliver a message. It should get ID 2.

363 mf, err := CreateMessageTemp(log, "account-test")

364 tcheck(t, err, "creating temp message file")

365 _, err = mf.Write(msgData)

366 tcheck(t, err, "write file")

367 defer CloseRemoveTempFile(log, mf, "temp message file")

368 m := Message{

369 Size: int64(len(msgData)),

370 }

371 err = acc.DeliverMailbox(log, "Inbox", &m, mf)

372 tcheck(t, err, "deliver mailbox")

373 if m.ID != 2 {

374 t.Fatalf("got message id %d, expected 2", m.ID)

375 }

376

377 // Ensure account consistency check won't complain.

378 err = os.Remove(msgPath)

379 tcheck(t, err, "removing message path")

380 err = os.Remove(msgPathBogus)

381 tcheck(t, err, "removing message path")

382 err = os.Remove(msgPathBadID)

383 tcheck(t, err, "removing message path")

384

385 err = acc.Close()

386 tcheck(t, err, "closing account")

387 acc.WaitClosed()

388

389 // Try again, but also create next message directory, but no file.

390 os.MkdirAll(filepath.Join(msgDir, "b"), 0700)

391 os.MkdirAll(filepath.Join(msgDir, "d"), 0700) // Not used.

392

393 // Open account again, increasing next message ID.

394 acc, err = OpenAccount(log, "mjl", false)

395 tcheck(t, err, "open account")

396

397 // Deliver a message. It should get ID $msgFilesPerDir+1.

398 mf, err = CreateMessageTemp(log, "account-test")

399 tcheck(t, err, "creating temp message file")

400 _, err = mf.Write(msgData)

401 tcheck(t, err, "write file")

402 defer CloseRemoveTempFile(log, mf, "temp message file")

403 m = Message{

404 Size: int64(len(msgData)),

405 }

406 err = acc.DeliverMailbox(log, "Inbox", &m, mf)

407 tcheck(t, err, "deliver mailbox")

408 if m.ID != msgFilesPerDir+1 {

409 t.Fatalf("got message id %d, expected $msgFilesPerDir+1", m.ID)

410 }

411

412 err = acc.Close()

413 tcheck(t, err, "closing account")

414 acc.WaitClosed()

415}

416

417func TestRemove(t *testing.T) {

418 log := mlog.New("store", nil)

419 os.RemoveAll("../testdata/store/data")

420 mox.ConfigStaticPath = filepath.FromSlash("../testdata/store/mox.conf")

421 mox.MustLoadConfig(true, false)

422 err := Init(ctxbg)

423 tcheck(t, err, "init")

424 defer func() {

425 err := Close()

426 tcheck(t, err, "close")

427 }()

428 defer Switchboard()()

429

430 // Note: we are not removing the account from the config file. Nothing currently

431 // has a problem with that.

432

433 // Ensure account exists.

434 acc, err := OpenAccount(log, "mjl", false)

435 tcheck(t, err, "open account")

436

437 // Mark account removed. It will only be removed when we close the account.

438 err = acc.Remove(context.Background())

439 tcheck(t, err, "remove account")

440

441 p := filepath.Join(mox.DataDirPath("accounts"), "mjl")

442 _, err = os.Stat(p)

443 tcheck(t, err, "stat account dir")

444

445 err = acc.Close()

446 tcheck(t, err, "closing account")

447 acc.WaitClosed()

448 acc = nil

449

450 if _, err := os.Stat(p); err == nil || !errors.Is(err, fs.ErrNotExist) {

451 t.Fatalf(`got stat err %v for account directory, expected "does not exist"`, err)

452 }

453

454 // Recreate files and directories. We will reinitialize store/ without closing our

455 // account reference. This will apply the account removal. We only drop our (now

456 // broken) account reference when done.

457 acc, err = OpenAccount(log, "mjl", false)

458 tcheck(t, err, "open account")

459 defer func() {

460 acc.Close() // Ignore errors.

461 acc.WaitClosed()

462 CheckConsistencyOnClose = true

463 }()

464

465 // Init below will remove the directory, we are no longer consistent.

466 CheckConsistencyOnClose = false

467

468 err = acc.Remove(context.Background())

469 tcheck(t, err, "remove account")

470

471 _, err = os.Stat(p)

472 tcheck(t, err, "stat account dir")

473

474 err = Close()

475 tcheck(t, err, "close store")

476 err = Init(ctxbg)

477 tcheck(t, err, "init store")

478 if _, err := os.Stat(p); err == nil || !errors.Is(err, fs.ErrNotExist) {

479 t.Fatalf(`got stat err %v for account directory, expected "does not exist"`, err)

480 }

481 exists, err := bstore.QueryDB[AccountRemove](ctxbg, AuthDB).Exists()

482 tcheck(t, err, "checking for account removals")

483 tcompare(t, exists, false)

484}

485