1/*

2Package store implements storage for accounts, their mailboxes, IMAP

3subscriptions and messages, and broadcasts updates (e.g. mail delivery) to

4interested sessions (e.g. IMAP connections).

5

6Layout of storage for accounts:

7

8 <DataDir>/accounts/<name>/index.db

9 <DataDir>/accounts/<name>/msg/[a-zA-Z0-9_-]+/<id>

10

11Index.db holds tables for user information, mailboxes, and messages. Messages

12are stored in the msg/ subdirectory, each in their own file. The on-disk message

13does not contain headers generated during an incoming SMTP transaction, such as

14Received and Authentication-Results headers. Those are in the database to

15prevent having to rewrite incoming messages (e.g. Authentication-Result for DKIM

16signatures can only be determined after having read the message). Messages must

17be read through MsgReader, which transparently adds the prefix from the

18database.

19*/

20package store

21

22// todo: make up a function naming scheme that indicates whether caller should broadcast changes.

23

24import (

25 "context"

26 "crypto/md5"

27 cryptorand "crypto/rand"

28 "crypto/sha1"

29 "crypto/sha256"

30 "encoding"

31 "encoding/json"

32 "errors"

33 "fmt"

34 "hash"

35 "io"

36 "os"

37 "path/filepath"

38 "runtime/debug"

39 "sort"

40 "strconv"

41 "strings"

42 "sync"

43 "time"

44

45 "golang.org/x/crypto/bcrypt"

46 "golang.org/x/exp/slices"

47 "golang.org/x/exp/slog"

48 "golang.org/x/text/unicode/norm"

49

50 "github.com/mjl-/bstore"

51

52 "github.com/mjl-/mox/config"

53 "github.com/mjl-/mox/dns"

54 "github.com/mjl-/mox/message"

55 "github.com/mjl-/mox/metrics"

56 "github.com/mjl-/mox/mlog"

57 "github.com/mjl-/mox/mox-"

58 "github.com/mjl-/mox/moxio"

59 "github.com/mjl-/mox/publicsuffix"

60 "github.com/mjl-/mox/scram"

61 "github.com/mjl-/mox/smtp"

62)

63

64// If true, each time an account is closed its database file is checked for

65// consistency. If an inconsistency is found, panic is called. Set by default

66// because of all the packages with tests, the mox main function sets it to

67// false again.

68var CheckConsistencyOnClose = true

69

70var (

71 ErrUnknownMailbox = errors.New("no such mailbox")

72 ErrUnknownCredentials = errors.New("credentials not found")

73 ErrAccountUnknown = errors.New("no such account")

74 ErrOverQuota = errors.New("account over quota")

75)

76

77var DefaultInitialMailboxes = config.InitialMailboxes{

78 SpecialUse: config.SpecialUseMailboxes{

79 Sent: "Sent",

80 Archive: "Archive",

81 Trash: "Trash",

82 Draft: "Drafts",

83 Junk: "Junk",

84 },

85}

86

87type SCRAM struct {

88 Salt []byte

89 Iterations int

90 SaltedPassword []byte

91}

92

93// CRAMMD5 holds HMAC ipad and opad hashes that are initialized with the first

94// block with (a derivation of) the key/password, so we don't store the password in plain

95// text.

96type CRAMMD5 struct {

97 Ipad hash.Hash

98 Opad hash.Hash

99}

100

101// BinaryMarshal is used by bstore to store the ipad/opad hash states.

102func (c CRAMMD5) MarshalBinary() ([]byte, error) {

103 if c.Ipad == nil || c.Opad == nil {

104 return nil, nil

105 }

106

107 ipad, err := c.Ipad.(encoding.BinaryMarshaler).MarshalBinary()

108 if err != nil {

109 return nil, fmt.Errorf("marshal ipad: %v", err)

110 }

111 opad, err := c.Opad.(encoding.BinaryMarshaler).MarshalBinary()

112 if err != nil {

113 return nil, fmt.Errorf("marshal opad: %v", err)

114 }

115 buf := make([]byte, 2+len(ipad)+len(opad))

116 ipadlen := uint16(len(ipad))

117 buf[0] = byte(ipadlen >> 8)

118 buf[1] = byte(ipadlen >> 0)

119 copy(buf[2:], ipad)

120 copy(buf[2+len(ipad):], opad)

121 return buf, nil

122}

123

124// BinaryUnmarshal is used by bstore to restore the ipad/opad hash states.

125func (c *CRAMMD5) UnmarshalBinary(buf []byte) error {

126 if len(buf) == 0 {

127 *c = CRAMMD5{}

128 return nil

129 }

130 if len(buf) < 2 {

131 return fmt.Errorf("short buffer")

132 }

133 ipadlen := int(uint16(buf[0])<<8 | uint16(buf[1])<<0)

134 if len(buf) < 2+ipadlen {

135 return fmt.Errorf("buffer too short for ipadlen")

136 }

137 ipad := md5.New()

138 opad := md5.New()

139 if err := ipad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2 : 2+ipadlen]); err != nil {

140 return fmt.Errorf("unmarshal ipad: %v", err)

141 }

142 if err := opad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2+ipadlen:]); err != nil {

143 return fmt.Errorf("unmarshal opad: %v", err)

144 }

145 *c = CRAMMD5{ipad, opad}

146 return nil

147}

148

149// Password holds credentials in various forms, for logging in with SMTP/IMAP.

150type Password struct {

151 Hash string // bcrypt hash for IMAP LOGIN, SASL PLAIN and HTTP basic authentication.

152 CRAMMD5 CRAMMD5 // For SASL CRAM-MD5.

153 SCRAMSHA1 SCRAM // For SASL SCRAM-SHA-1.

154 SCRAMSHA256 SCRAM // For SASL SCRAM-SHA-256.

155}

156

157// Subjectpass holds the secret key used to sign subjectpass tokens.

158type Subjectpass struct {

159 Email string // Our destination address (canonical, with catchall localpart stripped).

160 Key string

161}

162

163// NextUIDValidity is a singleton record in the database with the next UIDValidity

164// to use for the next mailbox.

165type NextUIDValidity struct {

166 ID int // Just a single record with ID 1.

167 Next uint32

168}

169

170// SyncState track ModSeqs.

171type SyncState struct {

172 ID int // Just a single record with ID 1.

173

174 // Last used, next assigned will be one higher. The first value we hand out is 2.

175 // That's because 0 (the default value for old existing messages, from before the

176 // Message.ModSeq field) is special in IMAP, so we return it as 1.

177 LastModSeq ModSeq `bstore:"nonzero"`

178

179 // Highest ModSeq of expunged record that we deleted. When a clients synchronizes

180 // and requests changes based on a modseq before this one, we don't have the

181 // history to provide information about deletions. We normally keep these expunged

182 // records around, but we may periodically truly delete them to reclaim storage

183 // space. Initially set to -1 because we don't want to match with any ModSeq in the

184 // database, which can be zero values.

185 HighestDeletedModSeq ModSeq

186}

187

188// Mailbox is collection of messages, e.g. Inbox or Sent.

189type Mailbox struct {

190 ID int64

191

192 // "Inbox" is the name for the special IMAP "INBOX". Slash separated

193 // for hierarchy.

194 Name string `bstore:"nonzero,unique"`

195

196 // If UIDs are invalidated, e.g. when renaming a mailbox to a previously existing

197 // name, UIDValidity must be changed. Used by IMAP for synchronization.

198 UIDValidity uint32

199

200 // UID likely to be assigned to next message. Used by IMAP to detect messages

201 // delivered to a mailbox.

202 UIDNext UID

203

204 SpecialUse

205

206 // Keywords as used in messages. Storing a non-system keyword for a message

207 // automatically adds it to this list. Used in the IMAP FLAGS response. Only

208 // "atoms" are allowed (IMAP syntax), keywords are case-insensitive, only stored in

209 // lower case (for JMAP), sorted.

210 Keywords []string

211

212 HaveCounts bool // Whether MailboxCounts have been initialized.

213 MailboxCounts // Statistics about messages, kept up to date whenever a change happens.

214}

215

216// MailboxCounts tracks statistics about messages for a mailbox.

217type MailboxCounts struct {

218 Total int64 // Total number of messages, excluding \Deleted. For JMAP.

219 Deleted int64 // Number of messages with \Deleted flag. Used for IMAP message count that includes messages with \Deleted.

220 Unread int64 // Messages without \Seen, excluding those with \Deleted, for JMAP.

221 Unseen int64 // Messages without \Seen, including those with \Deleted, for IMAP.

222 Size int64 // Number of bytes for all messages.

223}

224

225func (mc MailboxCounts) String() string {

226 return fmt.Sprintf("%d total, %d deleted, %d unread, %d unseen, size %d bytes", mc.Total, mc.Deleted, mc.Unread, mc.Unseen, mc.Size)

227}

228

229// Add increases mailbox counts mc with those of delta.

230func (mc *MailboxCounts) Add(delta MailboxCounts) {

231 mc.Total += delta.Total

232 mc.Deleted += delta.Deleted

233 mc.Unread += delta.Unread

234 mc.Unseen += delta.Unseen

235 mc.Size += delta.Size

236}

237

238// Add decreases mailbox counts mc with those of delta.

239func (mc *MailboxCounts) Sub(delta MailboxCounts) {

240 mc.Total -= delta.Total

241 mc.Deleted -= delta.Deleted

242 mc.Unread -= delta.Unread

243 mc.Unseen -= delta.Unseen

244 mc.Size -= delta.Size

245}

246

247// SpecialUse identifies a specific role for a mailbox, used by clients to

248// understand where messages should go.

249type SpecialUse struct {

250 Archive bool

251 Draft bool

252 Junk bool

253 Sent bool

254 Trash bool

255}

256

257// CalculateCounts calculates the full current counts for messages in the mailbox.

258func (mb *Mailbox) CalculateCounts(tx *bstore.Tx) (mc MailboxCounts, err error) {

259 q := bstore.QueryTx[Message](tx)

260 q.FilterNonzero(Message{MailboxID: mb.ID})

261 q.FilterEqual("Expunged", false)

262 err = q.ForEach(func(m Message) error {

263 mc.Add(m.MailboxCounts())

264 return nil

265 })

266 return

267}

268

269// ChangeSpecialUse returns a change for special-use flags, for broadcasting to

270// other connections.

271func (mb Mailbox) ChangeSpecialUse() ChangeMailboxSpecialUse {

272 return ChangeMailboxSpecialUse{mb.ID, mb.Name, mb.SpecialUse}

273}

274

275// ChangeKeywords returns a change with new keywords for a mailbox (e.g. after

276// setting a new keyword on a message in the mailbox), for broadcasting to other

277// connections.

278func (mb Mailbox) ChangeKeywords() ChangeMailboxKeywords {

279 return ChangeMailboxKeywords{mb.ID, mb.Name, mb.Keywords}

280}

281

282// KeywordsChanged returns whether the keywords in a mailbox have changed.

283func (mb Mailbox) KeywordsChanged(origmb Mailbox) bool {

284 if len(mb.Keywords) != len(origmb.Keywords) {

285 return true

286 }

287 // Keywords are stored sorted.

288 for i, kw := range mb.Keywords {

289 if origmb.Keywords[i] != kw {

290 return true

291 }

292 }

293 return false

294}

295

296// CountsChange returns a change with mailbox counts.

297func (mb Mailbox) ChangeCounts() ChangeMailboxCounts {

298 return ChangeMailboxCounts{mb.ID, mb.Name, mb.MailboxCounts}

299}

300

301// Subscriptions are separate from existence of mailboxes.

302type Subscription struct {

303 Name string

304}

305

306// Flags for a mail message.

307type Flags struct {

308 Seen bool

309 Answered bool

310 Flagged bool

311 Forwarded bool

312 Junk bool

313 Notjunk bool

314 Deleted bool

315 Draft bool

316 Phishing bool

317 MDNSent bool

318}

319

320// FlagsAll is all flags set, for use as mask.

321var FlagsAll = Flags{true, true, true, true, true, true, true, true, true, true}

322

323// Validation of "message From" domain.

324type Validation uint8

325

326const (

327 ValidationUnknown Validation = 0

328 ValidationStrict Validation = 1 // Like DMARC, with strict policies.

329 ValidationDMARC Validation = 2 // Actual DMARC policy.

330 ValidationRelaxed Validation = 3 // Like DMARC, with relaxed policies.

331 ValidationPass Validation = 4 // For SPF.

332 ValidationNeutral Validation = 5 // For SPF.

333 ValidationTemperror Validation = 6

334 ValidationPermerror Validation = 7

335 ValidationFail Validation = 8

336 ValidationSoftfail Validation = 9 // For SPF.

337 ValidationNone Validation = 10 // E.g. No records.

338)

339

340// Message stored in database and per-message file on disk.

341//

342// Contents are always the combined data from MsgPrefix and the on-disk file named

343// based on ID.

344//

345// Messages always have a header section, even if empty. Incoming messages without

346// header section must get an empty header section added before inserting.

347type Message struct {

348 // ID, unchanged over lifetime, determines path to on-disk msg file.

349 // Set during deliver.

350 ID int64

351

352 UID UID `bstore:"nonzero"` // UID, for IMAP. Set during deliver.

353 MailboxID int64 `bstore:"nonzero,unique MailboxID+UID,index MailboxID+Received,index MailboxID+ModSeq,ref Mailbox"`

354

355 // Modification sequence, for faster syncing with IMAP QRESYNC and JMAP.

356 // ModSeq is the last modification. CreateSeq is the Seq the message was inserted,

357 // always <= ModSeq. If Expunged is set, the message has been removed and should not

358 // be returned to the user. In this case, ModSeq is the Seq where the message is

359 // removed, and will never be changed again.

360 // We have an index on both ModSeq (for JMAP that synchronizes per account) and

361 // MailboxID+ModSeq (for IMAP that synchronizes per mailbox).

362 // The index on CreateSeq helps efficiently finding created messages for JMAP.

363 // The value of ModSeq is special for IMAP. Messages that existed before ModSeq was

364 // added have 0 as value. But modseq 0 in IMAP is special, so we return it as 1. If

365 // we get modseq 1 from a client, the IMAP server will translate it to 0. When we

366 // return modseq to clients, we turn 0 into 1.

367 ModSeq ModSeq `bstore:"index"`

368 CreateSeq ModSeq `bstore:"index"`

369 Expunged bool

370

371 // If set, this message was delivered to a Rejects mailbox. When it is moved to a

372 // different mailbox, its MailboxOrigID is set to the destination mailbox and this

373 // flag cleared.

374 IsReject bool

375

376 // If set, this is a forwarded message (through a ruleset with IsForward). This

377 // causes fields used during junk analysis to be moved to their Orig variants, and

378 // masked IP fields cleared, so they aren't used in junk classifications for

379 // incoming messages. This ensures the forwarded messages don't cause negative

380 // reputation for the forwarding mail server, which may also be sending regular

381 // messages.

382 IsForward bool

383

384 // MailboxOrigID is the mailbox the message was originally delivered to. Typically

385 // Inbox or Rejects, but can also be a mailbox configured in a Ruleset, or

386 // Postmaster, TLS/DMARC reporting addresses. MailboxOrigID is not changed when the

387 // message is moved to another mailbox, e.g. Archive/Trash/Junk. Used for

388 // per-mailbox reputation.

389 //

390 // MailboxDestinedID is normally 0, but when a message is delivered to the Rejects

391 // mailbox, it is set to the intended mailbox according to delivery rules,

392 // typically that of Inbox. When such a message is moved out of Rejects, the

393 // MailboxOrigID is corrected by setting it to MailboxDestinedID. This ensures the

394 // message is used for reputation calculation for future deliveries to that

395 // mailbox.

396 //

397 // These are not bstore references to prevent having to update all messages in a

398 // mailbox when the original mailbox is removed. Use of these fields requires

399 // checking if the mailbox still exists.

400 MailboxOrigID int64

401 MailboxDestinedID int64

402

403 Received time.Time `bstore:"default now,index"`

404

405 // Full IP address of remote SMTP server. Empty if not delivered over SMTP. The

406 // masked IPs are used to classify incoming messages. They are left empty for

407 // messages matching a ruleset for forwarded messages.

408 RemoteIP string

409 RemoteIPMasked1 string `bstore:"index RemoteIPMasked1+Received"` // For IPv4 /32, for IPv6 /64, for reputation.

410 RemoteIPMasked2 string `bstore:"index RemoteIPMasked2+Received"` // For IPv4 /26, for IPv6 /48.

411 RemoteIPMasked3 string `bstore:"index RemoteIPMasked3+Received"` // For IPv4 /21, for IPv6 /32.

412

413 // Only set if present and not an IP address. Unicode string. Empty for forwarded

414 // messages.

415 EHLODomain string `bstore:"index EHLODomain+Received"`

416 MailFrom string // With localpart and domain. Can be empty.

417 MailFromLocalpart smtp.Localpart // SMTP "MAIL FROM", can be empty.

418 // Only set if it is a domain, not an IP. Unicode string. Empty for forwarded

419 // messages, but see OrigMailFromDomain.

420 MailFromDomain string `bstore:"index MailFromDomain+Received"`

421 RcptToLocalpart smtp.Localpart // SMTP "RCPT TO", can be empty.

422 RcptToDomain string // Unicode string.

423

424 // Parsed "From" message header, used for reputation along with domain validation.

425 MsgFromLocalpart smtp.Localpart

426 MsgFromDomain string `bstore:"index MsgFromDomain+Received"` // Unicode string.

427 MsgFromOrgDomain string `bstore:"index MsgFromOrgDomain+Received"` // Unicode string.

428

429 // Simplified statements of the Validation fields below, used for incoming messages

430 // to check reputation.

431 EHLOValidated bool

432 MailFromValidated bool

433 MsgFromValidated bool

434

435 EHLOValidation Validation // Validation can also take reverse IP lookup into account, not only SPF.

436 MailFromValidation Validation // Can have SPF-specific validations like ValidationSoftfail.

437 MsgFromValidation Validation // Desirable validations: Strict, DMARC, Relaxed. Will not be just Pass.

438

439 // Domains with verified DKIM signatures. Unicode string. For forwarded messages, a

440 // DKIM domain that matched a ruleset's verified domain is left out, but included

441 // in OrigDKIMDomains.

442 DKIMDomains []string `bstore:"index DKIMDomains+Received"`

443

444 // For forwarded messages,

445 OrigEHLODomain string

446 OrigDKIMDomains []string

447

448 // Canonicalized Message-Id, always lower-case and normalized quoting, without

449 // <>'s. Empty if missing. Used for matching message threads, and to prevent

450 // duplicate reject delivery.

451 MessageID string `bstore:"index"`

452 // lower-case: ../rfc/5256:495

453

454 // For matching threads in case there is no References/In-Reply-To header. It is

455 // lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed.

456 SubjectBase string `bstore:"index"`

457 // ../rfc/5256:90

458

459 // Hash of message. For rejects delivery in case there is no Message-ID, only set

460 // when delivered as reject.

461 MessageHash []byte

462

463 // ID of message starting this thread.

464 ThreadID int64 `bstore:"index"`

465 // IDs of parent messages, from closest parent to the root message. Parent messages

466 // may be in a different mailbox, or may no longer exist. ThreadParentIDs must

467 // never contain the message id itself (a cycle), and parent messages must

468 // reference the same ancestors.

469 ThreadParentIDs []int64

470 // ThreadMissingLink is true if there is no match with a direct parent. E.g. first

471 // ID in ThreadParentIDs is not the direct ancestor (an intermediate message may

472 // have been deleted), or subject-based matching was done.

473 ThreadMissingLink bool

474 // If set, newly delivered child messages are automatically marked as read. This

475 // field is copied to new child messages. Changes are propagated to the webmail

476 // client.

477 ThreadMuted bool

478 // If set, this (sub)thread is collapsed in the webmail client, for threading mode

479 // "on" (mode "unread" ignores it). This field is copied to new child message.

480 // Changes are propagated to the webmail client.

481 ThreadCollapsed bool

482

483 // If received message was known to match a mailing list rule (with modified junk

484 // filtering).

485 IsMailingList bool

486

487 ReceivedTLSVersion uint16 // 0 if unknown, 1 if plaintext/no TLS, otherwise TLS cipher suite.

488 ReceivedTLSCipherSuite uint16

489 ReceivedRequireTLS bool // Whether RequireTLS was known to be used for incoming delivery.

490

491 Flags

492 // For keywords other than system flags or the basic well-known $-flags. Only in

493 // "atom" syntax (IMAP), they are case-insensitive, always stored in lower-case

494 // (for JMAP), sorted.

495 Keywords []string `bstore:"index"`

496 Size int64

497 TrainedJunk *bool // If nil, no training done yet. Otherwise, true is trained as junk, false trained as nonjunk.

498 MsgPrefix []byte // Typically holds received headers and/or header separator.

499

500 // ParsedBuf message structure. Currently saved as JSON of message.Part because bstore

501 // cannot yet store recursive types. Created when first needed, and saved in the

502 // database.

503 // todo: once replaced with non-json storage, remove date fixup in ../message/part.go.

504 ParsedBuf []byte

505}

506

507// MailboxCounts returns the delta to counts this message means for its

508// mailbox.

509func (m Message) MailboxCounts() (mc MailboxCounts) {

510 if m.Expunged {

511 return

512 }

513 if m.Deleted {

514 mc.Deleted++

515 } else {

516 mc.Total++

517 }

518 if !m.Seen {

519 mc.Unseen++

520 if !m.Deleted {

521 mc.Unread++

522 }

523 }

524 mc.Size += m.Size

525 return

526}

527

528func (m Message) ChangeAddUID() ChangeAddUID {

529 return ChangeAddUID{m.MailboxID, m.UID, m.ModSeq, m.Flags, m.Keywords}

530}

531

532func (m Message) ChangeFlags(orig Flags) ChangeFlags {

533 mask := m.Flags.Changed(orig)

534 return ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, ModSeq: m.ModSeq, Mask: mask, Flags: m.Flags, Keywords: m.Keywords}

535}

536

537func (m Message) ChangeThread() ChangeThread {

538 return ChangeThread{[]int64{m.ID}, m.ThreadMuted, m.ThreadCollapsed}

539}

540

541// ModSeq represents a modseq as stored in the database. ModSeq 0 in the

542// database is sent to the client as 1, because modseq 0 is special in IMAP.

543// ModSeq coming from the client are of type int64.

544type ModSeq int64

545

546func (ms ModSeq) Client() int64 {

547 if ms == 0 {

548 return 1

549 }

550 return int64(ms)

551}

552

553// ModSeqFromClient converts a modseq from a client to a modseq for internal

554// use, e.g. in a database query.

555// ModSeq 1 is turned into 0 (the Go zero value for ModSeq).

556func ModSeqFromClient(modseq int64) ModSeq {

557 if modseq == 1 {

558 return 0

559 }

560 return ModSeq(modseq)

561}

562

563// PrepareExpunge clears fields that are no longer needed after an expunge, so

564// almost all fields. Does not change ModSeq, but does set Expunged.

565func (m *Message) PrepareExpunge() {

566 *m = Message{

567 ID: m.ID,

568 UID: m.UID,

569 MailboxID: m.MailboxID,

570 CreateSeq: m.CreateSeq,

571 ModSeq: m.ModSeq,

572 Expunged: true,

573 ThreadID: m.ThreadID,

574 }

575}

576

577// PrepareThreading sets MessageID and SubjectBase (used in threading) based on the

578// envelope in part.

579func (m *Message) PrepareThreading(log mlog.Log, part *message.Part) {

580 if part.Envelope == nil {

581 return

582 }

583 messageID, raw, err := message.MessageIDCanonical(part.Envelope.MessageID)

584 if err != nil {

585 log.Debugx("parsing message-id, ignoring", err, slog.String("messageid", part.Envelope.MessageID))

586 } else if raw {

587 log.Debug("could not parse message-id as address, continuing with raw value", slog.String("messageid", part.Envelope.MessageID))

588 }

589 m.MessageID = messageID

590 m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false)

591}

592

593// LoadPart returns a message.Part by reading from m.ParsedBuf.

594func (m Message) LoadPart(r io.ReaderAt) (message.Part, error) {

595 if m.ParsedBuf == nil {

596 return message.Part{}, fmt.Errorf("message not parsed")

597 }

598 var p message.Part

599 err := json.Unmarshal(m.ParsedBuf, &p)

600 if err != nil {

601 return p, fmt.Errorf("unmarshal message part")

602 }

603 p.SetReaderAt(r)

604 return p, nil

605}

606

607// NeedsTraining returns whether message needs a training update, based on

608// TrainedJunk (current training status) and new Junk/Notjunk flags.

609func (m Message) NeedsTraining() bool {

610 untrain := m.TrainedJunk != nil

611 untrainJunk := untrain && *m.TrainedJunk

612 train := m.Junk || m.Notjunk && !(m.Junk && m.Notjunk)

613 trainJunk := m.Junk

614 return untrain != train || untrain && train && untrainJunk != trainJunk

615}

616

617// JunkFlagsForMailbox sets Junk and Notjunk flags based on mailbox name if configured. Often

618// used when delivering/moving/copying messages to a mailbox. Mail clients are not

619// very helpful with setting junk/notjunk flags. But clients can move/copy messages

620// to other mailboxes. So we set flags when clients move a message.

621func (m *Message) JunkFlagsForMailbox(mb Mailbox, conf config.Account) {

622 if mb.Junk {

623 m.Junk = true

624 m.Notjunk = false

625 return

626 }

627

628 if !conf.AutomaticJunkFlags.Enabled {

629 return

630 }

631

632 lmailbox := strings.ToLower(mb.Name)

633

634 if conf.JunkMailbox != nil && conf.JunkMailbox.MatchString(lmailbox) {

635 m.Junk = true

636 m.Notjunk = false

637 } else if conf.NeutralMailbox != nil && conf.NeutralMailbox.MatchString(lmailbox) {

638 m.Junk = false

639 m.Notjunk = false

640 } else if conf.NotJunkMailbox != nil && conf.NotJunkMailbox.MatchString(lmailbox) {

641 m.Junk = false

642 m.Notjunk = true

643 } else if conf.JunkMailbox == nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox != nil {

644 m.Junk = true

645 m.Notjunk = false

646 } else if conf.JunkMailbox != nil && conf.NeutralMailbox == nil && conf.NotJunkMailbox != nil {

647 m.Junk = false

648 m.Notjunk = false

649 } else if conf.JunkMailbox != nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox == nil {

650 m.Junk = false

651 m.Notjunk = true

652 }

653}

654

655// Recipient represents the recipient of a message. It is tracked to allow

656// first-time incoming replies from users this account has sent messages to. When a

657// mailbox is added to the Sent mailbox the message is parsed and recipients are

658// inserted as recipient. Recipients are never removed other than for removing the

659// message. On move/copy of a message, recipients aren't modified either. For IMAP,

660// this assumes a client simply appends messages to the Sent mailbox (as opposed to

661// copying messages from some place).

662type Recipient struct {

663 ID int64

664 MessageID int64 `bstore:"nonzero,ref Message"` // Ref gives it its own index, useful for fast removal as well.

665 Localpart smtp.Localpart `bstore:"nonzero"`

666 Domain string `bstore:"nonzero,index Domain+Localpart"` // Unicode string.

667 OrgDomain string `bstore:"nonzero,index"` // Unicode string.

668 Sent time.Time `bstore:"nonzero"`

669}

670

671// Outgoing is a message submitted for delivery from the queue. Used to enforce

672// maximum outgoing messages.

673type Outgoing struct {

674 ID int64

675 Recipient string `bstore:"nonzero,index"` // Canonical international address with utf8 domain.

676 Submitted time.Time `bstore:"nonzero,default now"`

677}

678

679// RecipientDomainTLS stores TLS capabilities of a recipient domain as encountered

680// during most recent connection (delivery attempt).

681type RecipientDomainTLS struct {

682 Domain string // Unicode.

683 Updated time.Time `bstore:"default now"`

684 STARTTLS bool // Supports STARTTLS.

685 RequireTLS bool // Supports RequireTLS SMTP extension.

686}

687

688// DiskUsage tracks quota use.

689type DiskUsage struct {

690 ID int64 // Always one record with ID 1.

691 MessageSize int64 // Sum of all messages, for quota accounting.

692}

693

694// SessionToken and CSRFToken are types to prevent mixing them up.

695// Base64 raw url encoded.

696type SessionToken string

697type CSRFToken string

698

699// LoginSession represents a login session. We keep a limited number of sessions

700// for a user, removing the oldest session when a new one is created.

701type LoginSession struct {

702 ID int64

703 Created time.Time `bstore:"nonzero,default now"` // Of original login.

704 Expires time.Time `bstore:"nonzero"` // Extended each time it is used.

705 SessionTokenBinary [16]byte `bstore:"nonzero"` // Stored in cookie, like "webmailsession" or "webaccountsession".

706 CSRFTokenBinary [16]byte // For API requests, in "x-mox-csrf" header.

707 AccountName string `bstore:"nonzero"`

708 LoginAddress string `bstore:"nonzero"`

709

710 // Set when loading from database.

711 sessionToken SessionToken

712 csrfToken CSRFToken

713}

714

715// Types stored in DB.

716var DBTypes = []any{NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{}, SyncState{}, Upgrade{}, RecipientDomainTLS{}, DiskUsage{}, LoginSession{}}

717

718// Account holds the information about a user, includings mailboxes, messages, imap subscriptions.

719type Account struct {

720 Name string // Name, according to configuration.

721 Dir string // Directory where account files, including the database, bloom filter, and mail messages, are stored for this account.

722 DBPath string // Path to database with mailboxes, messages, etc.

723 DB *bstore.DB // Open database connection.

724

725 // Channel that is closed if/when account has/gets "threads" accounting (see

726 // Upgrade.Threads).

727 threadsCompleted chan struct{}

728 // If threads upgrade completed with error, this is set. Used for warning during

729 // delivery, or aborting when importing.

730 threadsErr error

731

732 // Write lock must be held for account/mailbox modifications including message delivery.

733 // Read lock for reading mailboxes/messages.

734 // When making changes to mailboxes/messages, changes must be broadcasted before

735 // releasing the lock to ensure proper UID ordering.

736 sync.RWMutex

737

738 nused int // Reference count, while >0, this account is alive and shared.

739}

740

741type Upgrade struct {

742 ID byte

743 Threads byte // 0: None, 1: Adding MessageID's completed, 2: Adding ThreadID's completed.

744}

745

746// InitialUIDValidity returns a UIDValidity used for initializing an account.

747// It can be replaced during tests with a predictable value.

748var InitialUIDValidity = func() uint32 {

749 return uint32(time.Now().Unix() >> 1) // A 2-second resolution will get us far enough beyond 2038.

750}

751

752var openAccounts = struct {

753 names map[string]*Account

754 sync.Mutex

755}{

756 names: map[string]*Account{},

757}

758

759func closeAccount(acc *Account) (rerr error) {

760 openAccounts.Lock()

761 acc.nused--

762 defer openAccounts.Unlock()

763 if acc.nused == 0 {

764 // threadsCompleted must be closed now because it increased nused.

765 rerr = acc.DB.Close()

766 acc.DB = nil

767 delete(openAccounts.names, acc.Name)

768 }

769 return

770}

771

772// OpenAccount opens an account by name.

773//

774// No additional data path prefix or ".db" suffix should be added to the name.

775// A single shared account exists per name.

776func OpenAccount(log mlog.Log, name string) (*Account, error) {

777 openAccounts.Lock()

778 defer openAccounts.Unlock()

779 if acc, ok := openAccounts.names[name]; ok {

780 acc.nused++

781 return acc, nil

782 }

783

784 if _, ok := mox.Conf.Account(name); !ok {

785 return nil, ErrAccountUnknown

786 }

787

788 acc, err := openAccount(log, name)

789 if err != nil {

790 return nil, err

791 }

792 openAccounts.names[name] = acc

793 return acc, nil

794}

795

796// openAccount opens an existing account, or creates it if it is missing.

797func openAccount(log mlog.Log, name string) (a *Account, rerr error) {

798 dir := filepath.Join(mox.DataDirPath("accounts"), name)

799 return OpenAccountDB(log, dir, name)

800}

801

802// OpenAccountDB opens an account database file and returns an initialized account

803// or error. Only exported for use by subcommands that verify the database file.

804// Almost all account opens must go through OpenAccount/OpenEmail/OpenEmailAuth.

805func OpenAccountDB(log mlog.Log, accountDir, accountName string) (a *Account, rerr error) {

806 dbpath := filepath.Join(accountDir, "index.db")

807

808 // Create account if it doesn't exist yet.

809 isNew := false

810 if _, err := os.Stat(dbpath); err != nil && os.IsNotExist(err) {

811 isNew = true

812 os.MkdirAll(accountDir, 0770)

813 }

814

815 db, err := bstore.Open(context.TODO(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)

816 if err != nil {

817 return nil, err

818 }

819

820 defer func() {

821 if rerr != nil {

822 db.Close()

823 if isNew {

824 os.Remove(dbpath)

825 }

826 }

827 }()

828

829 acc := &Account{

830 Name: accountName,

831 Dir: accountDir,

832 DBPath: dbpath,

833 DB: db,

834 nused: 1,

835 threadsCompleted: make(chan struct{}),

836 }

837

838 if isNew {

839 if err := initAccount(db); err != nil {

840 return nil, fmt.Errorf("initializing account: %v", err)

841 }

842 close(acc.threadsCompleted)

843 return acc, nil

844 }

845

846 // Ensure mailbox counts and total message size are set.

847 var mentioned bool

848 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

849 err := bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error {

850 if !mentioned {

851 mentioned = true

852 log.Info("first calculation of mailbox counts for account", slog.String("account", accountName))

853 }

854 mc, err := mb.CalculateCounts(tx)

855 if err != nil {

856 return err

857 }

858 mb.HaveCounts = true

859 mb.MailboxCounts = mc

860 return tx.Update(&mb)

861 })

862 if err != nil {

863 return err

864 }

865

866 du := DiskUsage{ID: 1}

867 err = tx.Get(&du)

868 if err == nil || !errors.Is(err, bstore.ErrAbsent) {

869 return err

870 }

871 // No DiskUsage record yet, calculate total size and insert.

872 err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {

873 du.MessageSize += mb.Size

874 return nil

875 })

876 if err != nil {

877 return err

878 }

879 return tx.Insert(&du)

880 })

881 if err != nil {

882 return nil, fmt.Errorf("calculating counts for mailbox: %v", err)

883 }

884

885 // Start adding threading if needed.

886 up := Upgrade{ID: 1}

887 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

888 err := tx.Get(&up)

889 if err == bstore.ErrAbsent {

890 if err := tx.Insert(&up); err != nil {

891 return fmt.Errorf("inserting initial upgrade record: %v", err)

892 }

893 err = nil

894 }

895 return err

896 })

897 if err != nil {

898 return nil, fmt.Errorf("checking message threading: %v", err)

899 }

900 if up.Threads == 2 {

901 close(acc.threadsCompleted)

902 return acc, nil

903 }

904

905 // Increase account use before holding on to account in background.

906 // Caller holds the lock. The goroutine below decreases nused by calling

907 // closeAccount.

908 acc.nused++

909

910 // Ensure all messages have a MessageID and SubjectBase, which are needed when

911 // matching threads.

912 // Then assign messages to threads, in the same way we do during imports.

913 log.Info("upgrading account for threading, in background", slog.String("account", acc.Name))

914 go func() {

915 defer func() {

916 err := closeAccount(acc)

917 log.Check(err, "closing use of account after upgrading account storage for threads", slog.String("account", a.Name))

918 }()

919

920 defer func() {

921 x := recover() // Should not happen, but don't take program down if it does.

922 if x != nil {

923 log.Error("upgradeThreads panic", slog.Any("err", x))

924 debug.PrintStack()

925 metrics.PanicInc(metrics.Upgradethreads)

926 acc.threadsErr = fmt.Errorf("panic during upgradeThreads: %v", x)

927 }

928

929 // Mark that upgrade has finished, possibly error is indicated in threadsErr.

930 close(acc.threadsCompleted)

931 }()

932

933 err := upgradeThreads(mox.Shutdown, log, acc, &up)

934 if err != nil {

935 a.threadsErr = err

936 log.Errorx("upgrading account for threading, aborted", err, slog.String("account", a.Name))

937 } else {

938 log.Info("upgrading account for threading, completed", slog.String("account", a.Name))

939 }

940 }()

941 return acc, nil

942}

943

944// ThreadingWait blocks until the one-time account threading upgrade for the

945// account has completed, and returns an error if not successful.

946//

947// To be used before starting an import of messages.

948func (a *Account) ThreadingWait(log mlog.Log) error {

949 select {

950 case <-a.threadsCompleted:

951 return a.threadsErr

952 default:

953 }

954 log.Debug("waiting for account upgrade to complete")

955

956 <-a.threadsCompleted

957 return a.threadsErr

958}

959

960func initAccount(db *bstore.DB) error {

961 return db.Write(context.TODO(), func(tx *bstore.Tx) error {

962 uidvalidity := InitialUIDValidity()

963

964 if err := tx.Insert(&Upgrade{ID: 1, Threads: 2}); err != nil {

965 return err

966 }

967 if err := tx.Insert(&DiskUsage{ID: 1}); err != nil {

968 return err

969 }

970

971 if len(mox.Conf.Static.DefaultMailboxes) > 0 {

972 // Deprecated in favor of InitialMailboxes.

973 defaultMailboxes := mox.Conf.Static.DefaultMailboxes

974 mailboxes := []string{"Inbox"}

975 for _, name := range defaultMailboxes {

976 if strings.EqualFold(name, "Inbox") {

977 continue

978 }

979 mailboxes = append(mailboxes, name)

980 }

981 for _, name := range mailboxes {

982 mb := Mailbox{Name: name, UIDValidity: uidvalidity, UIDNext: 1, HaveCounts: true}

983 if strings.HasPrefix(name, "Archive") {

984 mb.Archive = true

985 } else if strings.HasPrefix(name, "Drafts") {

986 mb.Draft = true

987 } else if strings.HasPrefix(name, "Junk") {

988 mb.Junk = true

989 } else if strings.HasPrefix(name, "Sent") {

990 mb.Sent = true

991 } else if strings.HasPrefix(name, "Trash") {

992 mb.Trash = true

993 }

994 if err := tx.Insert(&mb); err != nil {

995 return fmt.Errorf("creating mailbox: %w", err)

996 }

997 if err := tx.Insert(&Subscription{name}); err != nil {

998 return fmt.Errorf("adding subscription: %w", err)

999 }

1000 }

1001 } else {

1002 mailboxes := mox.Conf.Static.InitialMailboxes

1003 var zerouse config.SpecialUseMailboxes

1004 if mailboxes.SpecialUse == zerouse && len(mailboxes.Regular) == 0 {

1005 mailboxes = DefaultInitialMailboxes

1006 }

1007

1008 add := func(name string, use SpecialUse) error {

1009 mb := Mailbox{Name: name, UIDValidity: uidvalidity, UIDNext: 1, SpecialUse: use, HaveCounts: true}

1010 if err := tx.Insert(&mb); err != nil {

1011 return fmt.Errorf("creating mailbox: %w", err)

1012 }

1013 if err := tx.Insert(&Subscription{name}); err != nil {

1014 return fmt.Errorf("adding subscription: %w", err)

1015 }

1016 return nil

1017 }

1018 addSpecialOpt := func(nameOpt string, use SpecialUse) error {

1019 if nameOpt == "" {

1020 return nil

1021 }

1022 return add(nameOpt, use)

1023 }

1024 l := []struct {

1025 nameOpt string

1026 use SpecialUse

1027 }{

1028 {"Inbox", SpecialUse{}},

1029 {mailboxes.SpecialUse.Archive, SpecialUse{Archive: true}},

1030 {mailboxes.SpecialUse.Draft, SpecialUse{Draft: true}},

1031 {mailboxes.SpecialUse.Junk, SpecialUse{Junk: true}},

1032 {mailboxes.SpecialUse.Sent, SpecialUse{Sent: true}},

1033 {mailboxes.SpecialUse.Trash, SpecialUse{Trash: true}},

1034 }

1035 for _, e := range l {

1036 if err := addSpecialOpt(e.nameOpt, e.use); err != nil {

1037 return err

1038 }

1039 }

1040 for _, name := range mailboxes.Regular {

1041 if err := add(name, SpecialUse{}); err != nil {

1042 return err

1043 }

1044 }

1045 }

1046

1047 uidvalidity++

1048 if err := tx.Insert(&NextUIDValidity{1, uidvalidity}); err != nil {

1049 return fmt.Errorf("inserting nextuidvalidity: %w", err)

1050 }

1051 return nil

1052 })

1053}

1054

1055// Close reduces the reference count, and closes the database connection when

1056// it was the last user.

1057func (a *Account) Close() error {

1058 if CheckConsistencyOnClose {

1059 xerr := a.CheckConsistency()

1060 err := closeAccount(a)

1061 if xerr != nil {

1062 panic(xerr)

1063 }

1064 return err

1065 }

1066 return closeAccount(a)

1067}

1068

1069// CheckConsistency checks the consistency of the database and returns a non-nil

1070// error for these cases:

1071//

1072// - Missing on-disk file for message.

1073// - Mismatch between message size and length of MsgPrefix and on-disk file.

1074// - Missing HaveCounts.

1075// - Incorrect mailbox counts.

1076// - Incorrect total message size.

1077// - Message with UID >= mailbox uid next.

1078// - Mailbox uidvalidity >= account uid validity.

1079// - ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq.

1080// - All messages have a nonzero ThreadID, and no cycles in ThreadParentID, and parent messages the same ThreadParentIDs tail.

1081func (a *Account) CheckConsistency() error {

1082 var uidErrors []string // With a limit, could be many.

1083 var modseqErrors []string // With limit.

1084 var fileErrors []string // With limit.

1085 var threadidErrors []string // With limit.

1086 var threadParentErrors []string // With limit.

1087 var threadAncestorErrors []string // With limit.

1088 var errors []string

1089

1090 err := a.DB.Read(context.Background(), func(tx *bstore.Tx) error {

1091 nuv := NextUIDValidity{ID: 1}

1092 err := tx.Get(&nuv)

1093 if err != nil {

1094 return fmt.Errorf("fetching next uid validity: %v", err)

1095 }

1096

1097 mailboxes := map[int64]Mailbox{}

1098 err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {

1099 mailboxes[mb.ID] = mb

1100

1101 if mb.UIDValidity >= nuv.Next {

1102 errmsg := fmt.Sprintf("mailbox %q (id %d) has uidvalidity %d >= account next uidvalidity %d", mb.Name, mb.ID, mb.UIDValidity, nuv.Next)

1103 errors = append(errors, errmsg)

1104 }

1105 return nil

1106 })

1107 if err != nil {

1108 return fmt.Errorf("listing mailboxes: %v", err)

1109 }

1110

1111 counts := map[int64]MailboxCounts{}

1112 err = bstore.QueryTx[Message](tx).ForEach(func(m Message) error {

1113 mc := counts[m.MailboxID]

1114 mc.Add(m.MailboxCounts())

1115 counts[m.MailboxID] = mc

1116

1117 mb := mailboxes[m.MailboxID]

1118

1119 if (m.ModSeq == 0 || m.CreateSeq == 0 || m.CreateSeq > m.ModSeq) && len(modseqErrors) < 20 {

1120 modseqerr := fmt.Sprintf("message %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and createseq <= modseq", m.ID, mb.Name, mb.ID, m.ModSeq, m.CreateSeq)

1121 modseqErrors = append(modseqErrors, modseqerr)

1122 }

1123 if m.UID >= mb.UIDNext && len(uidErrors) < 20 {

1124 uiderr := fmt.Sprintf("message %d in mailbox %q (id %d) has uid %d >= mailbox uidnext %d", m.ID, mb.Name, mb.ID, m.UID, mb.UIDNext)

1125 uidErrors = append(uidErrors, uiderr)

1126 }

1127 if m.Expunged {

1128 return nil

1129 }

1130 p := a.MessagePath(m.ID)

1131 st, err := os.Stat(p)

1132 if err != nil {

1133 existserr := fmt.Sprintf("message %d in mailbox %q (id %d) on-disk file %s: %v", m.ID, mb.Name, mb.ID, p, err)

1134 fileErrors = append(fileErrors, existserr)

1135 } else if len(fileErrors) < 20 && m.Size != int64(len(m.MsgPrefix))+st.Size() {

1136 sizeerr := fmt.Sprintf("message %d in mailbox %q (id %d) has size %d != len msgprefix %d + on-disk file size %d = %d", m.ID, mb.Name, mb.ID, m.Size, len(m.MsgPrefix), st.Size(), int64(len(m.MsgPrefix))+st.Size())

1137 fileErrors = append(fileErrors, sizeerr)

1138 }

1139

1140 if m.ThreadID <= 0 && len(threadidErrors) < 20 {

1141 err := fmt.Sprintf("message %d in mailbox %q (id %d) has threadid 0", m.ID, mb.Name, mb.ID)

1142 threadidErrors = append(threadidErrors, err)

1143 }

1144 if slices.Contains(m.ThreadParentIDs, m.ID) && len(threadParentErrors) < 20 {

1145 err := fmt.Sprintf("message %d in mailbox %q (id %d) references itself in threadparentids", m.ID, mb.Name, mb.ID)

1146 threadParentErrors = append(threadParentErrors, err)

1147 }

1148 for i, pid := range m.ThreadParentIDs {

1149 am := Message{ID: pid}

1150 if err := tx.Get(&am); err == bstore.ErrAbsent {

1151 continue

1152 } else if err != nil {

1153 return fmt.Errorf("get ancestor message: %v", err)

1154 } else if !slices.Equal(m.ThreadParentIDs[i+1:], am.ThreadParentIDs) && len(threadAncestorErrors) < 20 {

1155 err := fmt.Sprintf("message %d, thread %d has ancestor ids %v, and ancestor at index %d with id %d should have the same tail but has %v\n", m.ID, m.ThreadID, m.ThreadParentIDs, i, am.ID, am.ThreadParentIDs)

1156 threadAncestorErrors = append(threadAncestorErrors, err)

1157 } else {

1158 break

1159 }

1160 }

1161 return nil

1162 })

1163 if err != nil {

1164 return fmt.Errorf("reading messages: %v", err)

1165 }

1166

1167 var totalSize int64

1168 for _, mb := range mailboxes {

1169 totalSize += mb.Size

1170 if !mb.HaveCounts {

1171 errmsg := fmt.Sprintf("mailbox %q (id %d) does not have counts, should be %#v", mb.Name, mb.ID, counts[mb.ID])

1172 errors = append(errors, errmsg)

1173 } else if mb.MailboxCounts != counts[mb.ID] {

1174 mbcounterr := fmt.Sprintf("mailbox %q (id %d) has wrong counts %s, should be %s", mb.Name, mb.ID, mb.MailboxCounts, counts[mb.ID])

1175 errors = append(errors, mbcounterr)

1176 }

1177 }

1178

1179 du := DiskUsage{ID: 1}

1180 if err := tx.Get(&du); err != nil {

1181 return fmt.Errorf("get diskusage")

1182 }

1183 if du.MessageSize != totalSize {

1184 errmsg := fmt.Sprintf("total message size in database is %d, sum of mailbox message sizes is %d", du.MessageSize, totalSize)

1185 errors = append(errors, errmsg)

1186 }

1187

1188 return nil

1189 })

1190 if err != nil {

1191 return err

1192 }

1193 errors = append(errors, uidErrors...)

1194 errors = append(errors, modseqErrors...)

1195 errors = append(errors, fileErrors...)

1196 errors = append(errors, threadidErrors...)

1197 errors = append(errors, threadParentErrors...)

1198 errors = append(errors, threadAncestorErrors...)

1199 if len(errors) > 0 {

1200 return fmt.Errorf("%s", strings.Join(errors, "; "))

1201 }

1202 return nil

1203}

1204

1205// Conf returns the configuration for this account if it still exists. During

1206// an SMTP session, a configuration update may drop an account.

1207func (a *Account) Conf() (config.Account, bool) {

1208 return mox.Conf.Account(a.Name)

1209}

1210

1211// NextUIDValidity returns the next new/unique uidvalidity to use for this account.

1212func (a *Account) NextUIDValidity(tx *bstore.Tx) (uint32, error) {

1213 nuv := NextUIDValidity{ID: 1}

1214 if err := tx.Get(&nuv); err != nil {

1215 return 0, err

1216 }

1217 v := nuv.Next

1218 nuv.Next++

1219 if err := tx.Update(&nuv); err != nil {

1220 return 0, err

1221 }

1222 return v, nil

1223}

1224

1225// NextModSeq returns the next modification sequence, which is global per account,

1226// over all types.

1227func (a *Account) NextModSeq(tx *bstore.Tx) (ModSeq, error) {

1228 v := SyncState{ID: 1}

1229 if err := tx.Get(&v); err == bstore.ErrAbsent {

1230 // We start assigning from modseq 2. Modseq 0 is not usable, so returned as 1, so

1231 // already used.

1232 // HighestDeletedModSeq is -1 so comparison against the default ModSeq zero value

1233 // makes sense.

1234 v = SyncState{1, 2, -1}

1235 return v.LastModSeq, tx.Insert(&v)

1236 } else if err != nil {

1237 return 0, err

1238 }

1239 v.LastModSeq++

1240 return v.LastModSeq, tx.Update(&v)

1241}

1242

1243func (a *Account) HighestDeletedModSeq(tx *bstore.Tx) (ModSeq, error) {

1244 v := SyncState{ID: 1}

1245 err := tx.Get(&v)

1246 if err == bstore.ErrAbsent {

1247 return 0, nil

1248 }

1249 return v.HighestDeletedModSeq, err

1250}

1251

1252// WithWLock runs fn with account writelock held. Necessary for account/mailbox modification. For message delivery, a read lock is required.

1253func (a *Account) WithWLock(fn func()) {

1254 a.Lock()

1255 defer a.Unlock()

1256 fn()

1257}

1258

1259// WithRLock runs fn with account read lock held. Needed for message delivery.

1260func (a *Account) WithRLock(fn func()) {

1261 a.RLock()

1262 defer a.RUnlock()

1263 fn()

1264}

1265

1266// DeliverMessage delivers a mail message to the account.

1267//

1268// The message, with msg.MsgPrefix and msgFile combined, must have a header

1269// section. The caller is responsible for adding a header separator to

1270// msg.MsgPrefix if missing from an incoming message.

1271//

1272// If the destination mailbox has the Sent special-use flag, the message is parsed

1273// for its recipients (to/cc/bcc). Their domains are added to Recipients for use in

1274// dmarc reputation.

1275//

1276// If sync is true, the message file and its directory are synced. Should be true

1277// for regular mail delivery, but can be false when importing many messages.

1278//

1279// If updateDiskUsage is true, the account total message size (for quota) is

1280// updated. Callers must check if a message can be added within quota before

1281// calling DeliverMessage.

1282//

1283// If CreateSeq/ModSeq is not set, it is assigned automatically.

1284//

1285// Must be called with account rlock or wlock.

1286//

1287// Caller must broadcast new message.

1288//

1289// Caller must update mailbox counts.

1290func (a *Account) DeliverMessage(log mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, sync, notrain, nothreads, updateDiskUsage bool) error {

1291 if m.Expunged {

1292 return fmt.Errorf("cannot deliver expunged message")

1293 }

1294

1295 mb := Mailbox{ID: m.MailboxID}

1296 if err := tx.Get(&mb); err != nil {

1297 return fmt.Errorf("get mailbox: %w", err)

1298 }

1299 m.UID = mb.UIDNext

1300 mb.UIDNext++

1301 if err := tx.Update(&mb); err != nil {

1302 return fmt.Errorf("updating mailbox nextuid: %w", err)

1303 }

1304

1305 if updateDiskUsage {

1306 du := DiskUsage{ID: 1}

1307 if err := tx.Get(&du); err != nil {

1308 return fmt.Errorf("get disk usage: %v", err)

1309 }

1310 du.MessageSize += m.Size

1311 if err := tx.Update(&du); err != nil {

1312 return fmt.Errorf("update disk usage: %v", err)

1313 }

1314 }

1315

1316 conf, _ := a.Conf()

1317 m.JunkFlagsForMailbox(mb, conf)

1318

1319 mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile.

1320 var part *message.Part

1321 if m.ParsedBuf == nil {

1322 p, err := message.EnsurePart(log.Logger, false, mr, m.Size)

1323 if err != nil {

1324 log.Infox("parsing delivered message", err, slog.String("parse", ""), slog.Int64("message", m.ID))

1325 // We continue, p is still valid.

1326 }

1327 part = &p

1328 buf, err := json.Marshal(part)

1329 if err != nil {

1330 return fmt.Errorf("marshal parsed message: %w", err)

1331 }

1332 m.ParsedBuf = buf

1333 } else {

1334 var p message.Part

1335 if err := json.Unmarshal(m.ParsedBuf, &p); err != nil {

1336 log.Errorx("unmarshal parsed message, continuing", err, slog.String("parse", ""))

1337 } else {

1338 part = &p

1339 }

1340 }

1341

1342 // If we are delivering to the originally intended mailbox, no need to store the mailbox ID again.

1343 if m.MailboxDestinedID != 0 && m.MailboxDestinedID == m.MailboxOrigID {

1344 m.MailboxDestinedID = 0

1345 }

1346 if m.CreateSeq == 0 || m.ModSeq == 0 {

1347 modseq, err := a.NextModSeq(tx)

1348 if err != nil {

1349 return fmt.Errorf("assigning next modseq: %w", err)

1350 }

1351 m.CreateSeq = modseq

1352 m.ModSeq = modseq

1353 }

1354

1355 if part != nil && m.MessageID == "" && m.SubjectBase == "" {

1356 m.PrepareThreading(log, part)

1357 }

1358

1359 // Assign to thread (if upgrade has completed).

1360 noThreadID := nothreads

1361 if m.ThreadID == 0 && !nothreads && part != nil {

1362 select {

1363 case <-a.threadsCompleted:

1364 if a.threadsErr != nil {

1365 log.Info("not assigning threads for new delivery, upgrading to threads failed")

1366 noThreadID = true

1367 } else {

1368 if err := assignThread(log, tx, m, part); err != nil {

1369 return fmt.Errorf("assigning thread: %w", err)

1370 }

1371 }

1372 default:

1373 // note: since we have a write transaction to get here, we can't wait for the

1374 // thread upgrade to finish.

1375 // If we don't assign a threadid the upgrade process will do it.

1376 log.Info("not assigning threads for new delivery, upgrading to threads in progress which will assign this message")

1377 noThreadID = true

1378 }

1379 }

1380

1381 if err := tx.Insert(m); err != nil {

1382 return fmt.Errorf("inserting message: %w", err)

1383 }

1384 if !noThreadID && m.ThreadID == 0 {

1385 m.ThreadID = m.ID

1386 if err := tx.Update(m); err != nil {

1387 return fmt.Errorf("updating message for its own thread id: %w", err)

1388 }

1389 }

1390

1391 // todo: perhaps we should match the recipients based on smtp submission and a matching message-id? we now miss the addresses in bcc's. for webmail, we could insert the recipients directly.

1392 if mb.Sent && part != nil && part.Envelope != nil {

1393 e := part.Envelope

1394 sent := e.Date

1395 if sent.IsZero() {

1396 sent = m.Received

1397 }

1398 if sent.IsZero() {

1399 sent = time.Now()

1400 }

1401 addrs := append(append(e.To, e.CC...), e.BCC...)

1402 for _, addr := range addrs {

1403 if addr.User == "" {

1404 // Would trigger error because Recipient.Localpart must be nonzero. todo: we could allow empty localpart in db, and filter by not using FilterNonzero.

1405 log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", slog.Any("address", addr))

1406 continue

1407 }

1408 d, err := dns.ParseDomain(addr.Host)

1409 if err != nil {

1410 log.Debugx("parsing domain in to/cc/bcc address", err, slog.Any("address", addr))

1411 continue

1412 }

1413 mr := Recipient{

1414 MessageID: m.ID,

1415 Localpart: smtp.Localpart(addr.User),

1416 Domain: d.Name(),

1417 OrgDomain: publicsuffix.Lookup(context.TODO(), log.Logger, d).Name(),

1418 Sent: sent,

1419 }

1420 if err := tx.Insert(&mr); err != nil {

1421 return fmt.Errorf("inserting sent message recipients: %w", err)

1422 }

1423 }

1424 }

1425

1426 msgPath := a.MessagePath(m.ID)

1427 msgDir := filepath.Dir(msgPath)

1428 os.MkdirAll(msgDir, 0770)

1429

1430 // Sync file data to disk.

1431 if sync {

1432 if err := msgFile.Sync(); err != nil {

1433 return fmt.Errorf("fsync message file: %w", err)

1434 }

1435 }

1436

1437 if err := moxio.LinkOrCopy(log, msgPath, msgFile.Name(), &moxio.AtReader{R: msgFile}, true); err != nil {

1438 return fmt.Errorf("linking/copying message to new file: %w", err)

1439 }

1440

1441 if sync {

1442 if err := moxio.SyncDir(log, msgDir); err != nil {

1443 xerr := os.Remove(msgPath)

1444 log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath))

1445 return fmt.Errorf("sync directory: %w", err)

1446 }

1447 }

1448

1449 if !notrain && m.NeedsTraining() {

1450 l := []Message{*m}

1451 if err := a.RetrainMessages(context.TODO(), log, tx, l, false); err != nil {

1452 xerr := os.Remove(msgPath)

1453 log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath))

1454 return fmt.Errorf("training junkfilter: %w", err)

1455 }

1456 *m = l[0]

1457 }

1458

1459 return nil

1460}

1461

1462// SetPassword saves a new password for this account. This password is used for

1463// IMAP, SMTP (submission) sessions and the HTTP account web page.

1464func (a *Account) SetPassword(log mlog.Log, password string) error {

1465 hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)

1466 if err != nil {

1467 return fmt.Errorf("generating password hash: %w", err)

1468 }

1469

1470 err = a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1471 if _, err := bstore.QueryTx[Password](tx).Delete(); err != nil {

1472 return fmt.Errorf("deleting existing password: %v", err)

1473 }

1474 var pw Password

1475 pw.Hash = string(hash)

1476

1477 // CRAM-MD5 calculates an HMAC-MD5, with the password as key, over a per-attempt

1478 // unique text that includes a timestamp. HMAC performs two hashes. Both times, the

1479 // first block is based on the key/password. We hash those first blocks now, and

1480 // store the hash state in the database. When we actually authenticate, we'll

1481 // complete the HMAC by hashing only the text. We cannot store crypto/hmac's hash,

1482 // because it does not expose its internal state and isn't a BinaryMarshaler.

1483 // ../rfc/2104:121

1484 pw.CRAMMD5.Ipad = md5.New()

1485 pw.CRAMMD5.Opad = md5.New()

1486 key := []byte(password)

1487 if len(key) > 64 {

1488 t := md5.Sum(key)

1489 key = t[:]

1490 }

1491 ipad := make([]byte, md5.BlockSize)

1492 opad := make([]byte, md5.BlockSize)

1493 copy(ipad, key)

1494 copy(opad, key)

1495 for i := range ipad {

1496 ipad[i] ^= 0x36

1497 opad[i] ^= 0x5c

1498 }

1499 pw.CRAMMD5.Ipad.Write(ipad)

1500 pw.CRAMMD5.Opad.Write(opad)

1501

1502 pw.SCRAMSHA1.Salt = scram.MakeRandom()

1503 pw.SCRAMSHA1.Iterations = 2 * 4096

1504 pw.SCRAMSHA1.SaltedPassword = scram.SaltPassword(sha1.New, password, pw.SCRAMSHA1.Salt, pw.SCRAMSHA1.Iterations)

1505

1506 pw.SCRAMSHA256.Salt = scram.MakeRandom()

1507 pw.SCRAMSHA256.Iterations = 4096

1508 pw.SCRAMSHA256.SaltedPassword = scram.SaltPassword(sha256.New, password, pw.SCRAMSHA256.Salt, pw.SCRAMSHA256.Iterations)

1509

1510 if err := tx.Insert(&pw); err != nil {

1511 return fmt.Errorf("inserting new password: %v", err)

1512 }

1513

1514 return sessionRemoveAll(context.TODO(), log, tx, a.Name)

1515 })

1516 if err == nil {

1517 log.Info("new password set for account", slog.String("account", a.Name))

1518 }

1519 return err

1520}

1521

1522// Subjectpass returns the signing key for use with subjectpass for the given

1523// email address with canonical localpart.

1524func (a *Account) Subjectpass(email string) (key string, err error) {

1525 return key, a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1526 v := Subjectpass{Email: email}

1527 err := tx.Get(&v)

1528 if err == nil {

1529 key = v.Key

1530 return nil

1531 }

1532 if !errors.Is(err, bstore.ErrAbsent) {

1533 return fmt.Errorf("get subjectpass key from accounts database: %w", err)

1534 }

1535 key = ""

1536 const chars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"

1537 buf := make([]byte, 16)

1538 if _, err := cryptorand.Read(buf); err != nil {

1539 return err

1540 }

1541 for _, b := range buf {

1542 key += string(chars[int(b)%len(chars)])

1543 }

1544 v.Key = key

1545 return tx.Insert(&v)

1546 })

1547}

1548

1549// Ensure mailbox is present in database, adding records for the mailbox and its

1550// parents if they aren't present.

1551//

1552// If subscribe is true, any mailboxes that were created will also be subscribed to.

1553// Caller must hold account wlock.

1554// Caller must propagate changes if any.

1555func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool) (mb Mailbox, changes []Change, rerr error) {

1556 if norm.NFC.String(name) != name {

1557 return Mailbox{}, nil, fmt.Errorf("mailbox name not normalized")

1558 }

1559

1560 // Quick sanity check.

1561 if strings.EqualFold(name, "inbox") && name != "Inbox" {

1562 return Mailbox{}, nil, fmt.Errorf("bad casing for inbox")

1563 }

1564

1565 elems := strings.Split(name, "/")

1566 q := bstore.QueryTx[Mailbox](tx)

1567 q.FilterFn(func(mb Mailbox) bool {

1568 return mb.Name == elems[0] || strings.HasPrefix(mb.Name, elems[0]+"/")

1569 })

1570 l, err := q.List()

1571 if err != nil {

1572 return Mailbox{}, nil, fmt.Errorf("list mailboxes: %v", err)

1573 }

1574

1575 mailboxes := map[string]Mailbox{}

1576 for _, xmb := range l {

1577 mailboxes[xmb.Name] = xmb

1578 }

1579

1580 p := ""

1581 for _, elem := range elems {

1582 if p != "" {

1583 p += "/"

1584 }

1585 p += elem

1586 var ok bool

1587 mb, ok = mailboxes[p]

1588 if ok {

1589 continue

1590 }

1591 uidval, err := a.NextUIDValidity(tx)

1592 if err != nil {

1593 return Mailbox{}, nil, fmt.Errorf("next uid validity: %v", err)

1594 }

1595 mb = Mailbox{

1596 Name: p,

1597 UIDValidity: uidval,

1598 UIDNext: 1,

1599 HaveCounts: true,

1600 }

1601 err = tx.Insert(&mb)

1602 if err != nil {

1603 return Mailbox{}, nil, fmt.Errorf("creating new mailbox: %v", err)

1604 }

1605

1606 var flags []string

1607 if subscribe {

1608 if tx.Get(&Subscription{p}) != nil {

1609 err := tx.Insert(&Subscription{p})

1610 if err != nil {

1611 return Mailbox{}, nil, fmt.Errorf("subscribing to mailbox: %v", err)

1612 }

1613 }

1614 flags = []string{`\Subscribed`}

1615 }

1616 changes = append(changes, ChangeAddMailbox{mb, flags})

1617 }

1618 return mb, changes, nil

1619}

1620

1621// MailboxExists checks if mailbox exists.

1622// Caller must hold account rlock.

1623func (a *Account) MailboxExists(tx *bstore.Tx, name string) (bool, error) {

1624 q := bstore.QueryTx[Mailbox](tx)

1625 q.FilterEqual("Name", name)

1626 return q.Exists()

1627}

1628

1629// MailboxFind finds a mailbox by name, returning a nil mailbox and nil error if mailbox does not exist.

1630func (a *Account) MailboxFind(tx *bstore.Tx, name string) (*Mailbox, error) {

1631 q := bstore.QueryTx[Mailbox](tx)

1632 q.FilterEqual("Name", name)

1633 mb, err := q.Get()

1634 if err == bstore.ErrAbsent {

1635 return nil, nil

1636 }

1637 if err != nil {

1638 return nil, fmt.Errorf("looking up mailbox: %w", err)

1639 }

1640 return &mb, nil

1641}

1642

1643// SubscriptionEnsure ensures a subscription for name exists. The mailbox does not

1644// have to exist. Any parents are not automatically subscribed.

1645// Changes are returned and must be broadcasted by the caller.

1646func (a *Account) SubscriptionEnsure(tx *bstore.Tx, name string) ([]Change, error) {

1647 if err := tx.Get(&Subscription{name}); err == nil {

1648 return nil, nil

1649 }

1650

1651 if err := tx.Insert(&Subscription{name}); err != nil {

1652 return nil, fmt.Errorf("inserting subscription: %w", err)

1653 }

1654

1655 q := bstore.QueryTx[Mailbox](tx)

1656 q.FilterEqual("Name", name)

1657 _, err := q.Get()

1658 if err == nil {

1659 return []Change{ChangeAddSubscription{name, nil}}, nil

1660 } else if err != bstore.ErrAbsent {

1661 return nil, fmt.Errorf("looking up mailbox for subscription: %w", err)

1662 }

1663 return []Change{ChangeAddSubscription{name, []string{`\NonExistent`}}}, nil

1664}

1665

1666// MessageRuleset returns the first ruleset (if any) that message the message

1667// represented by msgPrefix and msgFile, with smtp and validation fields from m.

1668func MessageRuleset(log mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset {

1669 if len(dest.Rulesets) == 0 {

1670 return nil

1671 }

1672

1673 mr := FileMsgReader(msgPrefix, msgFile) // We don't close, it would close the msgFile.

1674 p, err := message.Parse(log.Logger, false, mr)

1675 if err != nil {

1676 log.Errorx("parsing message for evaluating rulesets, continuing with headers", err, slog.String("parse", ""))

1677 // note: part is still set.

1678 }

1679 // todo optimize: only parse header if needed for rulesets. and probably reuse an earlier parsing.

1680 header, err := p.Header()

1681 if err != nil {

1682 log.Errorx("parsing message headers for evaluating rulesets, delivering to default mailbox", err, slog.String("parse", ""))

1683 // todo: reject message?

1684 return nil

1685 }

1686

1687ruleset:

1688 for _, rs := range dest.Rulesets {

1689 if rs.SMTPMailFromRegexpCompiled != nil {

1690 if !rs.SMTPMailFromRegexpCompiled.MatchString(m.MailFrom) {

1691 continue ruleset

1692 }

1693 }

1694

1695 if !rs.VerifiedDNSDomain.IsZero() {

1696 d := rs.VerifiedDNSDomain.Name()

1697 suffix := "." + d

1698 matchDomain := func(s string) bool {

1699 return s == d || strings.HasSuffix(s, suffix)

1700 }

1701 var ok bool

1702 if m.EHLOValidated && matchDomain(m.EHLODomain) {

1703 ok = true

1704 }

1705 if m.MailFromValidated && matchDomain(m.MailFromDomain) {

1706 ok = true

1707 }

1708 for _, d := range m.DKIMDomains {

1709 if matchDomain(d) {

1710 ok = true

1711 break

1712 }

1713 }

1714 if !ok {

1715 continue ruleset

1716 }

1717 }

1718

1719 header:

1720 for _, t := range rs.HeadersRegexpCompiled {

1721 for k, vl := range header {

1722 k = strings.ToLower(k)

1723 if !t[0].MatchString(k) {

1724 continue

1725 }

1726 for _, v := range vl {

1727 v = strings.ToLower(strings.TrimSpace(v))

1728 if t[1].MatchString(v) {

1729 continue header

1730 }

1731 }

1732 }

1733 continue ruleset

1734 }

1735 return &rs

1736 }

1737 return nil

1738}

1739

1740// MessagePath returns the file system path of a message.

1741func (a *Account) MessagePath(messageID int64) string {

1742 return strings.Join(append([]string{a.Dir, "msg"}, messagePathElems(messageID)...), string(filepath.Separator))

1743}

1744

1745// MessageReader opens a message for reading, transparently combining the

1746// message prefix with the original incoming message.

1747func (a *Account) MessageReader(m Message) *MsgReader {

1748 return &MsgReader{prefix: m.MsgPrefix, path: a.MessagePath(m.ID), size: m.Size}

1749}

1750

1751// DeliverDestination delivers an email to dest, based on the configured rulesets.

1752//

1753// Returns ErrOverQuota when account would be over quota after adding message.

1754//

1755// Caller must hold account wlock (mailbox may be created).

1756// Message delivery, possible mailbox creation, and updated mailbox counts are

1757// broadcasted.

1758func (a *Account) DeliverDestination(log mlog.Log, dest config.Destination, m *Message, msgFile *os.File) error {

1759 var mailbox string

1760 rs := MessageRuleset(log, dest, m, m.MsgPrefix, msgFile)

1761 if rs != nil {

1762 mailbox = rs.Mailbox

1763 } else if dest.Mailbox == "" {

1764 mailbox = "Inbox"

1765 } else {

1766 mailbox = dest.Mailbox

1767 }

1768 return a.DeliverMailbox(log, mailbox, m, msgFile)

1769}

1770

1771// DeliverMailbox delivers an email to the specified mailbox.

1772//

1773// Returns ErrOverQuota when account would be over quota after adding message.

1774//

1775// Caller must hold account wlock (mailbox may be created).

1776// Message delivery, possible mailbox creation, and updated mailbox counts are

1777// broadcasted.

1778func (a *Account) DeliverMailbox(log mlog.Log, mailbox string, m *Message, msgFile *os.File) error {

1779 var changes []Change

1780 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1781 if ok, _, err := a.CanAddMessageSize(tx, m.Size); err != nil {

1782 return err

1783 } else if !ok {

1784 return ErrOverQuota

1785 }

1786

1787 mb, chl, err := a.MailboxEnsure(tx, mailbox, true)

1788 if err != nil {

1789 return fmt.Errorf("ensuring mailbox: %w", err)

1790 }

1791 m.MailboxID = mb.ID

1792 m.MailboxOrigID = mb.ID

1793

1794 // Update count early, DeliverMessage will update mb too and we don't want to fetch

1795 // it again before updating.

1796 mb.MailboxCounts.Add(m.MailboxCounts())

1797 if err := tx.Update(&mb); err != nil {

1798 return fmt.Errorf("updating mailbox for delivery: %w", err)

1799 }

1800

1801 if err := a.DeliverMessage(log, tx, m, msgFile, true, false, false, true); err != nil {

1802 return err

1803 }

1804

1805 changes = append(changes, chl...)

1806 changes = append(changes, m.ChangeAddUID(), mb.ChangeCounts())

1807 return nil

1808 })

1809 // todo: if rename succeeded but transaction failed, we should remove the file.

1810 if err != nil {

1811 return err

1812 }

1813

1814 BroadcastChanges(a, changes)

1815 return nil

1816}

1817

1818// TidyRejectsMailbox removes old reject emails, and returns whether there is space for a new delivery.

1819//

1820// Caller most hold account wlock.

1821// Changes are broadcasted.

1822func (a *Account) TidyRejectsMailbox(log mlog.Log, rejectsMailbox string) (hasSpace bool, rerr error) {

1823 var changes []Change

1824

1825 var remove []Message

1826 defer func() {

1827 for _, m := range remove {

1828 p := a.MessagePath(m.ID)

1829 err := os.Remove(p)

1830 log.Check(err, "removing rejects message file", slog.String("path", p))

1831 }

1832 }()

1833

1834 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1835 mb, err := a.MailboxFind(tx, rejectsMailbox)

1836 if err != nil {

1837 return fmt.Errorf("finding mailbox: %w", err)

1838 }

1839 if mb == nil {

1840 // No messages have been delivered yet.

1841 hasSpace = true

1842 return nil

1843 }

1844

1845 // Gather old messages to remove.

1846 old := time.Now().Add(-14 * 24 * time.Hour)

1847 qdel := bstore.QueryTx[Message](tx)

1848 qdel.FilterNonzero(Message{MailboxID: mb.ID})

1849 qdel.FilterEqual("Expunged", false)

1850 qdel.FilterLess("Received", old)

1851 remove, err = qdel.List()

1852 if err != nil {

1853 return fmt.Errorf("listing old messages: %w", err)

1854 }

1855

1856 changes, err = a.rejectsRemoveMessages(context.TODO(), log, tx, mb, remove)

1857 if err != nil {

1858 return fmt.Errorf("removing messages: %w", err)

1859 }

1860

1861 // We allow up to n messages.

1862 qcount := bstore.QueryTx[Message](tx)

1863 qcount.FilterNonzero(Message{MailboxID: mb.ID})

1864 qcount.FilterEqual("Expunged", false)

1865 qcount.Limit(1000)

1866 n, err := qcount.Count()

1867 if err != nil {

1868 return fmt.Errorf("counting rejects: %w", err)

1869 }

1870 hasSpace = n < 1000

1871

1872 return nil

1873 })

1874 if err != nil {

1875 remove = nil // Don't remove files on failure.

1876 return false, err

1877 }

1878

1879 BroadcastChanges(a, changes)

1880

1881 return hasSpace, nil

1882}

1883

1884func (a *Account) rejectsRemoveMessages(ctx context.Context, log mlog.Log, tx *bstore.Tx, mb *Mailbox, l []Message) ([]Change, error) {

1885 if len(l) == 0 {

1886 return nil, nil

1887 }

1888 ids := make([]int64, len(l))

1889 anyids := make([]any, len(l))

1890 for i, m := range l {

1891 ids[i] = m.ID

1892 anyids[i] = m.ID

1893 }

1894

1895 // Remove any message recipients. Should not happen, but a user can move messages

1896 // from a Sent mailbox to the rejects mailbox...

1897 qdmr := bstore.QueryTx[Recipient](tx)

1898 qdmr.FilterEqual("MessageID", anyids...)

1899 if _, err := qdmr.Delete(); err != nil {

1900 return nil, fmt.Errorf("deleting from message recipient: %w", err)

1901 }

1902

1903 // Assign new modseq.

1904 modseq, err := a.NextModSeq(tx)

1905 if err != nil {

1906 return nil, fmt.Errorf("assign next modseq: %w", err)

1907 }

1908

1909 // Expunge the messages.

1910 qx := bstore.QueryTx[Message](tx)

1911 qx.FilterIDs(ids)

1912 var expunged []Message

1913 qx.Gather(&expunged)

1914 if _, err := qx.UpdateNonzero(Message{ModSeq: modseq, Expunged: true}); err != nil {

1915 return nil, fmt.Errorf("expunging messages: %w", err)

1916 }

1917

1918 var totalSize int64

1919 for _, m := range expunged {

1920 m.Expunged = false // Was set by update, but would cause wrong count.

1921 mb.MailboxCounts.Sub(m.MailboxCounts())

1922 totalSize += m.Size

1923 }

1924 if err := tx.Update(mb); err != nil {

1925 return nil, fmt.Errorf("updating mailbox counts: %w", err)

1926 }

1927 if err := a.AddMessageSize(log, tx, -totalSize); err != nil {

1928 return nil, fmt.Errorf("updating disk usage: %w", err)

1929 }

1930

1931 // Mark as neutral and train so junk filter gets untrained with these (junk) messages.

1932 for i := range expunged {

1933 expunged[i].Junk = false

1934 expunged[i].Notjunk = false

1935 }

1936 if err := a.RetrainMessages(ctx, log, tx, expunged, true); err != nil {

1937 return nil, fmt.Errorf("retraining expunged messages: %w", err)

1938 }

1939

1940 changes := make([]Change, len(l), len(l)+1)

1941 for i, m := range l {

1942 changes[i] = ChangeRemoveUIDs{mb.ID, []UID{m.UID}, modseq}

1943 }

1944 changes = append(changes, mb.ChangeCounts())

1945 return changes, nil

1946}

1947

1948// RejectsRemove removes a message from the rejects mailbox if present.

1949// Caller most hold account wlock.

1950// Changes are broadcasted.

1951func (a *Account) RejectsRemove(log mlog.Log, rejectsMailbox, messageID string) error {

1952 var changes []Change

1953

1954 var remove []Message

1955 defer func() {

1956 for _, m := range remove {

1957 p := a.MessagePath(m.ID)

1958 err := os.Remove(p)

1959 log.Check(err, "removing rejects message file", slog.String("path", p))

1960 }

1961 }()

1962

1963 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1964 mb, err := a.MailboxFind(tx, rejectsMailbox)

1965 if err != nil {

1966 return fmt.Errorf("finding mailbox: %w", err)

1967 }

1968 if mb == nil {

1969 return nil

1970 }

1971

1972 q := bstore.QueryTx[Message](tx)

1973 q.FilterNonzero(Message{MailboxID: mb.ID, MessageID: messageID})

1974 q.FilterEqual("Expunged", false)

1975 remove, err = q.List()

1976 if err != nil {

1977 return fmt.Errorf("listing messages to remove: %w", err)

1978 }

1979

1980 changes, err = a.rejectsRemoveMessages(context.TODO(), log, tx, mb, remove)

1981 if err != nil {

1982 return fmt.Errorf("removing messages: %w", err)

1983 }

1984

1985 return nil

1986 })

1987 if err != nil {

1988 remove = nil // Don't remove files on failure.

1989 return err

1990 }

1991

1992 BroadcastChanges(a, changes)

1993

1994 return nil

1995}

1996

1997// AddMessageSize adjusts the DiskUsage.MessageSize by size.

1998func (a *Account) AddMessageSize(log mlog.Log, tx *bstore.Tx, size int64) error {

1999 du := DiskUsage{ID: 1}

2000 if err := tx.Get(&du); err != nil {

2001 return fmt.Errorf("get diskusage: %v", err)

2002 }

2003 du.MessageSize += size

2004 if du.MessageSize < 0 {

2005 log.Error("negative total message size", slog.Int64("delta", size), slog.Int64("newtotalsize", du.MessageSize))

2006 }

2007 if err := tx.Update(&du); err != nil {

2008 return fmt.Errorf("update total message size: %v", err)

2009 }

2010 return nil

2011}

2012

2013// QuotaMessageSize returns the effective maximum total message size for an

2014// account. Returns 0 if there is no maximum.

2015func (a *Account) QuotaMessageSize() int64 {

2016 conf, _ := a.Conf()

2017 size := conf.QuotaMessageSize

2018 if size <= 0 {

2019 size = mox.Conf.Static.QuotaMessageSize

2020 }

2021 if size < 0 {

2022 size = 0

2023 }

2024 return size

2025}

2026

2027// CanAddMessageSize checks if a message of size bytes can be added, depending on

2028// total message size and configured quota for account.

2029func (a *Account) CanAddMessageSize(tx *bstore.Tx, size int64) (ok bool, maxSize int64, err error) {

2030 maxSize = a.QuotaMessageSize()

2031 if maxSize <= 0 {

2032 return true, 0, nil

2033 }

2034

2035 du := DiskUsage{ID: 1}

2036 if err := tx.Get(&du); err != nil {

2037 return false, maxSize, fmt.Errorf("get diskusage: %v", err)

2038 }

2039 return du.MessageSize+size <= maxSize, maxSize, nil

2040}

2041

2042// We keep a cache of recent successful authentications, so we don't have to bcrypt successful calls each time.

2043var authCache = struct {

2044 sync.Mutex

2045 success map[authKey]string

2046}{

2047 success: map[authKey]string{},

2048}

2049

2050type authKey struct {

2051 email, hash string

2052}

2053

2054// StartAuthCache starts a goroutine that regularly clears the auth cache.

2055func StartAuthCache() {

2056 go manageAuthCache()

2057}

2058

2059func manageAuthCache() {

2060 for {

2061 authCache.Lock()

2062 authCache.success = map[authKey]string{}

2063 authCache.Unlock()

2064 time.Sleep(15 * time.Minute)

2065 }

2066}

2067

2068// OpenEmailAuth opens an account given an email address and password.

2069//

2070// The email address may contain a catchall separator.

2071func OpenEmailAuth(log mlog.Log, email string, password string) (acc *Account, rerr error) {

2072 acc, _, rerr = OpenEmail(log, email)

2073 if rerr != nil {

2074 return

2075 }

2076

2077 defer func() {

2078 if rerr != nil && acc != nil {

2079 err := acc.Close()

2080 log.Check(err, "closing account after open auth failure")

2081 acc = nil

2082 }

2083 }()

2084

2085 pw, err := bstore.QueryDB[Password](context.TODO(), acc.DB).Get()

2086 if err != nil {

2087 if err == bstore.ErrAbsent {

2088 return acc, ErrUnknownCredentials

2089 }

2090 return acc, fmt.Errorf("looking up password: %v", err)

2091 }

2092 authCache.Lock()

2093 ok := len(password) >= 8 && authCache.success[authKey{email, pw.Hash}] == password

2094 authCache.Unlock()

2095 if ok {

2096 return

2097 }

2098 if err := bcrypt.CompareHashAndPassword([]byte(pw.Hash), []byte(password)); err != nil {

2099 rerr = ErrUnknownCredentials

2100 } else {

2101 authCache.Lock()

2102 authCache.success[authKey{email, pw.Hash}] = password

2103 authCache.Unlock()

2104 }

2105 return

2106}

2107

2108// OpenEmail opens an account given an email address.

2109//

2110// The email address may contain a catchall separator.

2111func OpenEmail(log mlog.Log, email string) (*Account, config.Destination, error) {

2112 addr, err := smtp.ParseAddress(email)

2113 if err != nil {

2114 return nil, config.Destination{}, fmt.Errorf("%w: %v", ErrUnknownCredentials, err)

2115 }

2116 accountName, _, dest, err := mox.FindAccount(addr.Localpart, addr.Domain, false)

2117 if err != nil && (errors.Is(err, mox.ErrAccountNotFound) || errors.Is(err, mox.ErrDomainNotFound)) {

2118 return nil, config.Destination{}, ErrUnknownCredentials

2119 } else if err != nil {

2120 return nil, config.Destination{}, fmt.Errorf("looking up address: %v", err)

2121 }

2122 acc, err := OpenAccount(log, accountName)

2123 if err != nil {

2124 return nil, config.Destination{}, err

2125 }

2126 return acc, dest, nil

2127}

2128

2129// 64 characters, must be power of 2 for MessagePath

2130const msgDirChars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ-_"

2131

2132// MessagePath returns the filename of the on-disk filename, relative to the

2133// containing directory such as <account>/msg or queue.

2134// Returns names like "AB/1".

2135func MessagePath(messageID int64) string {

2136 return strings.Join(messagePathElems(messageID), string(filepath.Separator))

2137}

2138

2139// messagePathElems returns the elems, for a single join without intermediate

2140// string allocations.

2141func messagePathElems(messageID int64) []string {

2142 v := messageID >> 13 // 8k files per directory.

2143 dir := ""

2144 for {

2145 dir += string(msgDirChars[int(v)&(len(msgDirChars)-1)])

2146 v >>= 6

2147 if v == 0 {

2148 break

2149 }

2150 }

2151 return []string{dir, strconv.FormatInt(messageID, 10)}

2152}

2153

2154// Set returns a copy of f, with each flag that is true in mask set to the

2155// value from flags.

2156func (f Flags) Set(mask, flags Flags) Flags {

2157 set := func(d *bool, m, v bool) {

2158 if m {

2159 *d = v

2160 }

2161 }

2162 r := f

2163 set(&r.Seen, mask.Seen, flags.Seen)

2164 set(&r.Answered, mask.Answered, flags.Answered)

2165 set(&r.Flagged, mask.Flagged, flags.Flagged)

2166 set(&r.Forwarded, mask.Forwarded, flags.Forwarded)

2167 set(&r.Junk, mask.Junk, flags.Junk)

2168 set(&r.Notjunk, mask.Notjunk, flags.Notjunk)

2169 set(&r.Deleted, mask.Deleted, flags.Deleted)

2170 set(&r.Draft, mask.Draft, flags.Draft)

2171 set(&r.Phishing, mask.Phishing, flags.Phishing)

2172 set(&r.MDNSent, mask.MDNSent, flags.MDNSent)

2173 return r

2174}

2175

2176// Changed returns a mask of flags that have been between f and other.

2177func (f Flags) Changed(other Flags) (mask Flags) {

2178 mask.Seen = f.Seen != other.Seen

2179 mask.Answered = f.Answered != other.Answered

2180 mask.Flagged = f.Flagged != other.Flagged

2181 mask.Forwarded = f.Forwarded != other.Forwarded

2182 mask.Junk = f.Junk != other.Junk

2183 mask.Notjunk = f.Notjunk != other.Notjunk

2184 mask.Deleted = f.Deleted != other.Deleted

2185 mask.Draft = f.Draft != other.Draft

2186 mask.Phishing = f.Phishing != other.Phishing

2187 mask.MDNSent = f.MDNSent != other.MDNSent

2188 return

2189}

2190

2191var systemWellKnownFlags = map[string]bool{

2192 `\answered`: true,

2193 `\flagged`: true,

2194 `\deleted`: true,

2195 `\seen`: true,

2196 `\draft`: true,

2197 `$junk`: true,

2198 `$notjunk`: true,

2199 `$forwarded`: true,

2200 `$phishing`: true,

2201 `$mdnsent`: true,

2202}

2203

2204// ParseFlagsKeywords parses a list of textual flags into system/known flags, and

2205// other keywords. Keywords are lower-cased and sorted and check for valid syntax.

2206func ParseFlagsKeywords(l []string) (flags Flags, keywords []string, rerr error) {

2207 fields := map[string]*bool{

2208 `\answered`: &flags.Answered,

2209 `\flagged`: &flags.Flagged,

2210 `\deleted`: &flags.Deleted,

2211 `\seen`: &flags.Seen,

2212 `\draft`: &flags.Draft,

2213 `$junk`: &flags.Junk,

2214 `$notjunk`: &flags.Notjunk,

2215 `$forwarded`: &flags.Forwarded,

2216 `$phishing`: &flags.Phishing,

2217 `$mdnsent`: &flags.MDNSent,

2218 }

2219 seen := map[string]bool{}

2220 for _, f := range l {

2221 f = strings.ToLower(f)

2222 if field, ok := fields[f]; ok {

2223 *field = true

2224 } else if seen[f] {

2225 if mox.Pedantic {

2226 return Flags{}, nil, fmt.Errorf("duplicate keyword %s", f)

2227 }

2228 } else {

2229 if err := CheckKeyword(f); err != nil {

2230 return Flags{}, nil, fmt.Errorf("invalid keyword %s", f)

2231 }

2232 keywords = append(keywords, f)

2233 seen[f] = true

2234 }

2235 }

2236 sort.Strings(keywords)

2237 return flags, keywords, nil

2238}

2239

2240// RemoveKeywords removes keywords from l, returning whether any modifications were

2241// made, and a slice, a new slice in case of modifications. Keywords must have been

2242// validated earlier, e.g. through ParseFlagKeywords or CheckKeyword. Should only

2243// be used with valid keywords, not with system flags like \Seen.

2244func RemoveKeywords(l, remove []string) ([]string, bool) {

2245 var copied bool

2246 var changed bool

2247 for _, k := range remove {

2248 if i := slices.Index(l, k); i >= 0 {

2249 if !copied {

2250 l = append([]string{}, l...)

2251 copied = true

2252 }

2253 copy(l[i:], l[i+1:])

2254 l = l[:len(l)-1]

2255 changed = true

2256 }

2257 }

2258 return l, changed

2259}

2260

2261// MergeKeywords adds keywords from add into l, returning whether it added any

2262// keyword, and the slice with keywords, a new slice if modifications were made.

2263// Keywords are only added if they aren't already present. Should only be used with

2264// keywords, not with system flags like \Seen.

2265func MergeKeywords(l, add []string) ([]string, bool) {

2266 var copied bool

2267 var changed bool

2268 for _, k := range add {

2269 if !slices.Contains(l, k) {

2270 if !copied {

2271 l = append([]string{}, l...)

2272 copied = true

2273 }

2274 l = append(l, k)

2275 changed = true

2276 }

2277 }

2278 if changed {

2279 sort.Strings(l)

2280 }

2281 return l, changed

2282}

2283

2284// CheckKeyword returns an error if kw is not a valid keyword. Kw should

2285// already be in lower-case.

2286func CheckKeyword(kw string) error {

2287 if kw == "" {

2288 return fmt.Errorf("keyword cannot be empty")

2289 }

2290 if systemWellKnownFlags[kw] {

2291 return fmt.Errorf("cannot use well-known flag as keyword")

2292 }

2293 for _, c := range kw {

2294 // ../rfc/9051:6334

2295 if c <= ' ' || c > 0x7e || c >= 'A' && c <= 'Z' || strings.ContainsRune(`(){%*"\]`, c) {

2296 return errors.New(`not a valid keyword, must be lower-case ascii without spaces and without any of these characters: (){%*"\]`)

2297 }

2298 }

2299 return nil

2300}

2301

2302// SendLimitReached checks whether sending a message to recipients would reach

2303// the limit of outgoing messages for the account. If so, the message should

2304// not be sent. If the returned numbers are >= 0, the limit was reached and the

2305// values are the configured limits.

2306//

2307// To limit damage to the internet and our reputation in case of account

2308// compromise, we limit the max number of messages sent in a 24 hour window, both

2309// total number of messages and number of first-time recipients.

2310func (a *Account) SendLimitReached(tx *bstore.Tx, recipients []smtp.Path) (msglimit, rcptlimit int, rerr error) {

2311 conf, _ := a.Conf()

2312 msgmax := conf.MaxOutgoingMessagesPerDay

2313 if msgmax == 0 {

2314 // For human senders, 1000 recipients in a day is quite a lot.

2315 msgmax = 1000

2316 }

2317 rcptmax := conf.MaxFirstTimeRecipientsPerDay

2318 if rcptmax == 0 {

2319 // Human senders may address a new human-sized list of people once in a while. In

2320 // case of a compromise, a spammer will probably try to send to many new addresses.

2321 rcptmax = 200

2322 }

2323

2324 rcpts := map[string]time.Time{}

2325 n := 0

2326 err := bstore.QueryTx[Outgoing](tx).FilterGreater("Submitted", time.Now().Add(-24*time.Hour)).ForEach(func(o Outgoing) error {

2327 n++

2328 if rcpts[o.Recipient].IsZero() || o.Submitted.Before(rcpts[o.Recipient]) {

2329 rcpts[o.Recipient] = o.Submitted

2330 }

2331 return nil

2332 })

2333 if err != nil {

2334 return -1, -1, fmt.Errorf("querying message recipients in past 24h: %w", err)

2335 }

2336 if n+len(recipients) > msgmax {

2337 return msgmax, -1, nil

2338 }

2339

2340 // Only check if max first-time recipients is reached if there are enough messages

2341 // to trigger the limit.

2342 if n+len(recipients) < rcptmax {

2343 return -1, -1, nil

2344 }

2345

2346 isFirstTime := func(rcpt string, before time.Time) (bool, error) {

2347 exists, err := bstore.QueryTx[Outgoing](tx).FilterNonzero(Outgoing{Recipient: rcpt}).FilterLess("Submitted", before).Exists()

2348 return !exists, err

2349 }

2350

2351 firsttime := 0

2352 now := time.Now()

2353 for _, r := range recipients {

2354 if first, err := isFirstTime(r.XString(true), now); err != nil {

2355 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

2356 } else if first {

2357 firsttime++

2358 }

2359 }

2360 for r, t := range rcpts {

2361 if first, err := isFirstTime(r, t); err != nil {

2362 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

2363 } else if first {

2364 firsttime++

2365 }

2366 }

2367 if firsttime > rcptmax {

2368 return -1, rcptmax, nil

2369 }

2370 return -1, -1, nil

2371}

2372

2373// MailboxCreate creates a new mailbox, including any missing parent mailboxes,

2374// the total list of created mailboxes is returned in created. On success, if

2375// exists is false and rerr nil, the changes must be broadcasted by the caller.

2376//

2377// Name must be in normalized form.

2378func (a *Account) MailboxCreate(tx *bstore.Tx, name string) (changes []Change, created []string, exists bool, rerr error) {

2379 elems := strings.Split(name, "/")

2380 var p string

2381 for i, elem := range elems {

2382 if i > 0 {

2383 p += "/"

2384 }

2385 p += elem

2386 exists, err := a.MailboxExists(tx, p)

2387 if err != nil {

2388 return nil, nil, false, fmt.Errorf("checking if mailbox exists")

2389 }

2390 if exists {

2391 if i == len(elems)-1 {

2392 return nil, nil, true, fmt.Errorf("mailbox already exists")

2393 }

2394 continue

2395 }

2396 _, nchanges, err := a.MailboxEnsure(tx, p, true)

2397 if err != nil {

2398 return nil, nil, false, fmt.Errorf("ensuring mailbox exists")

2399 }

2400 changes = append(changes, nchanges...)

2401 created = append(created, p)

2402 }

2403 return changes, created, false, nil

2404}

2405

2406// MailboxRename renames mailbox mbsrc to dst, and any missing parents for the

2407// destination, and any children of mbsrc and the destination.

2408//

2409// Names must be normalized and cannot be Inbox.

2410func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc Mailbox, dst string) (changes []Change, isInbox, notExists, alreadyExists bool, rerr error) {

2411 if mbsrc.Name == "Inbox" || dst == "Inbox" {

2412 return nil, true, false, false, fmt.Errorf("inbox cannot be renamed")

2413 }

2414

2415 // We gather existing mailboxes that we need for deciding what to create/delete/update.

2416 q := bstore.QueryTx[Mailbox](tx)

2417 srcPrefix := mbsrc.Name + "/"

2418 dstRoot := strings.SplitN(dst, "/", 2)[0]

2419 dstRootPrefix := dstRoot + "/"

2420 q.FilterFn(func(mb Mailbox) bool {

2421 return mb.Name == mbsrc.Name || strings.HasPrefix(mb.Name, srcPrefix) || mb.Name == dstRoot || strings.HasPrefix(mb.Name, dstRootPrefix)

2422 })

2423 q.SortAsc("Name") // We'll rename the parents before children.

2424 l, err := q.List()

2425 if err != nil {

2426 return nil, false, false, false, fmt.Errorf("listing relevant mailboxes: %v", err)

2427 }

2428

2429 mailboxes := map[string]Mailbox{}

2430 for _, mb := range l {

2431 mailboxes[mb.Name] = mb

2432 }

2433

2434 if _, ok := mailboxes[mbsrc.Name]; !ok {

2435 return nil, false, true, false, fmt.Errorf("mailbox does not exist")

2436 }

2437

2438 uidval, err := a.NextUIDValidity(tx)

2439 if err != nil {

2440 return nil, false, false, false, fmt.Errorf("next uid validity: %v", err)

2441 }

2442

2443 // Ensure parent mailboxes for the destination paths exist.

2444 var parent string

2445 dstElems := strings.Split(dst, "/")

2446 for i, elem := range dstElems[:len(dstElems)-1] {

2447 if i > 0 {

2448 parent += "/"

2449 }

2450 parent += elem

2451

2452 mb, ok := mailboxes[parent]

2453 if ok {

2454 continue

2455 }

2456 omb := mb

2457 mb = Mailbox{

2458 ID: omb.ID,

2459 Name: parent,

2460 UIDValidity: uidval,

2461 UIDNext: 1,

2462 HaveCounts: true,

2463 }

2464 if err := tx.Insert(&mb); err != nil {

2465 return nil, false, false, false, fmt.Errorf("creating parent mailbox %q: %v", mb.Name, err)

2466 }

2467 if err := tx.Get(&Subscription{Name: parent}); err != nil {

2468 if err := tx.Insert(&Subscription{Name: parent}); err != nil {

2469 return nil, false, false, false, fmt.Errorf("creating subscription for %q: %v", parent, err)

2470 }

2471 }

2472 changes = append(changes, ChangeAddMailbox{Mailbox: mb, Flags: []string{`\Subscribed`}})

2473 }

2474

2475 // Process src mailboxes, renaming them to dst.

2476 for _, srcmb := range l {

2477 if srcmb.Name != mbsrc.Name && !strings.HasPrefix(srcmb.Name, srcPrefix) {

2478 continue

2479 }

2480 srcName := srcmb.Name

2481 dstName := dst + srcmb.Name[len(mbsrc.Name):]

2482 if _, ok := mailboxes[dstName]; ok {

2483 return nil, false, false, true, fmt.Errorf("destination mailbox %q already exists", dstName)

2484 }

2485

2486 srcmb.Name = dstName

2487 srcmb.UIDValidity = uidval

2488 if err := tx.Update(&srcmb); err != nil {

2489 return nil, false, false, false, fmt.Errorf("renaming mailbox: %v", err)

2490 }

2491

2492 var dstFlags []string

2493 if tx.Get(&Subscription{Name: dstName}) == nil {

2494 dstFlags = []string{`\Subscribed`}

2495 }

2496 changes = append(changes, ChangeRenameMailbox{MailboxID: srcmb.ID, OldName: srcName, NewName: dstName, Flags: dstFlags})

2497 }

2498

2499 // If we renamed e.g. a/b to a/b/c/d, and a/b/c to a/b/c/d/c, we'll have to recreate a/b and a/b/c.

2500 srcElems := strings.Split(mbsrc.Name, "/")

2501 xsrc := mbsrc.Name

2502 for i := 0; i < len(dstElems) && strings.HasPrefix(dst, xsrc+"/"); i++ {

2503 mb := Mailbox{

2504 UIDValidity: uidval,

2505 UIDNext: 1,

2506 Name: xsrc,

2507 HaveCounts: true,

2508 }

2509 if err := tx.Insert(&mb); err != nil {

2510 return nil, false, false, false, fmt.Errorf("creating mailbox at old path %q: %v", mb.Name, err)

2511 }

2512 xsrc += "/" + dstElems[len(srcElems)+i]

2513 }

2514 return changes, false, false, false, nil

2515}

2516

2517// MailboxDelete deletes a mailbox by ID. If it has children, the return value

2518// indicates that and an error is returned.

2519//

2520// Caller should broadcast the changes and remove files for the removed message IDs.

2521func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx, mailbox Mailbox) (changes []Change, removeMessageIDs []int64, hasChildren bool, rerr error) {

2522 // Look for existence of child mailboxes. There is a lot of text in the IMAP RFCs about

2523 // NoInferior and NoSelect. We just require only leaf mailboxes are deleted.

2524 qmb := bstore.QueryTx[Mailbox](tx)

2525 mbprefix := mailbox.Name + "/"

2526 qmb.FilterFn(func(mb Mailbox) bool {

2527 return strings.HasPrefix(mb.Name, mbprefix)

2528 })

2529 if childExists, err := qmb.Exists(); err != nil {

2530 return nil, nil, false, fmt.Errorf("checking if mailbox has child: %v", err)

2531 } else if childExists {

2532 return nil, nil, true, fmt.Errorf("mailbox has a child, only leaf mailboxes can be deleted")

2533 }

2534

2535 // todo jmap: instead of completely deleting a mailbox and its messages, we need to mark them all as expunged.

2536

2537 qm := bstore.QueryTx[Message](tx)

2538 qm.FilterNonzero(Message{MailboxID: mailbox.ID})

2539 remove, err := qm.List()

2540 if err != nil {

2541 return nil, nil, false, fmt.Errorf("listing messages to remove: %v", err)

2542 }

2543

2544 if len(remove) > 0 {

2545 removeIDs := make([]any, len(remove))

2546 for i, m := range remove {

2547 removeIDs[i] = m.ID

2548 }

2549 qmr := bstore.QueryTx[Recipient](tx)

2550 qmr.FilterEqual("MessageID", removeIDs...)

2551 if _, err = qmr.Delete(); err != nil {

2552 return nil, nil, false, fmt.Errorf("removing message recipients for messages: %v", err)

2553 }

2554

2555 qm = bstore.QueryTx[Message](tx)

2556 qm.FilterNonzero(Message{MailboxID: mailbox.ID})

2557 if _, err := qm.Delete(); err != nil {

2558 return nil, nil, false, fmt.Errorf("removing messages: %v", err)

2559 }

2560

2561 var totalSize int64

2562 for _, m := range remove {

2563 if !m.Expunged {

2564 removeMessageIDs = append(removeMessageIDs, m.ID)

2565 totalSize += m.Size

2566 }

2567 }

2568 if err := a.AddMessageSize(log, tx, -totalSize); err != nil {

2569 return nil, nil, false, fmt.Errorf("updating disk usage: %v", err)

2570 }

2571

2572 // Mark messages as not needing training. Then retrain them, so they are untrained if they were.

2573 n := 0

2574 o := 0

2575 for _, m := range remove {

2576 if !m.Expunged {

2577 remove[o] = m

2578 remove[o].Junk = false

2579 remove[o].Notjunk = false

2580 n++

2581 }

2582 }

2583 remove = remove[:n]

2584 if err := a.RetrainMessages(ctx, log, tx, remove, true); err != nil {

2585 return nil, nil, false, fmt.Errorf("untraining deleted messages: %v", err)

2586 }

2587 }

2588

2589 if err := tx.Delete(&Mailbox{ID: mailbox.ID}); err != nil {

2590 return nil, nil, false, fmt.Errorf("removing mailbox: %v", err)

2591 }

2592 return []Change{ChangeRemoveMailbox{MailboxID: mailbox.ID, Name: mailbox.Name}}, removeMessageIDs, false, nil

2593}

2594

2595// CheckMailboxName checks if name is valid, returning an INBOX-normalized name.

2596// I.e. it changes various casings of INBOX and INBOX/* to Inbox and Inbox/*.

2597// Name is invalid if it contains leading/trailing/double slashes, or when it isn't

2598// unicode-normalized, or when empty or has special characters.

2599//

2600// If name is the inbox, and allowInbox is false, this is indicated with the isInbox return parameter.

2601// For that case, and for other invalid names, an error is returned.

2602func CheckMailboxName(name string, allowInbox bool) (normalizedName string, isInbox bool, rerr error) {

2603 first := strings.SplitN(name, "/", 2)[0]

2604 if strings.EqualFold(first, "inbox") {

2605 if len(name) == len("inbox") && !allowInbox {

2606 return "", true, fmt.Errorf("special mailbox name Inbox not allowed")

2607 }

2608 name = "Inbox" + name[len("Inbox"):]

2609 }

2610

2611 if norm.NFC.String(name) != name {

2612 return "", false, errors.New("non-unicode-normalized mailbox names not allowed")

2613 }

2614

2615 if name == "" {

2616 return "", false, errors.New("empty mailbox name")

2617 }

2618 if strings.HasPrefix(name, "/") || strings.HasSuffix(name, "/") || strings.Contains(name, "//") {

2619 return "", false, errors.New("bad slashes in mailbox name")

2620 }

2621

2622 // "%" and "*" are difficult to use with the IMAP LIST command, but we allow mostly

2623 // allow them. ../rfc/3501:1002 ../rfc/9051:983

2624 if strings.HasPrefix(name, "#") {

2625 return "", false, errors.New("mailbox name cannot start with hash due to conflict with imap namespaces")

2626 }

2627

2628 // "#" and "&" are special in IMAP mailbox names. "#" for namespaces, "&" for

2629 // IMAP-UTF-7 encoding. We do allow them. ../rfc/3501:1018 ../rfc/9051:991

2630

2631 for _, c := range name {

2632 // ../rfc/3501:999 ../rfc/6855:192 ../rfc/9051:979

2633 if c <= 0x1f || c >= 0x7f && c <= 0x9f || c == 0x2028 || c == 0x2029 {

2634 return "", false, errors.New("control characters not allowed in mailbox name")

2635 }

2636 }

2637 return name, false, nil

2638}

2639