1/*

2Package store implements storage for accounts, their mailboxes, IMAP

3subscriptions and messages, and broadcasts updates (e.g. mail delivery) to

4interested sessions (e.g. IMAP connections).

6Layout of storage for accounts:

8 <DataDir>/accounts/<name>/index.db

9 <DataDir>/accounts/<name>/msg/[a-zA-Z0-9_-]+/<id>

11Index.db holds tables for user information, mailboxes, and messages. Message contents

12are stored in the msg/ subdirectory, each in their own file. The on-disk message

13does not contain headers generated during an incoming SMTP transaction, such as

14Received and Authentication-Results headers. Those are in the database to

15prevent having to rewrite incoming messages (e.g. Authentication-Result for DKIM

16signatures can only be determined after having read the message). Messages must

17be read through MsgReader, which transparently adds the prefix from the

18database.

19*/

20package store

22// todo: make up a function naming scheme that indicates whether caller should broadcast changes.

24import (

25 "context"

26 "crypto/md5"

27 cryptorand "crypto/rand"

28 "crypto/sha1"

29 "crypto/sha256"

30 "encoding"

31 "encoding/json"

32 "errors"

33 "fmt"

34 "hash"

35 "io"

36 "io/fs"

37 "log/slog"

38 "os"

39 "path/filepath"

40 "reflect"

41 "runtime/debug"

42 "slices"

43 "sort"

44 "strconv"

45 "strings"

46 "sync"

47 "time"

49 "golang.org/x/crypto/bcrypt"

50 "golang.org/x/text/secure/precis"

51 "golang.org/x/text/unicode/norm"

53 "github.com/mjl-/bstore"

55 "github.com/mjl-/mox/config"

56 "github.com/mjl-/mox/dns"

57 "github.com/mjl-/mox/junk"

58 "github.com/mjl-/mox/message"

59 "github.com/mjl-/mox/metrics"

60 "github.com/mjl-/mox/mlog"

61 "github.com/mjl-/mox/mox-"

62 "github.com/mjl-/mox/moxio"

63 "github.com/mjl-/mox/moxvar"

64 "github.com/mjl-/mox/publicsuffix"

65 "github.com/mjl-/mox/scram"

66 "github.com/mjl-/mox/smtp"

67)

69// If true, each time an account is closed its database file is checked for

70// consistency. If an inconsistency is found, panic is called. Set by default

71// because of all the packages with tests, the mox main function sets it to

72// false again.

73var CheckConsistencyOnClose = true

75var (

76 ErrUnknownMailbox = errors.New("no such mailbox")

77 ErrUnknownCredentials = errors.New("credentials not found")

78 ErrAccountUnknown = errors.New("no such account")

79 ErrOverQuota = errors.New("account over quota")

80 ErrLoginDisabled = errors.New("login disabled for account")

81)

83var DefaultInitialMailboxes = config.InitialMailboxes{

84 SpecialUse: config.SpecialUseMailboxes{

85 Sent: "Sent",

86 Archive: "Archive",

87 Trash: "Trash",

88 Draft: "Drafts",

89 Junk: "Junk",

90 },

91}

93type SCRAM struct {

94 Salt []byte

95 Iterations int

96 SaltedPassword []byte

97}

99// CRAMMD5 holds HMAC ipad and opad hashes that are initialized with the first

100// block with (a derivation of) the key/password, so we don't store the password in plain

101// text.

102type CRAMMD5 struct {

103 Ipad hash.Hash

104 Opad hash.Hash

105}

106

107// BinaryMarshal is used by bstore to store the ipad/opad hash states.

108func (c CRAMMD5) MarshalBinary() ([]byte, error) {

109 if c.Ipad == nil || c.Opad == nil {

110 return nil, nil

111 }

112

113 ipad, err := c.Ipad.(encoding.BinaryMarshaler).MarshalBinary()

114 if err != nil {

115 return nil, fmt.Errorf("marshal ipad: %v", err)

116 }

117 opad, err := c.Opad.(encoding.BinaryMarshaler).MarshalBinary()

118 if err != nil {

119 return nil, fmt.Errorf("marshal opad: %v", err)

120 }

121 buf := make([]byte, 2+len(ipad)+len(opad))

122 ipadlen := uint16(len(ipad))

123 buf[0] = byte(ipadlen >> 8)

124 buf[1] = byte(ipadlen >> 0)

125 copy(buf[2:], ipad)

126 copy(buf[2+len(ipad):], opad)

127 return buf, nil

128}

129

130// BinaryUnmarshal is used by bstore to restore the ipad/opad hash states.

131func (c *CRAMMD5) UnmarshalBinary(buf []byte) error {

132 if len(buf) == 0 {

133 *c = CRAMMD5{}

134 return nil

135 }

136 if len(buf) < 2 {

137 return fmt.Errorf("short buffer")

138 }

139 ipadlen := int(uint16(buf[0])<<8 | uint16(buf[1])<<0)

140 if len(buf) < 2+ipadlen {

141 return fmt.Errorf("buffer too short for ipadlen")

142 }

143 ipad := md5.New()

144 opad := md5.New()

145 if err := ipad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2 : 2+ipadlen]); err != nil {

146 return fmt.Errorf("unmarshal ipad: %v", err)

147 }

148 if err := opad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2+ipadlen:]); err != nil {

149 return fmt.Errorf("unmarshal opad: %v", err)

150 }

151 *c = CRAMMD5{ipad, opad}

152 return nil

153}

154

155// Password holds credentials in various forms, for logging in with SMTP/IMAP.

156type Password struct {

157 Hash string // bcrypt hash for IMAP LOGIN, SASL PLAIN and HTTP basic authentication.

158 CRAMMD5 CRAMMD5 // For SASL CRAM-MD5.

159 SCRAMSHA1 SCRAM // For SASL SCRAM-SHA-1.

160 SCRAMSHA256 SCRAM // For SASL SCRAM-SHA-256.

161}

162

163// Subjectpass holds the secret key used to sign subjectpass tokens.

164type Subjectpass struct {

165 Email string // Our destination address (canonical, with catchall localpart stripped).

166 Key string

167}

168

169// NextUIDValidity is a singleton record in the database with the next UIDValidity

170// to use for the next mailbox.

171type NextUIDValidity struct {

172 ID int // Just a single record with ID 1.

173 Next uint32

174}

175

176// SyncState track ModSeqs.

177type SyncState struct {

178 ID int // Just a single record with ID 1.

179

180 // Last used, next assigned will be one higher. The first value we hand out is 2.

181 // That's because 0 (the default value for old existing messages, from before the

182 // Message.ModSeq field) is special in IMAP, so we return it as 1.

183 LastModSeq ModSeq `bstore:"nonzero"`

184

185 // Highest ModSeq of expunged record that we deleted. When a clients synchronizes

186 // and requests changes based on a modseq before this one, we don't have the

187 // history to provide information about deletions. We normally keep these expunged

188 // records around, but we may periodically truly delete them to reclaim storage

189 // space. Initially set to -1 because we don't want to match with any ModSeq in the

190 // database, which can be zero values.

191 HighestDeletedModSeq ModSeq

192}

193

194// Mailbox is collection of messages, e.g. Inbox or Sent.

195type Mailbox struct {

196 ID int64

197

198 CreateSeq ModSeq

199 ModSeq ModSeq `bstore:"index"` // Of last change, or when deleted.

200 Expunged bool

201

202 ParentID int64 `bstore:"ref Mailbox"` // Zero for top-level mailbox.

203

204 // "Inbox" is the name for the special IMAP "INBOX". Slash separated for hierarchy.

205 // Names must be unique for mailboxes that are not expunged.

206 Name string `bstore:"nonzero"`

207

208 // If UIDs are invalidated, e.g. when renaming a mailbox to a previously existing

209 // name, UIDValidity must be changed. Used by IMAP for synchronization.

210 UIDValidity uint32

211

212 // UID likely to be assigned to next message. Used by IMAP to detect messages

213 // delivered to a mailbox.

214 UIDNext UID

215

216 SpecialUse

217

218 // Keywords as used in messages. Storing a non-system keyword for a message

219 // automatically adds it to this list. Used in the IMAP FLAGS response. Only

220 // "atoms" are allowed (IMAP syntax), keywords are case-insensitive, only stored in

221 // lower case (for JMAP), sorted.

222 Keywords []string

223

224 HaveCounts bool // Deprecated. Covered by Upgrade.MailboxCounts. No longer read.

225 MailboxCounts // Statistics about messages, kept up to date whenever a change happens.

226}

227

228// Annotation is a per-mailbox or global (per-account) annotation for the IMAP

229// metadata extension, currently always a private annotation.

230type Annotation struct {

231 ID int64

232

233 CreateSeq ModSeq

234 ModSeq ModSeq `bstore:"index"`

235 Expunged bool

236

237 // Can be zero, indicates global (per-account) annotation.

238 MailboxID int64 `bstore:"ref Mailbox,index MailboxID+Key"`

239

240 // "Entry name", always starts with "/private/" or "/shared/". Stored lower-case,

241 // comparisons must be done case-insensitively.

242 Key string `bstore:"nonzero"`

243

244 IsString bool // If true, the value is a string instead of bytes.

245 Value []byte

246}

247

248// Change returns a broadcastable change for the annotation.

249func (a Annotation) Change(mailboxName string) ChangeAnnotation {

250 return ChangeAnnotation{a.MailboxID, mailboxName, a.Key, a.ModSeq}

251}

252

253// MailboxCounts tracks statistics about messages for a mailbox.

254type MailboxCounts struct {

255 Total int64 // Total number of messages, excluding \Deleted. For JMAP.

256 Deleted int64 // Number of messages with \Deleted flag. Used for IMAP message count that includes messages with \Deleted.

257 Unread int64 // Messages without \Seen, excluding those with \Deleted, for JMAP.

258 Unseen int64 // Messages without \Seen, including those with \Deleted, for IMAP.

259 Size int64 // Number of bytes for all messages.

260}

261

262// MessageCountIMAP returns the total message count for use in IMAP. In IMAP,

263// message marked \Deleted are included, in JMAP they those messages are not

264// visible at all.

265func (mc MailboxCounts) MessageCountIMAP() uint32 {

266 return uint32(mc.Total + mc.Deleted)

267}

268

269func (mc MailboxCounts) String() string {

270 return fmt.Sprintf("%d total, %d deleted, %d unread, %d unseen, size %d bytes", mc.Total, mc.Deleted, mc.Unread, mc.Unseen, mc.Size)

271}

272

273// Add increases mailbox counts mc with those of delta.

274func (mc *MailboxCounts) Add(delta MailboxCounts) {

275 mc.Total += delta.Total

276 mc.Deleted += delta.Deleted

277 mc.Unread += delta.Unread

278 mc.Unseen += delta.Unseen

279 mc.Size += delta.Size

280}

281

282// Add decreases mailbox counts mc with those of delta.

283func (mc *MailboxCounts) Sub(delta MailboxCounts) {

284 mc.Total -= delta.Total

285 mc.Deleted -= delta.Deleted

286 mc.Unread -= delta.Unread

287 mc.Unseen -= delta.Unseen

288 mc.Size -= delta.Size

289}

290

291// SpecialUse identifies a specific role for a mailbox, used by clients to

292// understand where messages should go.

293type SpecialUse struct {

294 Archive bool

295 Draft bool // "Drafts"

296 Junk bool

297 Sent bool

298 Trash bool

299}

300

301// UIDNextAdd increases the UIDNext value by n, returning an error on overflow.

302func (mb *Mailbox) UIDNextAdd(n int) error {

303 uidnext := mb.UIDNext + UID(n)

304 if uidnext < mb.UIDNext {

305 return fmt.Errorf("uid overflow on mailbox %q (id %d): uidnext %d, adding %d; consider recreating the mailbox and copying its messages to compact", mb.Name, mb.ID, mb.UIDNext, n)

306 }

307 mb.UIDNext = uidnext

308 return nil

309}

310

311// CalculateCounts calculates the full current counts for messages in the mailbox.

312func (mb *Mailbox) CalculateCounts(tx *bstore.Tx) (mc MailboxCounts, err error) {

313 q := bstore.QueryTx[Message](tx)

314 q.FilterNonzero(Message{MailboxID: mb.ID})

315 q.FilterEqual("Expunged", false)

316 err = q.ForEach(func(m Message) error {

317 mc.Add(m.MailboxCounts())

318 return nil

319 })

320 return

321}

322

323// ChangeSpecialUse returns a change for special-use flags, for broadcasting to

324// other connections.

325func (mb Mailbox) ChangeSpecialUse() ChangeMailboxSpecialUse {

326 return ChangeMailboxSpecialUse{mb.ID, mb.Name, mb.SpecialUse, mb.ModSeq}

327}

328

329// ChangeKeywords returns a change with new keywords for a mailbox (e.g. after

330// setting a new keyword on a message in the mailbox), for broadcasting to other

331// connections.

332func (mb Mailbox) ChangeKeywords() ChangeMailboxKeywords {

333 return ChangeMailboxKeywords{mb.ID, mb.Name, mb.Keywords}

334}

335

336func (mb Mailbox) ChangeAddMailbox(flags []string) ChangeAddMailbox {

337 return ChangeAddMailbox{Mailbox: mb, Flags: flags}

338}

339

340func (mb Mailbox) ChangeRemoveMailbox() ChangeRemoveMailbox {

341 return ChangeRemoveMailbox{mb.ID, mb.Name, mb.ModSeq}

342}

343

344// KeywordsChanged returns whether the keywords in a mailbox have changed.

345func (mb Mailbox) KeywordsChanged(origmb Mailbox) bool {

346 if len(mb.Keywords) != len(origmb.Keywords) {

347 return true

348 }

349 // Keywords are stored sorted.

350 for i, kw := range mb.Keywords {

351 if origmb.Keywords[i] != kw {

352 return true

353 }

354 }

355 return false

356}

357

358// CountsChange returns a change with mailbox counts.

359func (mb Mailbox) ChangeCounts() ChangeMailboxCounts {

360 return ChangeMailboxCounts{mb.ID, mb.Name, mb.MailboxCounts}

361}

362

363// Subscriptions are separate from existence of mailboxes.

364type Subscription struct {

365 Name string

366}

367

368// Flags for a mail message.

369type Flags struct {

370 Seen bool

371 Answered bool

372 Flagged bool

373 Forwarded bool

374 Junk bool

375 Notjunk bool

376 Deleted bool

377 Draft bool

378 Phishing bool

379 MDNSent bool

380}

381

382// FlagsAll is all flags set, for use as mask.

383var FlagsAll = Flags{true, true, true, true, true, true, true, true, true, true}

384

385// Validation of "message From" domain.

386type Validation uint8

387

388const (

389 ValidationUnknown Validation = 0

390 ValidationStrict Validation = 1 // Like DMARC, with strict policies.

391 ValidationDMARC Validation = 2 // Actual DMARC policy.

392 ValidationRelaxed Validation = 3 // Like DMARC, with relaxed policies.

393 ValidationPass Validation = 4 // For SPF.

394 ValidationNeutral Validation = 5 // For SPF.

395 ValidationTemperror Validation = 6

396 ValidationPermerror Validation = 7

397 ValidationFail Validation = 8

398 ValidationSoftfail Validation = 9 // For SPF.

399 ValidationNone Validation = 10 // E.g. No records.

400)

401

402// Message stored in database and per-message file on disk.

403//

404// Contents are always the combined data from MsgPrefix and the on-disk file named

405// based on ID.

406//

407// Messages always have a header section, even if empty. Incoming messages without

408// header section must get an empty header section added before inserting.

409type Message struct {

410 // ID of the message, determines path to on-disk message file. Set when adding to a

411 // mailbox. When a message is moved to another mailbox, the mailbox ID is changed,

412 // but for synchronization purposes, a new Message record is inserted (which gets a

413 // new ID) with the Expunged field set and the MailboxID and UID copied.

414 ID int64

415

416 // UID, for IMAP. Set when adding to mailbox. Strictly increasing values, per

417 // mailbox. The UID of a message can never change (though messages can be copied),

418 // and the contents of a message/UID also never changes.

419 UID UID `bstore:"nonzero"`

420

421 MailboxID int64 `bstore:"nonzero,unique MailboxID+UID,index MailboxID+Received,index MailboxID+ModSeq,ref Mailbox"`

422

423 // Modification sequence, for faster syncing with IMAP QRESYNC and JMAP.

424 // ModSeq is the last modification. CreateSeq is the Seq the message was inserted,

425 // always <= ModSeq. If Expunged is set, the message has been removed and should not

426 // be returned to the user. In this case, ModSeq is the Seq where the message is

427 // removed, and will never be changed again.

428 // We have an index on both ModSeq (for JMAP that synchronizes per account) and

429 // MailboxID+ModSeq (for IMAP that synchronizes per mailbox).

430 // The index on CreateSeq helps efficiently finding created messages for JMAP.

431 // The value of ModSeq is special for IMAP. Messages that existed before ModSeq was

432 // added have 0 as value. But modseq 0 in IMAP is special, so we return it as 1. If

433 // we get modseq 1 from a client, the IMAP server will translate it to 0. When we

434 // return modseq to clients, we turn 0 into 1.

435 ModSeq ModSeq `bstore:"index"`

436 CreateSeq ModSeq `bstore:"index"`

437 Expunged bool

438

439 // If set, this message was delivered to a Rejects mailbox. When it is moved to a

440 // different mailbox, its MailboxOrigID is set to the destination mailbox and this

441 // flag cleared.

442 IsReject bool

443

444 // If set, this is a forwarded message (through a ruleset with IsForward). This

445 // causes fields used during junk analysis to be moved to their Orig variants, and

446 // masked IP fields cleared, so they aren't used in junk classifications for

447 // incoming messages. This ensures the forwarded messages don't cause negative

448 // reputation for the forwarding mail server, which may also be sending regular

449 // messages.

450 IsForward bool

451

452 // MailboxOrigID is the mailbox the message was originally delivered to. Typically

453 // Inbox or Rejects, but can also be a mailbox configured in a Ruleset, or

454 // Postmaster, TLS/DMARC reporting addresses. MailboxOrigID is not changed when the

455 // message is moved to another mailbox, e.g. Archive/Trash/Junk. Used for

456 // per-mailbox reputation.

457 //

458 // MailboxDestinedID is normally 0, but when a message is delivered to the Rejects

459 // mailbox, it is set to the intended mailbox according to delivery rules,

460 // typically that of Inbox. When such a message is moved out of Rejects, the

461 // MailboxOrigID is corrected by setting it to MailboxDestinedID. This ensures the

462 // message is used for reputation calculation for future deliveries to that

463 // mailbox.

464 //

465 // These are not bstore references to prevent having to update all messages in a

466 // mailbox when the original mailbox is removed. Use of these fields requires

467 // checking if the mailbox still exists.

468 MailboxOrigID int64

469 MailboxDestinedID int64

470

471 // Received indicates time of receival over SMTP, or of IMAP APPEND.

472 Received time.Time `bstore:"default now,index"`

473

474 // SaveDate is the time of copy/move/save to a mailbox, used with IMAP SAVEDATE

475 // extension. Must be updated each time a message is copied/moved to another

476 // mailbox. Can be nil for messages from before this functionality was introduced.

477 SaveDate *time.Time `bstore:"default now"`

478

479 // Full IP address of remote SMTP server. Empty if not delivered over SMTP. The

480 // masked IPs are used to classify incoming messages. They are left empty for

481 // messages matching a ruleset for forwarded messages.

482 RemoteIP string

483 RemoteIPMasked1 string `bstore:"index RemoteIPMasked1+Received"` // For IPv4 /32, for IPv6 /64, for reputation.

484 RemoteIPMasked2 string `bstore:"index RemoteIPMasked2+Received"` // For IPv4 /26, for IPv6 /48.

485 RemoteIPMasked3 string `bstore:"index RemoteIPMasked3+Received"` // For IPv4 /21, for IPv6 /32.

486

487 // Only set if present and not an IP address. Unicode string. Empty for forwarded

488 // messages.

489 EHLODomain string `bstore:"index EHLODomain+Received"`

490 MailFrom string // With localpart and domain. Can be empty.

491 MailFromLocalpart smtp.Localpart // SMTP "MAIL FROM", can be empty.

492 // Only set if it is a domain, not an IP. Unicode string. Empty for forwarded

493 // messages, but see OrigMailFromDomain.

494 MailFromDomain string `bstore:"index MailFromDomain+Received"`

495 RcptToLocalpart smtp.Localpart // SMTP "RCPT TO", can be empty.

496 RcptToDomain string // Unicode string.

497

498 // Parsed "From" message header, used for reputation along with domain validation.

499 MsgFromLocalpart smtp.Localpart

500 MsgFromDomain string `bstore:"index MsgFromDomain+Received"` // Unicode string.

501 MsgFromOrgDomain string `bstore:"index MsgFromOrgDomain+Received"` // Unicode string.

502

503 // Simplified statements of the Validation fields below, used for incoming messages

504 // to check reputation.

505 EHLOValidated bool

506 MailFromValidated bool

507 MsgFromValidated bool

508

509 EHLOValidation Validation // Validation can also take reverse IP lookup into account, not only SPF.

510 MailFromValidation Validation // Can have SPF-specific validations like ValidationSoftfail.

511 MsgFromValidation Validation // Desirable validations: Strict, DMARC, Relaxed. Will not be just Pass.

512

513 // Domains with verified DKIM signatures. Unicode string. For forwarded messages, a

514 // DKIM domain that matched a ruleset's verified domain is left out, but included

515 // in OrigDKIMDomains.

516 DKIMDomains []string `bstore:"index DKIMDomains+Received"`

517

518 // For forwarded messages,

519 OrigEHLODomain string

520 OrigDKIMDomains []string

521

522 // Canonicalized Message-Id, always lower-case and normalized quoting, without

523 // <>'s. Empty if missing. Used for matching message threads, and to prevent

524 // duplicate reject delivery.

525 MessageID string `bstore:"index"`

526 // lower-case: ../rfc/5256:495

527

528 // For matching threads in case there is no References/In-Reply-To header. It is

529 // lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed.

530 SubjectBase string `bstore:"index"`

531 // ../rfc/5256:90

532

533 // Hash of message. For rejects delivery in case there is no Message-ID, only set

534 // when delivered as reject.

535 MessageHash []byte

536

537 // ID of message starting this thread.

538 ThreadID int64 `bstore:"index"`

539 // IDs of parent messages, from closest parent to the root message. Parent messages

540 // may be in a different mailbox, or may no longer exist. ThreadParentIDs must

541 // never contain the message id itself (a cycle), and parent messages must

542 // reference the same ancestors. Moving a message to another mailbox keeps the

543 // message ID and changes the MailboxID (and UID) of the message, leaving threading

544 // parent ids intact.

545 ThreadParentIDs []int64

546 // ThreadMissingLink is true if there is no match with a direct parent. E.g. first

547 // ID in ThreadParentIDs is not the direct ancestor (an intermediate message may

548 // have been deleted), or subject-based matching was done.

549 ThreadMissingLink bool

550 // If set, newly delivered child messages are automatically marked as read. This

551 // field is copied to new child messages. Changes are propagated to the webmail

552 // client.

553 ThreadMuted bool

554 // If set, this (sub)thread is collapsed in the webmail client, for threading mode

555 // "on" (mode "unread" ignores it). This field is copied to new child message.

556 // Changes are propagated to the webmail client.

557 ThreadCollapsed bool

558

559 // If received message was known to match a mailing list rule (with modified junk

560 // filtering).

561 IsMailingList bool

562

563 // If this message is a DSN, generated by us or received. For DSNs, we don't look

564 // at the subject when matching threads.

565 DSN bool

566

567 ReceivedTLSVersion uint16 // 0 if unknown, 1 if plaintext/no TLS, otherwise TLS cipher suite.

568 ReceivedTLSCipherSuite uint16

569 ReceivedRequireTLS bool // Whether RequireTLS was known to be used for incoming delivery.

570

571 Flags

572 // For keywords other than system flags or the basic well-known $-flags. Only in

573 // "atom" syntax (IMAP), they are case-insensitive, always stored in lower-case

574 // (for JMAP), sorted.

575 Keywords []string `bstore:"index"`

576 Size int64

577 TrainedJunk *bool // If nil, no training done yet. Otherwise, true is trained as junk, false trained as nonjunk.

578 MsgPrefix []byte // Typically holds received headers and/or header separator.

579

580 // If non-nil, a preview of the message based on text and/or html parts of the

581 // message. Used in the webmail and IMAP PREVIEW extension. If non-nil, it is empty

582 // if no preview could be created, or the message has not textual content or

583 // couldn't be parsed.

584 // Previews are typically created when delivering a message, but not when importing

585 // messages, for speed. Previews are generated on first request (in the webmail, or

586 // through the IMAP fetch attribute "PREVIEW" (without "LAZY")), and stored with

587 // the message at that time.

588 // The preview is at most 256 characters (can be more bytes), with detected quoted

589 // text replaced with "[...]". Previews typically end with a newline, callers may

590 // want to strip whitespace.

591 Preview *string

592

593 // ParsedBuf message structure. Currently saved as JSON of message.Part because

594 // bstore wasn't able to store recursive types when this was implemented. Created

595 // when first needed, and saved in the database.

596 // todo: once replaced with non-json storage, remove date fixup in ../message/part.go.

597 ParsedBuf []byte

598}

599

600// MailboxCounts returns the delta to counts this message means for its

601// mailbox.

602func (m Message) MailboxCounts() (mc MailboxCounts) {

603 if m.Expunged {

604 return

605 }

606 if m.Deleted {

607 mc.Deleted++

608 } else {

609 mc.Total++

610 }

611 if !m.Seen {

612 mc.Unseen++

613 if !m.Deleted {

614 mc.Unread++

615 }

616 }

617 mc.Size += m.Size

618 return

619}

620

621func (m Message) ChangeAddUID(mb Mailbox) ChangeAddUID {

622 return ChangeAddUID{m.MailboxID, m.UID, m.ModSeq, m.Flags, m.Keywords, mb.MessageCountIMAP(), uint32(mb.MailboxCounts.Unseen)}

623}

624

625func (m Message) ChangeFlags(orig Flags, mb Mailbox) ChangeFlags {

626 mask := m.Flags.Changed(orig)

627 return ChangeFlags{m.MailboxID, m.UID, m.ModSeq, mask, m.Flags, m.Keywords, mb.UIDValidity, uint32(mb.MailboxCounts.Unseen)}

628}

629

630func (m Message) ChangeThread() ChangeThread {

631 return ChangeThread{[]int64{m.ID}, m.ThreadMuted, m.ThreadCollapsed}

632}

633

634// ModSeq represents a modseq as stored in the database. ModSeq 0 in the

635// database is sent to the client as 1, because modseq 0 is special in IMAP.

636// ModSeq coming from the client are of type int64.

637type ModSeq int64

638

639func (ms ModSeq) Client() int64 {

640 if ms == 0 {

641 return 1

642 }

643 return int64(ms)

644}

645

646// ModSeqFromClient converts a modseq from a client to a modseq for internal

647// use, e.g. in a database query.

648// ModSeq 1 is turned into 0 (the Go zero value for ModSeq).

649func ModSeqFromClient(modseq int64) ModSeq {

650 if modseq == 1 {

651 return 0

652 }

653 return ModSeq(modseq)

654}

655

656// Erase clears fields from a Message that are no longer needed after actually

657// removing the message file from the file system, after all references to the

658// message have gone away. Only the fields necessary for synchronisation are kept.

659func (m *Message) erase() {

660 if !m.Expunged {

661 panic("erase called on non-expunged message")

662 }

663 *m = Message{

664 ID: m.ID,

665 UID: m.UID,

666 MailboxID: m.MailboxID,

667 CreateSeq: m.CreateSeq,

668 ModSeq: m.ModSeq,

669 Expunged: true,

670 ThreadID: m.ThreadID,

671 }

672}

673

674// PrepareThreading sets MessageID, SubjectBase and DSN (used in threading) based

675// on the part.

676func (m *Message) PrepareThreading(log mlog.Log, part *message.Part) {

677 m.DSN = part.IsDSN()

678

679 if part.Envelope == nil {

680 return

681 }

682 messageID, raw, err := message.MessageIDCanonical(part.Envelope.MessageID)

683 if err != nil {

684 log.Debugx("parsing message-id, ignoring", err, slog.String("messageid", part.Envelope.MessageID))

685 } else if raw {

686 log.Debug("could not parse message-id as address, continuing with raw value", slog.String("messageid", part.Envelope.MessageID))

687 }

688 m.MessageID = messageID

689 m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false)

690}

691

692// LoadPart returns a message.Part by reading from m.ParsedBuf.

693func (m Message) LoadPart(r io.ReaderAt) (message.Part, error) {

694 if m.ParsedBuf == nil {

695 return message.Part{}, fmt.Errorf("message not parsed")

696 }

697 var p message.Part

698 err := json.Unmarshal(m.ParsedBuf, &p)

699 if err != nil {

700 return p, fmt.Errorf("unmarshal message part")

701 }

702 p.SetReaderAt(r)

703 return p, nil

704}

705

706// NeedsTraining returns whether message needs a training update, based on

707// TrainedJunk (current training status) and new Junk/Notjunk flags.

708func (m Message) NeedsTraining() bool {

709 needs, _, _, _, _ := m.needsTraining()

710 return needs

711}

712

713func (m Message) needsTraining() (needs, untrain, untrainJunk, train, trainJunk bool) {

714 untrain = m.TrainedJunk != nil

715 untrainJunk = untrain && *m.TrainedJunk

716 train = m.Junk != m.Notjunk

717 trainJunk = m.Junk

718 needs = untrain != train || untrain && train && untrainJunk != trainJunk

719 return

720}

721

722// JunkFlagsForMailbox sets Junk and Notjunk flags based on mailbox name if configured. Often

723// used when delivering/moving/copying messages to a mailbox. Mail clients are not

724// very helpful with setting junk/notjunk flags. But clients can move/copy messages

725// to other mailboxes. So we set flags when clients move a message.

726func (m *Message) JunkFlagsForMailbox(mb Mailbox, conf config.Account) {

727 if mb.Junk {

728 m.Junk = true

729 m.Notjunk = false

730 return

731 }

732

733 if !conf.AutomaticJunkFlags.Enabled {

734 return

735 }

736

737 lmailbox := strings.ToLower(mb.Name)

738

739 if conf.JunkMailbox != nil && conf.JunkMailbox.MatchString(lmailbox) {

740 m.Junk = true

741 m.Notjunk = false

742 } else if conf.NeutralMailbox != nil && conf.NeutralMailbox.MatchString(lmailbox) {

743 m.Junk = false

744 m.Notjunk = false

745 } else if conf.NotJunkMailbox != nil && conf.NotJunkMailbox.MatchString(lmailbox) {

746 m.Junk = false

747 m.Notjunk = true

748 } else if conf.JunkMailbox == nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox != nil {

749 m.Junk = true

750 m.Notjunk = false

751 } else if conf.JunkMailbox != nil && conf.NeutralMailbox == nil && conf.NotJunkMailbox != nil {

752 m.Junk = false

753 m.Notjunk = false

754 } else if conf.JunkMailbox != nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox == nil {

755 m.Junk = false

756 m.Notjunk = true

757 }

758}

759

760// Recipient represents the recipient of a message. It is tracked to allow

761// first-time incoming replies from users this account has sent messages to. When a

762// mailbox is added to the Sent mailbox the message is parsed and recipients are

763// inserted as recipient. Recipients are never removed other than for removing the

764// message. On move/copy of a message, recipients aren't modified either. For IMAP,

765// this assumes a client simply appends messages to the Sent mailbox (as opposed to

766// copying messages from some place).

767type Recipient struct {

768 ID int64

769 MessageID int64 `bstore:"nonzero,ref Message"` // Ref gives it its own index, useful for fast removal as well.

770 Localpart string `bstore:"nonzero"` // Encoded localpart.

771 Domain string `bstore:"nonzero,index Domain+Localpart"` // Unicode string.

772 OrgDomain string `bstore:"nonzero,index"` // Unicode string.

773 Sent time.Time `bstore:"nonzero"`

774}

775

776// Outgoing is a message submitted for delivery from the queue. Used to enforce

777// maximum outgoing messages.

778type Outgoing struct {

779 ID int64

780 Recipient string `bstore:"nonzero,index"` // Canonical international address with utf8 domain.

781 Submitted time.Time `bstore:"nonzero,default now"`

782}

783

784// RecipientDomainTLS stores TLS capabilities of a recipient domain as encountered

785// during most recent connection (delivery attempt).

786type RecipientDomainTLS struct {

787 Domain string // Unicode.

788 Updated time.Time `bstore:"default now"`

789 STARTTLS bool // Supports STARTTLS.

790 RequireTLS bool // Supports RequireTLS SMTP extension.

791}

792

793// DiskUsage tracks quota use.

794type DiskUsage struct {

795 ID int64 // Always one record with ID 1.

796 MessageSize int64 // Sum of all messages, for quota accounting.

797}

798

799// SessionToken and CSRFToken are types to prevent mixing them up.

800// Base64 raw url encoded.

801type SessionToken string

802type CSRFToken string

803

804// LoginSession represents a login session. We keep a limited number of sessions

805// for a user, removing the oldest session when a new one is created.

806type LoginSession struct {

807 ID int64

808 Created time.Time `bstore:"nonzero,default now"` // Of original login.

809 Expires time.Time `bstore:"nonzero"` // Extended each time it is used.

810 SessionTokenBinary [16]byte `bstore:"nonzero"` // Stored in cookie, like "webmailsession" or "webaccountsession".

811 CSRFTokenBinary [16]byte // For API requests, in "x-mox-csrf" header.

812 AccountName string `bstore:"nonzero"`

813 LoginAddress string `bstore:"nonzero"`

814

815 // Set when loading from database.

816 sessionToken SessionToken

817 csrfToken CSRFToken

818}

819

820// Quoting is a setting for how to quote in replies/forwards.

821type Quoting string

822

823const (

824 Default Quoting = "" // Bottom-quote if text is selected, top-quote otherwise.

825 Bottom Quoting = "bottom"

826 Top Quoting = "top"

827)

828

829// Settings are webmail client settings.

830type Settings struct {

831 ID uint8 // Singleton ID 1.

832

833 Signature string

834 Quoting Quoting

835

836 // Whether to show the bars underneath the address input fields indicating

837 // starttls/dnssec/dane/mtasts/requiretls support by address.

838 ShowAddressSecurity bool

839

840 // Show HTML version of message by default, instead of plain text.

841 ShowHTML bool

842

843 // If true, don't show shortcuts in webmail after mouse interaction.

844 NoShowShortcuts bool

845

846 // Additional headers to display in message view. E.g. Delivered-To, User-Agent, X-Mox-Reason.

847 ShowHeaders []string

848}

849

850// ViewMode how a message should be viewed: its text parts, html parts, or html

851// with loading external resources.

852type ViewMode string

853

854const (

855 ModeText ViewMode = "text"

856 ModeHTML ViewMode = "html"

857 ModeHTMLExt ViewMode = "htmlext" // HTML with external resources.

858)

859

860// FromAddressSettings are webmail client settings per "From" address.

861type FromAddressSettings struct {

862 FromAddress string // Unicode.

863 ViewMode ViewMode

864}

865

866// RulesetNoListID records a user "no" response to the question of

867// creating/removing a ruleset after moving a message with list-id header from/to

868// the inbox.

869type RulesetNoListID struct {

870 ID int64

871 RcptToAddress string `bstore:"nonzero"`

872 ListID string `bstore:"nonzero"`

873 ToInbox bool // Otherwise from Inbox to other mailbox.

874}

875

876// RulesetNoMsgFrom records a user "no" response to the question of

877// creating/moveing a ruleset after moving a mesage with message "from" address

878// from/to the inbox.

879type RulesetNoMsgFrom struct {

880 ID int64

881 RcptToAddress string `bstore:"nonzero"`

882 MsgFromAddress string `bstore:"nonzero"` // Unicode.

883 ToInbox bool // Otherwise from Inbox to other mailbox.

884}

885

886// RulesetNoMailbox represents a "never from/to this mailbox" response to the

887// question of adding/removing a ruleset after moving a message.

888type RulesetNoMailbox struct {

889 ID int64

890

891 // The mailbox from/to which the move has happened.

892 // Not a references, if mailbox is deleted, an entry becomes ineffective.

893 MailboxID int64 `bstore:"nonzero"`

894 ToMailbox bool // Whether MailboxID is the destination of the move (instead of source).

895}

896

897// MessageErase represents the need to remove a message file from disk, and clear

898// message fields from the database, but only when the last reference to the

899// message is gone (all IMAP sessions need to have applied the changes indicating

900// message removal).

901type MessageErase struct {

902 ID int64 // Same ID as Message.ID.

903

904 // Whether to subtract the size from the total disk usage. Useful for moving

905 // messages, which involves duplicating the message temporarily, while there are

906 // still references in the old mailbox, but which isn't counted as using twice the

907 // disk space..

908 SkipUpdateDiskUsage bool

909}

910

911// Types stored in DB.

912var DBTypes = []any{

913 NextUIDValidity{},

914 Message{},

915 Recipient{},

916 Mailbox{},

917 Subscription{},

918 Outgoing{},

919 Password{},

920 Subjectpass{},

921 SyncState{},

922 Upgrade{},

923 RecipientDomainTLS{},

924 DiskUsage{},

925 LoginSession{},

926 Settings{},

927 FromAddressSettings{},

928 RulesetNoListID{},

929 RulesetNoMsgFrom{},

930 RulesetNoMailbox{},

931 Annotation{},

932 MessageErase{},

933}

934

935// Account holds the information about a user, includings mailboxes, messages, imap subscriptions.

936type Account struct {

937 Name string // Name, according to configuration.

938 Dir string // Directory where account files, including the database, bloom filter, and mail messages, are stored for this account.

939 DBPath string // Path to database with mailboxes, messages, etc.

940 DB *bstore.DB // Open database connection.

941

942 // Channel that is closed if/when account has/gets "threads" accounting (see

943 // Upgrade.Threads).

944 threadsCompleted chan struct{}

945 // If threads upgrade completed with error, this is set. Used for warning during

946 // delivery, or aborting when importing.

947 threadsErr error

948

949 // Message directory of last delivery. Used to check we don't have to make that

950 // directory when delivering.

951 lastMsgDir string

952

953 // If set, consistency checks won't fail on message ModSeq/CreateSeq being zero.

954 skipMessageZeroSeqCheck bool

955

956 // Write lock must be held when modifying account/mailbox/message/flags/annotations

957 // if the change needs to be synchronized with client connections by broadcasting

958 // the changes. Changes that are not protocol-visible do not require a lock, the

959 // database transactions isolate activity, though locking may be necessary to

960 // protect in-memory-only access.

961 //

962 // Read lock for reading mailboxes/messages as a consistent snapsnot (i.e. not

963 // concurrent changes). For longer transactions, e.g. when reading many messages,

964 // the lock can be released while continuing to read from the transaction.

965 //

966 // When making changes to mailboxes/messages, changes must be broadcasted before

967 // releasing the lock to ensure proper UID ordering.

968 sync.RWMutex

969

970 // Reference count, while >0, this account is alive and shared. Protected by

971 // openAccounts, not by account wlock.

972 nused int

973 removed bool // Marked for removal. Last close removes the account directory.

974 closed chan struct{} // Closed when last reference is gone.

975}

976

977type Upgrade struct {

978 ID byte

979 Threads byte // 0: None, 1: Adding MessageID's completed, 2: Adding ThreadID's completed.

980 MailboxModSeq bool // Whether mailboxes have been assigned modseqs.

981 MailboxParentID bool // Setting ParentID on mailboxes.

982 MailboxCounts bool // Global flag about whether we have mailbox flags. Instead of previous per-mailbox boolean.

983 MessageParseVersion int // If different than latest, all messages will be reparsed.

984}

985

986const MessageParseVersionLatest = 2

987

988// upgradeInit is the value for new account database, which don't need any upgrading.

989var upgradeInit = Upgrade{

990 ID: 1, // Singleton.

991 Threads: 2,

992 MailboxModSeq: true,

993 MailboxParentID: true,

994 MailboxCounts: true,

995 MessageParseVersion: MessageParseVersionLatest,

996}

997

998// InitialUIDValidity returns a UIDValidity used for initializing an account.

999// It can be replaced during tests with a predictable value.

1000var InitialUIDValidity = func() uint32 {

1001 return uint32(time.Now().Unix() >> 1) // A 2-second resolution will get us far enough beyond 2038.

1002}

1003

1004var openAccounts = struct {

1005 sync.Mutex

1006 names map[string]*Account

1007}{

1008 names: map[string]*Account{},

1009}

1010

1011func closeAccount(acc *Account) (rerr error) {

1012 // If we need to remove the account files, we do so without the accounts lock.

1013 remove := false

1014 defer func() {

1015 if remove {

1016 log := mlog.New("store", nil)

1017 err := removeAccount(log, acc.Name)

1018 if rerr == nil {

1019 rerr = err

1020 }

1021 close(acc.closed)

1022 }

1023 }()

1024

1025 openAccounts.Lock()

1026 defer openAccounts.Unlock()

1027 acc.nused--

1028 if acc.nused > 0 {

1029 return

1030 }

1031 remove = acc.removed

1032

1033 defer func() {

1034 err := acc.DB.Close()

1035 acc.DB = nil

1036 delete(openAccounts.names, acc.Name)

1037 if !remove {

1038 close(acc.closed)

1039 }

1040

1041 if rerr == nil {

1042 rerr = err

1043 }

1044 }()

1045

1046 // Verify there are no more pending MessageErase records.

1047 l, err := bstore.QueryDB[MessageErase](context.TODO(), acc.DB).List()

1048 if err != nil {

1049 return fmt.Errorf("listing messageerase records: %v", err)

1050 } else if len(l) > 0 {

1051 return fmt.Errorf("messageerase records still present after last account reference is gone: %v", l)

1052 }

1053

1054 return nil

1055}

1056

1057// removeAccount moves the account directory for an account away and removes

1058// all files, and removes the AccountRemove struct from the database.

1059func removeAccount(log mlog.Log, accountName string) error {

1060 log = log.With(slog.String("account", accountName))

1061 log.Info("removing account directory and files")

1062

1063 // First move the account directory away.

1064 odir := filepath.Join(mox.DataDirPath("accounts"), accountName)

1065 tmpdir := filepath.Join(mox.DataDirPath("tmp"), "oldaccount-"+accountName)

1066 if err := os.Rename(odir, tmpdir); err != nil {

1067 return fmt.Errorf("moving account data directory %q out of the way to %q (account not removed): %v", odir, tmpdir, err)

1068 }

1069

1070 var errs []error

1071

1072 // Commit removal to database.

1073 err := AuthDB.Write(context.Background(), func(tx *bstore.Tx) error {

1074 if err := tx.Delete(&AccountRemove{accountName}); err != nil {

1075 return fmt.Errorf("deleting account removal request: %v", err)

1076 }

1077 if err := tlsPublicKeyRemoveForAccount(tx, accountName); err != nil {

1078 return fmt.Errorf("removing tls public keys for account: %v", err)

1079 }

1080

1081 if err := loginAttemptRemoveAccount(tx, accountName); err != nil {

1082 return fmt.Errorf("removing historic login attempts for account: %v", err)

1083 }

1084 return nil

1085 })

1086 if err != nil {

1087 errs = append(errs, fmt.Errorf("remove account from database: %w", err))

1088 }

1089

1090 // Remove the account directory and its message and other files.

1091 if err := os.RemoveAll(tmpdir); err != nil {

1092 errs = append(errs, fmt.Errorf("removing account data directory %q that was moved to %q: %v", odir, tmpdir, err))

1093 }

1094

1095 return errors.Join(errs...)

1096}

1097

1098// OpenAccount opens an account by name.

1099//

1100// No additional data path prefix or ".db" suffix should be added to the name.

1101// A single shared account exists per name.

1102func OpenAccount(log mlog.Log, name string, checkLoginDisabled bool) (*Account, error) {

1103 openAccounts.Lock()

1104 defer openAccounts.Unlock()

1105 if acc, ok := openAccounts.names[name]; ok {

1106 if acc.removed {

1107 return nil, fmt.Errorf("account has been removed")

1108 }

1109

1110 acc.nused++

1111 return acc, nil

1112 }

1113

1114 if a, ok := mox.Conf.Account(name); !ok {

1115 return nil, ErrAccountUnknown

1116 } else if checkLoginDisabled && a.LoginDisabled != "" {

1117 return nil, fmt.Errorf("%w: %s", ErrLoginDisabled, a.LoginDisabled)

1118 }

1119

1120 acc, err := openAccount(log, name)

1121 if err != nil {

1122 return nil, err

1123 }

1124 openAccounts.names[name] = acc

1125 return acc, nil

1126}

1127

1128// openAccount opens an existing account, or creates it if it is missing.

1129// Called with openAccounts lock held.

1130func openAccount(log mlog.Log, name string) (a *Account, rerr error) {

1131 dir := filepath.Join(mox.DataDirPath("accounts"), name)

1132 return OpenAccountDB(log, dir, name)

1133}

1134

1135// OpenAccountDB opens an account database file and returns an initialized account

1136// or error. Only exported for use by subcommands that verify the database file.

1137// Almost all account opens must go through OpenAccount/OpenEmail/OpenEmailAuth.

1138func OpenAccountDB(log mlog.Log, accountDir, accountName string) (a *Account, rerr error) {

1139 log = log.With(slog.String("account", accountName))

1140

1141 dbpath := filepath.Join(accountDir, "index.db")

1142

1143 // Create account if it doesn't exist yet.

1144 isNew := false

1145 if _, err := os.Stat(dbpath); err != nil && os.IsNotExist(err) {

1146 isNew = true

1147 os.MkdirAll(accountDir, 0770)

1148 }

1149

1150 opts := bstore.Options{Timeout: 5 * time.Second, Perm: 0660, RegisterLogger: moxvar.RegisterLogger(dbpath, log.Logger)}

1151 db, err := bstore.Open(context.TODO(), dbpath, &opts, DBTypes...)

1152 if err != nil {

1153 return nil, err

1154 }

1155

1156 defer func() {

1157 if rerr != nil {

1158 err := db.Close()

1159 log.Check(err, "closing database file after error")

1160 if isNew {

1161 err := os.Remove(dbpath)

1162 log.Check(err, "removing new database file after error")

1163 }

1164 }

1165 }()

1166

1167 acc := &Account{

1168 Name: accountName,

1169 Dir: accountDir,

1170 DBPath: dbpath,

1171 DB: db,

1172 nused: 1,

1173 closed: make(chan struct{}),

1174 threadsCompleted: make(chan struct{}),

1175 }

1176

1177 if isNew {

1178 if err := initAccount(db); err != nil {

1179 return nil, fmt.Errorf("initializing account: %v", err)

1180 }

1181

1182 close(acc.threadsCompleted)

1183 return acc, nil

1184 }

1185

1186 // Ensure singletons are present, like DiskUsage and Settings.

1187 // Process pending MessageErase records. Check that next the message ID assigned by

1188 // the database does not already have a file on disk, or increase the sequence so

1189 // it doesn't.

1190 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

1191 if tx.Get(&Settings{ID: 1}) == bstore.ErrAbsent {

1192 if err := tx.Insert(&Settings{ID: 1, ShowAddressSecurity: true}); err != nil {

1193 return err

1194 }

1195 }

1196

1197 du := DiskUsage{ID: 1}

1198 err = tx.Get(&du)

1199 if err == bstore.ErrAbsent {

1200 // No DiskUsage record yet, calculate total size and insert.

1201 err := bstore.QueryTx[Mailbox](tx).FilterEqual("Expunged", false).ForEach(func(mb Mailbox) error {

1202 du.MessageSize += mb.Size

1203 return nil

1204 })

1205 if err != nil {

1206 return err

1207 }

1208 if err := tx.Insert(&du); err != nil {

1209 return err

1210 }

1211 } else if err != nil {

1212 return err

1213 }

1214

1215 var erase []MessageErase

1216 if _, err := bstore.QueryTx[MessageErase](tx).Gather(&erase).Delete(); err != nil {

1217 return fmt.Errorf("fetching messages to erase: %w", err)

1218 }

1219 if len(erase) > 0 {

1220 log.Debug("deleting message files from message erase records", slog.Int("count", len(erase)))

1221 }

1222 var duChanged bool

1223 for _, me := range erase {

1224 // Clear the fields from the message not needed for synchronization.

1225 m := Message{ID: me.ID}

1226 if err := tx.Get(&m); err != nil {

1227 return fmt.Errorf("get message %d to expunge: %w", me.ID, err)

1228 } else if !m.Expunged {

1229 return fmt.Errorf("message %d to erase is not expunged", m.ID)

1230 }

1231

1232 // We remove before we update/commit the database, so we are sure we don't leave

1233 // files behind in case of an error/crash.

1234 p := acc.MessagePath(me.ID)

1235 err := os.Remove(p)

1236 log.Check(err, "removing message file for expunged message", slog.String("path", p))

1237

1238 if !me.SkipUpdateDiskUsage {

1239 du.MessageSize -= m.Size

1240 duChanged = true

1241 }

1242

1243 m.erase()

1244 if err := tx.Update(&m); err != nil {

1245 return fmt.Errorf("save erase of message %d in database: %w", m.ID, err)

1246 }

1247 }

1248

1249 if duChanged {

1250 if err := tx.Update(&du); err != nil {

1251 return fmt.Errorf("saving disk usage after erasing messages: %w", err)

1252 }

1253 }

1254

1255 // Ensure the message directories don't have a higher message ID than occurs in our

1256 // database. If so, increase the next ID used for inserting a message to prevent

1257 // clash during delivery.

1258 last, err := bstore.QueryTx[Message](tx).SortDesc("ID").Limit(1).Get()

1259 if err != nil && err != bstore.ErrAbsent {

1260 return fmt.Errorf("querying last message: %v", err)

1261 }

1262

1263 // We look in the directory where the message is stored (the id can be 0, which is fine).

1264 maxDBID := last.ID

1265 p := acc.MessagePath(maxDBID)

1266 dir := filepath.Dir(p)

1267 maxFSID := maxDBID

1268 // We also try looking for the next directories that would be created for messages,

1269 // until one doesn't exist anymore. We never delete these directories.

1270 for {

1271 np := acc.MessagePath(maxFSID + msgFilesPerDir)

1272 ndir := filepath.Dir(np)

1273 if _, err := os.Stat(ndir); err == nil {

1274 maxFSID = (maxFSID + msgFilesPerDir) &^ (msgFilesPerDir - 1) // First ID for dir.

1275 dir = ndir

1276 } else if errors.Is(err, fs.ErrNotExist) {

1277 break

1278 } else {

1279 return fmt.Errorf("stat next message directory %q: %v", ndir, err)

1280 }

1281 }

1282 // Find highest numbered file within the directory.

1283 entries, err := os.ReadDir(dir)

1284 if err != nil && !errors.Is(err, fs.ErrNotExist) {

1285 return fmt.Errorf("read message directory %q: %v", dir, err)

1286 }

1287 dirFirstID := maxFSID &^ (msgFilesPerDir - 1)

1288 for _, e := range entries {

1289 id, err := strconv.ParseInt(e.Name(), 10, 64)

1290 if err == nil && (id < dirFirstID || id >= dirFirstID+msgFilesPerDir) {

1291 err = fmt.Errorf("directory %s has message id %d outside of range [%d - %d), ignoring", dir, id, dirFirstID, dirFirstID+msgFilesPerDir)

1292 }

1293 if err != nil {

1294 p := filepath.Join(dir, e.Name())

1295 log.Errorx("unrecognized file in message directory, parsing filename as number", err, slog.String("path", p))

1296 } else {

1297 maxFSID = max(maxFSID, id)

1298 }

1299 }

1300 // Warn if we need to increase the message ID in the database.

1301 var mailboxID int64

1302 if maxFSID > maxDBID {

1303 log.Warn("unexpected message file with higher message id than highest id in database, moving database id sequence forward to prevent clashes during future deliveries", slog.Int64("maxdbmsgid", maxDBID), slog.Int64("maxfilemsgid", maxFSID))

1304

1305 mb, err := bstore.QueryTx[Mailbox](tx).Limit(1).Get()

1306 if err != nil {

1307 return fmt.Errorf("get a mailbox: %v", err)

1308 }

1309 mailboxID = mb.ID

1310 }

1311 for maxFSID > maxDBID {

1312 // Set fields that must be non-zero.

1313 m := Message{

1314 UID: ^UID(0),

1315 MailboxID: mailboxID,

1316 }

1317 // Insert and delete to increase the sequence, silly but effective.

1318 if err := tx.Insert(&m); err != nil {

1319 return fmt.Errorf("inserting message to increase id: %v", err)

1320 }

1321 if err := tx.Delete(&m); err != nil {

1322 return fmt.Errorf("deleting message after increasing id: %v", err)

1323 }

1324 maxDBID = m.ID

1325 }

1326

1327 return nil

1328 })

1329 if err != nil {

1330 return nil, fmt.Errorf("calculating counts for mailbox, inserting settings, expunging messages: %v", err)

1331 }

1332

1333 up := Upgrade{ID: 1}

1334 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

1335 err := tx.Get(&up)

1336 if err == bstore.ErrAbsent {

1337 if err := tx.Insert(&up); err != nil {

1338 return fmt.Errorf("inserting initial upgrade record: %v", err)

1339 }

1340 err = nil

1341 }

1342 return err

1343 })

1344 if err != nil {

1345 return nil, fmt.Errorf("checking message threading: %v", err)

1346 }

1347

1348 // Ensure all mailboxes have a modseq based on highest modseq message in each

1349 // mailbox, and a createseq.

1350 if !up.MailboxModSeq {

1351 log.Debug("upgrade: adding modseq to each mailbox")

1352 err := acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1353 var modseq ModSeq

1354

1355 mbl, err := bstore.QueryTx[Mailbox](tx).FilterEqual("Expunged", false).List()

1356 if err != nil {

1357 return fmt.Errorf("listing mailboxes: %v", err)

1358 }

1359 for _, mb := range mbl {

1360 // Get current highest modseq of message in account.

1361 qms := bstore.QueryTx[Message](tx)

1362 qms.FilterNonzero(Message{MailboxID: mb.ID})

1363 qms.SortDesc("ModSeq")

1364 qms.Limit(1)

1365 m, err := qms.Get()

1366 if err == nil {

1367 mb.ModSeq = ModSeq(m.ModSeq.Client())

1368 } else if err == bstore.ErrAbsent {

1369 if modseq == 0 {

1370 modseq, err = acc.NextModSeq(tx)

1371 if err != nil {

1372 return fmt.Errorf("get next mod seq for mailbox without messages: %v", err)

1373 }

1374 }

1375 mb.ModSeq = modseq

1376 } else {

1377 return fmt.Errorf("looking up highest modseq for mailbox: %v", err)

1378 }

1379 mb.CreateSeq = 1

1380 if err := tx.Update(&mb); err != nil {

1381 return fmt.Errorf("updating mailbox with modseq: %v", err)

1382 }

1383 }

1384

1385 up.MailboxModSeq = true

1386 if err := tx.Update(&up); err != nil {

1387 return fmt.Errorf("marking upgrade done: %v", err)

1388 }

1389

1390 return nil

1391 })

1392 if err != nil {

1393 return nil, fmt.Errorf("upgrade: adding modseq to each mailbox: %v", err)

1394 }

1395 }

1396

1397 // Add ParentID to mailboxes.

1398 if !up.MailboxParentID {

1399 log.Debug("upgrade: setting parentid on each mailbox")

1400

1401 err := acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1402 mbl, err := bstore.QueryTx[Mailbox](tx).FilterEqual("Expunged", false).SortAsc("Name").List()

1403 if err != nil {

1404 return fmt.Errorf("listing mailboxes: %w", err)

1405 }

1406

1407 names := map[string]Mailbox{}

1408 for _, mb := range mbl {

1409 names[mb.Name] = mb

1410 }

1411

1412 var modseq ModSeq

1413

1414 // Ensure a parent mailbox for name exists, creating it if needed, including any

1415 // grandparents, up to the top.

1416 var ensureParentMailboxID func(name string) (int64, error)

1417 ensureParentMailboxID = func(name string) (int64, error) {

1418 parentName := mox.ParentMailboxName(name)

1419 if parentName == "" {

1420 return 0, nil

1421 }

1422 parent := names[parentName]

1423 if parent.ID != 0 {

1424 return parent.ID, nil

1425 }

1426

1427 parentParentID, err := ensureParentMailboxID(parentName)

1428 if err != nil {

1429 return 0, fmt.Errorf("creating parent mailbox %q: %w", parentName, err)

1430 }

1431

1432 if modseq == 0 {

1433 modseq, err = a.NextModSeq(tx)

1434 if err != nil {

1435 return 0, fmt.Errorf("get next modseq: %w", err)

1436 }

1437 }

1438

1439 uidvalidity, err := a.NextUIDValidity(tx)

1440 if err != nil {

1441 return 0, fmt.Errorf("next uid validity: %w", err)

1442 }

1443

1444 parent = Mailbox{

1445 CreateSeq: modseq,

1446 ModSeq: modseq,

1447 ParentID: parentParentID,

1448 Name: parentName,

1449 UIDValidity: uidvalidity,

1450 UIDNext: 1,

1451 SpecialUse: SpecialUse{},

1452 HaveCounts: true,

1453 }

1454 if err := tx.Insert(&parent); err != nil {

1455 return 0, fmt.Errorf("creating parent mailbox: %w", err)

1456 }

1457 return parent.ID, nil

1458 }

1459

1460 for _, mb := range mbl {

1461 parentID, err := ensureParentMailboxID(mb.Name)

1462 if err != nil {

1463 return fmt.Errorf("creating missing parent mailbox for mailbox %q: %w", mb.Name, err)

1464 }

1465 mb.ParentID = parentID

1466 if err := tx.Update(&mb); err != nil {

1467 return fmt.Errorf("update mailbox with parentid: %w", err)

1468 }

1469 }

1470

1471 up.MailboxParentID = true

1472 if err := tx.Update(&up); err != nil {

1473 return fmt.Errorf("marking upgrade done: %w", err)

1474 }

1475 return nil

1476 })

1477 if err != nil {

1478 return nil, fmt.Errorf("upgrade: setting parentid on each mailbox: %w", err)

1479 }

1480 }

1481

1482 if !up.MailboxCounts {

1483 log.Debug("upgrade: ensuring all mailboxes have message counts")

1484

1485 err := acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1486 err := bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error {

1487 mc, err := mb.CalculateCounts(tx)

1488 if err != nil {

1489 return err

1490 }

1491 mb.HaveCounts = true

1492 mb.MailboxCounts = mc

1493 return tx.Update(&mb)

1494 })

1495 if err != nil {

1496 return err

1497 }

1498

1499 up.MailboxCounts = true

1500 if err := tx.Update(&up); err != nil {

1501 return fmt.Errorf("marking upgrade done: %w", err)

1502 }

1503 return nil

1504 })

1505 if err != nil {

1506 return nil, fmt.Errorf("upgrade: ensuring message counts on all mailboxes")

1507 }

1508 }

1509

1510 if up.MessageParseVersion != MessageParseVersionLatest {

1511 log.Debug("upgrade: reparsing message for mime structures for new message parse version", slog.Int("current", up.MessageParseVersion), slog.Int("latest", MessageParseVersionLatest))

1512

1513 // Unless we also need to upgrade threading, we'll be reparsing messages in the

1514 // background so opening of the account is quick.

1515 done := make(chan error, 1)

1516 bg := up.Threads == 2

1517

1518 // Increase account use before holding on to account in background.

1519 // Caller holds the lock. The goroutine below decreases nused by calling

1520 // closeAccount.

1521 acc.nused++

1522

1523 go func() {

1524 start := time.Now()

1525

1526 var rerr error

1527 defer func() {

1528 x := recover()

1529 if x != nil {

1530 rerr = fmt.Errorf("unhandled panic: %v", x)

1531 log.Error("unhandled panic reparsing messages", slog.Any("err", x))

1532 debug.PrintStack()

1533 metrics.PanicInc(metrics.Store)

1534 }

1535

1536 if bg && rerr != nil {

1537 log.Errorx("upgrade failed: reparsing message for mime structures for new message parse version", rerr, slog.Duration("duration", time.Since(start)))

1538 }

1539 done <- rerr

1540

1541 // Must be done at end of defer. Our parent context/goroutine has openAccounts lock

1542 // held, so we won't make progress until after the enclosing method has returned.

1543 err := closeAccount(acc)

1544 log.Check(err, "closing account after reparsing messages")

1545 }()

1546

1547 var total int

1548 total, rerr = acc.ReparseMessages(mox.Shutdown, log)

1549 if rerr != nil {

1550 rerr = fmt.Errorf("reparsing messages and updating mime structures in message index: %w", rerr)

1551 return

1552 }

1553

1554 up.MessageParseVersion = MessageParseVersionLatest

1555 rerr = acc.DB.Update(context.TODO(), &up)

1556 if rerr != nil {

1557 rerr = fmt.Errorf("marking latest message parse version: %w", rerr)

1558 return

1559 }

1560

1561 log.Info("upgrade completed: reparsing message for mime structures for new message parse version", slog.Int("total", total), slog.Duration("duration", time.Since(start)))

1562 }()

1563

1564 if !bg {

1565 err := <-done

1566 if err != nil {

1567 return nil, err

1568 }

1569 }

1570 }

1571

1572 if up.Threads == 2 {

1573 close(acc.threadsCompleted)

1574 return acc, nil

1575 }

1576

1577 // Increase account use before holding on to account in background.

1578 // Caller holds the lock. The goroutine below decreases nused by calling

1579 // closeAccount.

1580 acc.nused++

1581

1582 // Ensure all messages have a MessageID and SubjectBase, which are needed when

1583 // matching threads.

1584 // Then assign messages to threads, in the same way we do during imports.

1585 log.Info("upgrading account for threading, in background")

1586 go func() {

1587 defer func() {

1588 err := closeAccount(acc)

1589 log.Check(err, "closing use of account after upgrading account storage for threads")

1590

1591 // Mark that upgrade has finished, possibly error is indicated in threadsErr.

1592 close(acc.threadsCompleted)

1593 }()

1594

1595 defer func() {

1596 x := recover() // Should not happen, but don't take program down if it does.

1597 if x != nil {

1598 log.Error("upgradeThreads panic", slog.Any("err", x))

1599 debug.PrintStack()

1600 metrics.PanicInc(metrics.Upgradethreads)

1601 acc.threadsErr = fmt.Errorf("panic during upgradeThreads: %v", x)

1602 }

1603 }()

1604

1605 err := upgradeThreads(mox.Shutdown, log, acc, up)

1606 if err != nil {

1607 a.threadsErr = err

1608 log.Errorx("upgrading account for threading, aborted", err)

1609 } else {

1610 log.Info("upgrading account for threading, completed")

1611 }

1612 }()

1613 return acc, nil

1614}

1615

1616// ThreadingWait blocks until the one-time account threading upgrade for the

1617// account has completed, and returns an error if not successful.

1618//

1619// To be used before starting an import of messages.

1620func (a *Account) ThreadingWait(log mlog.Log) error {

1621 select {

1622 case <-a.threadsCompleted:

1623 return a.threadsErr

1624 default:

1625 }

1626 log.Debug("waiting for account upgrade to complete")

1627

1628 <-a.threadsCompleted

1629 return a.threadsErr

1630}

1631

1632func initAccount(db *bstore.DB) error {

1633 return db.Write(context.TODO(), func(tx *bstore.Tx) error {

1634 uidvalidity := InitialUIDValidity()

1635

1636 if err := tx.Insert(&upgradeInit); err != nil {

1637 return err

1638 }

1639 if err := tx.Insert(&DiskUsage{ID: 1}); err != nil {

1640 return err

1641 }

1642 if err := tx.Insert(&Settings{ID: 1}); err != nil {

1643 return err

1644 }

1645

1646 modseq, err := nextModSeq(tx)

1647 if err != nil {

1648 return fmt.Errorf("get next modseq: %v", err)

1649 }

1650

1651 if len(mox.Conf.Static.DefaultMailboxes) > 0 {

1652 // Deprecated in favor of InitialMailboxes.

1653 defaultMailboxes := mox.Conf.Static.DefaultMailboxes

1654 mailboxes := []string{"Inbox"}

1655 for _, name := range defaultMailboxes {

1656 if strings.EqualFold(name, "Inbox") {

1657 continue

1658 }

1659 mailboxes = append(mailboxes, name)

1660 }

1661 for _, name := range mailboxes {

1662 mb := Mailbox{

1663 CreateSeq: modseq,

1664 ModSeq: modseq,

1665 ParentID: 0,

1666 Name: name,

1667 UIDValidity: uidvalidity,

1668 UIDNext: 1,

1669 HaveCounts: true,

1670 }

1671 if strings.HasPrefix(name, "Archive") {

1672 mb.Archive = true

1673 } else if strings.HasPrefix(name, "Drafts") {

1674 mb.Draft = true

1675 } else if strings.HasPrefix(name, "Junk") {

1676 mb.Junk = true

1677 } else if strings.HasPrefix(name, "Sent") {

1678 mb.Sent = true

1679 } else if strings.HasPrefix(name, "Trash") {

1680 mb.Trash = true

1681 }

1682 if err := tx.Insert(&mb); err != nil {

1683 return fmt.Errorf("creating mailbox: %w", err)

1684 }

1685 if err := tx.Insert(&Subscription{name}); err != nil {

1686 return fmt.Errorf("adding subscription: %w", err)

1687 }

1688 }

1689 } else {

1690 mailboxes := mox.Conf.Static.InitialMailboxes

1691 var zerouse config.SpecialUseMailboxes

1692 if mailboxes.SpecialUse == zerouse && len(mailboxes.Regular) == 0 {

1693 mailboxes = DefaultInitialMailboxes

1694 }

1695

1696 add := func(name string, use SpecialUse) error {

1697 mb := Mailbox{

1698 CreateSeq: modseq,

1699 ModSeq: modseq,

1700 ParentID: 0,

1701 Name: name,

1702 UIDValidity: uidvalidity,

1703 UIDNext: 1,

1704 SpecialUse: use,

1705 HaveCounts: true,

1706 }

1707 if err := tx.Insert(&mb); err != nil {

1708 return fmt.Errorf("creating mailbox: %w", err)

1709 }

1710 if err := tx.Insert(&Subscription{name}); err != nil {

1711 return fmt.Errorf("adding subscription: %w", err)

1712 }

1713 return nil

1714 }

1715 addSpecialOpt := func(nameOpt string, use SpecialUse) error {

1716 if nameOpt == "" {

1717 return nil

1718 }

1719 return add(nameOpt, use)

1720 }

1721 l := []struct {

1722 nameOpt string

1723 use SpecialUse

1724 }{

1725 {"Inbox", SpecialUse{}},

1726 {mailboxes.SpecialUse.Archive, SpecialUse{Archive: true}},

1727 {mailboxes.SpecialUse.Draft, SpecialUse{Draft: true}},

1728 {mailboxes.SpecialUse.Junk, SpecialUse{Junk: true}},

1729 {mailboxes.SpecialUse.Sent, SpecialUse{Sent: true}},

1730 {mailboxes.SpecialUse.Trash, SpecialUse{Trash: true}},

1731 }

1732 for _, e := range l {

1733 if err := addSpecialOpt(e.nameOpt, e.use); err != nil {

1734 return err

1735 }

1736 }

1737 for _, name := range mailboxes.Regular {

1738 if err := add(name, SpecialUse{}); err != nil {

1739 return err

1740 }

1741 }

1742 }

1743

1744 uidvalidity++

1745 if err := tx.Insert(&NextUIDValidity{1, uidvalidity}); err != nil {

1746 return fmt.Errorf("inserting nextuidvalidity: %w", err)

1747 }

1748 return nil

1749 })

1750}

1751

1752// Remove schedules an account for removal. New opens will fail. When the last

1753// reference is closed, the account files are removed.

1754func (a *Account) Remove(ctx context.Context) error {

1755 openAccounts.Lock()

1756 defer openAccounts.Unlock()

1757

1758 if err := AuthDB.Insert(ctx, &AccountRemove{AccountName: a.Name}); err != nil {

1759 return fmt.Errorf("inserting account removal: %w", err)

1760 }

1761 a.removed = true

1762

1763 return nil

1764}

1765

1766// WaitClosed waits until the last reference to this account is gone and the

1767// account is closed. Used during tests, to ensure the consistency checks run after

1768// expunged messages have been erased.

1769func (a *Account) WaitClosed() {

1770 <-a.closed

1771}

1772

1773// Close reduces the reference count, and closes the database connection when

1774// it was the last user.

1775func (a *Account) Close() error {

1776 if CheckConsistencyOnClose {

1777 xerr := a.CheckConsistency()

1778 err := closeAccount(a)

1779 if xerr != nil {

1780 panic(xerr)

1781 }

1782 return err

1783 }

1784 return closeAccount(a)

1785}

1786

1787// SetSkipMessageModSeqZeroCheck skips consistency checks for Message.ModSeq and

1788// Message.CreateSeq being zero.

1789func (a *Account) SetSkipMessageModSeqZeroCheck(skip bool) {

1790 a.Lock()

1791 defer a.Unlock()

1792 a.skipMessageZeroSeqCheck = true

1793}

1794

1795// CheckConsistency checks the consistency of the database and returns a non-nil

1796// error for these cases:

1797//

1798// - Missing or unexpected on-disk message files.

1799// - Mismatch between message size and length of MsgPrefix and on-disk file.

1800// - Incorrect mailbox counts.

1801// - Incorrect total message size.

1802// - Message with UID >= mailbox uid next.

1803// - Mailbox uidvalidity >= account uid validity.

1804// - Mailbox ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq, and Modseq >= highest message ModSeq.

1805// - Mailbox must have a live parent ID if they are live themselves, live names must be unique.

1806// - Message ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq.

1807// - All messages have a nonzero ThreadID, and no cycles in ThreadParentID, and parent messages the same ThreadParentIDs tail.

1808// - Annotations must have ModSeq > 0, CreateSeq > 0, ModSeq >= CreateSeq and live keys must be unique per mailbox.

1809// - Recalculate junk filter (words and counts) and check they are the same.

1810func (a *Account) CheckConsistency() error {

1811 a.Lock()

1812 defer a.Unlock()

1813

1814 var uidErrors []string // With a limit, could be many.

1815 var modseqErrors []string // With limit.

1816 var fileErrors []string // With limit.

1817 var threadidErrors []string // With limit.

1818 var threadParentErrors []string // With limit.

1819 var threadAncestorErrors []string // With limit.

1820 var errmsgs []string

1821

1822 ctx := context.Background()

1823 log := mlog.New("store", nil)

1824

1825 err := a.DB.Read(ctx, func(tx *bstore.Tx) error {

1826 nuv := NextUIDValidity{ID: 1}

1827 err := tx.Get(&nuv)

1828 if err != nil {

1829 return fmt.Errorf("fetching next uid validity: %v", err)

1830 }

1831

1832 mailboxes := map[int64]Mailbox{} // Also expunged mailboxes.

1833 mailboxNames := map[string]Mailbox{} // Only live names.

1834 err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {

1835 mailboxes[mb.ID] = mb

1836 if !mb.Expunged {

1837 if xmb, ok := mailboxNames[mb.Name]; ok {

1838 errmsg := fmt.Sprintf("mailbox %q exists as id %d and id %d", mb.Name, mb.ID, xmb.ID)

1839 errmsgs = append(errmsgs, errmsg)

1840 }

1841 mailboxNames[mb.Name] = mb

1842 }

1843

1844 if mb.UIDValidity >= nuv.Next {

1845 errmsg := fmt.Sprintf("mailbox %q (id %d) has uidvalidity %d >= account next uidvalidity %d", mb.Name, mb.ID, mb.UIDValidity, nuv.Next)

1846 errmsgs = append(errmsgs, errmsg)

1847 }

1848

1849 if mb.ModSeq == 0 || mb.CreateSeq == 0 || mb.CreateSeq > mb.ModSeq {

1850 errmsg := fmt.Sprintf("mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and createseq <= modseq", mb.Name, mb.ID, mb.ModSeq, mb.CreateSeq)

1851 errmsgs = append(errmsgs, errmsg)

1852 return nil

1853 }

1854 m, err := bstore.QueryTx[Message](tx).FilterNonzero(Message{MailboxID: mb.ID}).SortDesc("ModSeq").Limit(1).Get()

1855 if err == bstore.ErrAbsent {

1856 return nil

1857 } else if err != nil {

1858 return fmt.Errorf("get message with highest modseq for mailbox: %v", err)

1859 } else if mb.ModSeq < m.ModSeq {

1860 errmsg := fmt.Sprintf("mailbox %q (id %d) has modseq %d < highest message modseq is %d", mb.Name, mb.ID, mb.ModSeq, m.ModSeq)

1861 errmsgs = append(errmsgs, errmsg)

1862 }

1863 return nil

1864 })

1865 if err != nil {

1866 return fmt.Errorf("checking mailboxes: %v", err)

1867 }

1868

1869 // Check ParentID and name of parent.

1870 for _, mb := range mailboxNames {

1871 if mox.ParentMailboxName(mb.Name) == "" {

1872 if mb.ParentID == 0 {

1873 continue

1874 }

1875 errmsg := fmt.Sprintf("mailbox %q (id %d) is a root mailbox but has parentid %d", mb.Name, mb.ID, mb.ParentID)

1876 errmsgs = append(errmsgs, errmsg)

1877 } else if mb.ParentID == 0 {

1878 errmsg := fmt.Sprintf("mailbox %q (id %d) is not a root mailbox but has a zero parentid", mb.Name, mb.ID)

1879 errmsgs = append(errmsgs, errmsg)

1880 } else if mox.ParentMailboxName(mb.Name) != mailboxes[mb.ParentID].Name {

1881 errmsg := fmt.Sprintf("mailbox %q (id %d) has parent mailbox id %d with name %q, but parent name should be %q", mb.Name, mb.ID, mb.ParentID, mailboxes[mb.ParentID].Name, mox.ParentMailboxName(mb.Name))

1882 errmsgs = append(errmsgs, errmsg)

1883 }

1884 }

1885

1886 type annotation struct {

1887 mailboxID int64 // Can be 0.

1888 key string

1889 }

1890 annotations := map[annotation]struct{}{}

1891 err = bstore.QueryTx[Annotation](tx).ForEach(func(a Annotation) error {

1892 if !a.Expunged {

1893 k := annotation{a.MailboxID, a.Key}

1894 if _, ok := annotations[k]; ok {

1895 errmsg := fmt.Sprintf("duplicate live annotation key %q for mailbox id %d", a.Key, a.MailboxID)

1896 errmsgs = append(errmsgs, errmsg)

1897 }

1898 annotations[k] = struct{}{}

1899 }

1900 if a.ModSeq == 0 || a.CreateSeq == 0 || a.CreateSeq > a.ModSeq {

1901 errmsg := fmt.Sprintf("annotation %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and modseq >= createseq", a.ID, mailboxes[a.MailboxID].Name, a.MailboxID, a.ModSeq, a.CreateSeq)

1902 errmsgs = append(errmsgs, errmsg)

1903 } else if a.MailboxID > 0 && mailboxes[a.MailboxID].ModSeq < a.ModSeq {

1904 errmsg := fmt.Sprintf("annotation %d in mailbox %q (id %d) has invalid modseq %d > mailbox modseq %d", a.ID, mailboxes[a.MailboxID].Name, a.MailboxID, a.ModSeq, mailboxes[a.MailboxID].ModSeq)

1905 errmsgs = append(errmsgs, errmsg)

1906 }

1907 return nil

1908 })

1909 if err != nil {

1910 return fmt.Errorf("checking mailbox annotations: %v", err)

1911 }

1912

1913 // All message id's from database. For checking for unexpected files afterwards.

1914 messageIDs := map[int64]struct{}{}

1915 eraseMessageIDs := map[int64]bool{} // Value indicates whether to skip updating disk usage.

1916

1917 // If configured, we'll be building up the junk filter for the messages, to compare

1918 // against the on-disk junk filter.

1919 var jf *junk.Filter

1920 conf, _ := a.Conf()

1921 if conf.JunkFilter != nil {

1922 random := make([]byte, 16)

1923 if _, err := cryptorand.Read(random); err != nil {

1924 return fmt.Errorf("reading random: %v", err)

1925 }

1926 dbpath := filepath.Join(mox.DataDirPath("tmp"), fmt.Sprintf("junkfilter-check-%x.db", random))

1927 bloompath := filepath.Join(mox.DataDirPath("tmp"), fmt.Sprintf("junkfilter-check-%x.bloom", random))

1928 os.MkdirAll(filepath.Dir(dbpath), 0700)

1929 defer func() {

1930 err := os.Remove(bloompath)

1931 log.Check(err, "removing temp bloom file")

1932 err = os.Remove(dbpath)

1933 log.Check(err, "removing temp junk filter database file")

1934 }()

1935 jf, err = junk.NewFilter(ctx, log, conf.JunkFilter.Params, dbpath, bloompath)

1936 if err != nil {

1937 return fmt.Errorf("new junk filter: %v", err)

1938 }

1939 defer func() {

1940 err := jf.Close()

1941 log.Check(err, "closing junk filter")

1942 }()

1943 }

1944 var ntrained int

1945

1946 // Get IDs of erase messages not yet removed, they'll have a message file.

1947 err = bstore.QueryTx[MessageErase](tx).ForEach(func(me MessageErase) error {

1948 eraseMessageIDs[me.ID] = me.SkipUpdateDiskUsage

1949 return nil

1950 })

1951 if err != nil {

1952 return fmt.Errorf("listing message erase records")

1953 }

1954

1955 counts := map[int64]MailboxCounts{}

1956 var totalExpungedSize int64

1957 err = bstore.QueryTx[Message](tx).ForEach(func(m Message) error {

1958 mc := counts[m.MailboxID]

1959 mc.Add(m.MailboxCounts())

1960 counts[m.MailboxID] = mc

1961

1962 mb := mailboxes[m.MailboxID]

1963

1964 if (!a.skipMessageZeroSeqCheck && (m.ModSeq == 0 || m.CreateSeq == 0) || m.CreateSeq > m.ModSeq) && len(modseqErrors) < 20 {

1965 modseqerr := fmt.Sprintf("message %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and createseq <= modseq", m.ID, mb.Name, mb.ID, m.ModSeq, m.CreateSeq)

1966 modseqErrors = append(modseqErrors, modseqerr)

1967 }

1968 if m.UID >= mb.UIDNext && len(uidErrors) < 20 {

1969 uiderr := fmt.Sprintf("message %d in mailbox %q (id %d) has uid %d >= mailbox uidnext %d", m.ID, mb.Name, mb.ID, m.UID, mb.UIDNext)

1970 uidErrors = append(uidErrors, uiderr)

1971 }

1972 if m.Expunged {

1973 if skip := eraseMessageIDs[m.ID]; !skip {

1974 totalExpungedSize += m.Size

1975 }

1976 return nil

1977 }

1978

1979 messageIDs[m.ID] = struct{}{}

1980 p := a.MessagePath(m.ID)

1981 st, err := os.Stat(p)

1982 if err != nil {

1983 existserr := fmt.Sprintf("message %d in mailbox %q (id %d) on-disk file %s: %v", m.ID, mb.Name, mb.ID, p, err)

1984 fileErrors = append(fileErrors, existserr)

1985 } else if len(fileErrors) < 20 && m.Size != int64(len(m.MsgPrefix))+st.Size() {

1986 sizeerr := fmt.Sprintf("message %d in mailbox %q (id %d) has size %d != len msgprefix %d + on-disk file size %d = %d", m.ID, mb.Name, mb.ID, m.Size, len(m.MsgPrefix), st.Size(), int64(len(m.MsgPrefix))+st.Size())

1987 fileErrors = append(fileErrors, sizeerr)

1988 }

1989

1990 if m.ThreadID <= 0 && len(threadidErrors) < 20 {

1991 err := fmt.Sprintf("message %d in mailbox %q (id %d) has threadid 0", m.ID, mb.Name, mb.ID)

1992 threadidErrors = append(threadidErrors, err)

1993 }

1994 if slices.Contains(m.ThreadParentIDs, m.ID) && len(threadParentErrors) < 20 {

1995 err := fmt.Sprintf("message %d in mailbox %q (id %d) references itself in threadparentids", m.ID, mb.Name, mb.ID)

1996 threadParentErrors = append(threadParentErrors, err)

1997 }

1998 for i, pid := range m.ThreadParentIDs {

1999 am := Message{ID: pid}

2000 if err := tx.Get(&am); err == bstore.ErrAbsent || err == nil && am.Expunged {

2001 continue

2002 } else if err != nil {

2003 return fmt.Errorf("get ancestor message: %v", err)

2004 } else if !slices.Equal(m.ThreadParentIDs[i+1:], am.ThreadParentIDs) && len(threadAncestorErrors) < 20 {

2005 err := fmt.Sprintf("message %d, thread %d has ancestor ids %v, and ancestor at index %d with id %d should have the same tail but has %v\n", m.ID, m.ThreadID, m.ThreadParentIDs, i, am.ID, am.ThreadParentIDs)

2006 threadAncestorErrors = append(threadAncestorErrors, err)

2007 } else {

2008 break

2009 }

2010 }

2011

2012 if jf != nil {

2013 if m.Junk != m.Notjunk {

2014 ntrained++

2015 if _, err := a.TrainMessage(ctx, log, jf, m.Notjunk, m); err != nil {

2016 return fmt.Errorf("train message: %v", err)

2017 }

2018 // We are not setting m.TrainedJunk, we were only recalculating the words.

2019 }

2020 }

2021

2022 return nil

2023 })

2024 if err != nil {

2025 return fmt.Errorf("reading messages: %v", err)

2026 }

2027

2028 msgdir := filepath.Join(a.Dir, "msg")

2029 err = filepath.WalkDir(msgdir, func(path string, entry fs.DirEntry, err error) error {

2030 if err != nil {

2031 if path == msgdir && errors.Is(err, fs.ErrNotExist) {

2032 return nil

2033 }

2034 return err

2035 }

2036 if entry.IsDir() {

2037 return nil

2038 }

2039 id, err := strconv.ParseInt(filepath.Base(path), 10, 64)

2040 if err != nil {

2041 return fmt.Errorf("parsing message id from path %q: %v", path, err)

2042 }

2043 _, mok := messageIDs[id]

2044 _, meok := eraseMessageIDs[id]

2045 if !mok && !meok {

2046 return fmt.Errorf("unexpected message file %q", path)

2047 }

2048 return nil

2049 })

2050 if err != nil {

2051 return fmt.Errorf("walking message dir: %v", err)

2052 }

2053

2054 var totalMailboxSize int64

2055 for _, mb := range mailboxNames {

2056 totalMailboxSize += mb.Size

2057 if mb.MailboxCounts != counts[mb.ID] {

2058 mbcounterr := fmt.Sprintf("mailbox %q (id %d) has wrong counts %s, should be %s", mb.Name, mb.ID, mb.MailboxCounts, counts[mb.ID])

2059 errmsgs = append(errmsgs, mbcounterr)

2060 }

2061 }

2062

2063 du := DiskUsage{ID: 1}

2064 if err := tx.Get(&du); err != nil {

2065 return fmt.Errorf("get diskusage")

2066 }

2067 if du.MessageSize != totalMailboxSize+totalExpungedSize {

2068 errmsg := fmt.Sprintf("total disk usage message size in database is %d != sum of mailbox message sizes %d + sum unerased expunged message sizes %d", du.MessageSize, totalMailboxSize, totalExpungedSize)

2069 errmsgs = append(errmsgs, errmsg)

2070 }

2071

2072 // Compare on-disk junk filter with our recalculated filter.

2073 if jf != nil {

2074 load := func(f *junk.Filter) (map[junk.Wordscore]struct{}, error) {

2075 words := map[junk.Wordscore]struct{}{}

2076 err := bstore.QueryDB[junk.Wordscore](ctx, f.DB()).ForEach(func(w junk.Wordscore) error {

2077 if w.Ham != 0 || w.Spam != 0 {

2078 words[w] = struct{}{}

2079 }

2080 return nil

2081 })

2082 if err != nil {

2083 return nil, fmt.Errorf("read junk filter wordscores: %v", err)

2084 }

2085 return words, nil

2086 }

2087 if err := jf.Save(); err != nil {

2088 return fmt.Errorf("save recalculated junk filter: %v", err)

2089 }

2090 wordsExp, err := load(jf)

2091 if err != nil {

2092 return fmt.Errorf("read recalculated junk filter: %v", err)

2093 }

2094

2095 ajf, _, err := a.OpenJunkFilter(ctx, log)

2096 if err != nil {

2097 return fmt.Errorf("open account junk filter: %v", err)

2098 }

2099 defer func() {

2100 err := ajf.Close()

2101 log.Check(err, "closing junk filter")

2102 }()

2103 wordsGot, err := load(ajf)

2104 if err != nil {

2105 return fmt.Errorf("read account junk filter: %v", err)

2106 }

2107

2108 if !reflect.DeepEqual(wordsGot, wordsExp) {

2109 errmsg := fmt.Sprintf("unexpected values in junk filter, trained %d of %d\ngot:\n%v\nexpected:\n%v", ntrained, len(messageIDs), wordsGot, wordsExp)

2110 errmsgs = append(errmsgs, errmsg)

2111 }

2112 }

2113

2114 return nil

2115 })

2116 if err != nil {

2117 return err

2118 }

2119 errmsgs = append(errmsgs, uidErrors...)

2120 errmsgs = append(errmsgs, modseqErrors...)

2121 errmsgs = append(errmsgs, fileErrors...)

2122 errmsgs = append(errmsgs, threadidErrors...)

2123 errmsgs = append(errmsgs, threadParentErrors...)

2124 errmsgs = append(errmsgs, threadAncestorErrors...)

2125 if len(errmsgs) > 0 {

2126 return fmt.Errorf("%s", strings.Join(errmsgs, "; "))

2127 }

2128 return nil

2129}

2130

2131// Conf returns the configuration for this account if it still exists. During

2132// an SMTP session, a configuration update may drop an account.

2133func (a *Account) Conf() (config.Account, bool) {

2134 return mox.Conf.Account(a.Name)

2135}

2136

2137// NextUIDValidity returns the next new/unique uidvalidity to use for this account.

2138func (a *Account) NextUIDValidity(tx *bstore.Tx) (uint32, error) {

2139 nuv := NextUIDValidity{ID: 1}

2140 if err := tx.Get(&nuv); err != nil {

2141 return 0, err

2142 }

2143 v := nuv.Next

2144 nuv.Next++

2145 if err := tx.Update(&nuv); err != nil {

2146 return 0, err

2147 }

2148 return v, nil

2149}

2150

2151// NextModSeq returns the next modification sequence, which is global per account,

2152// over all types.

2153func (a *Account) NextModSeq(tx *bstore.Tx) (ModSeq, error) {

2154 return nextModSeq(tx)

2155}

2156

2157func nextModSeq(tx *bstore.Tx) (ModSeq, error) {

2158 v := SyncState{ID: 1}

2159 if err := tx.Get(&v); err == bstore.ErrAbsent {

2160 // We start assigning from modseq 2. Modseq 0 is not usable, so returned as 1, so

2161 // already used.

2162 // HighestDeletedModSeq is -1 so comparison against the default ModSeq zero value

2163 // makes sense.

2164 v = SyncState{1, 2, -1}

2165 return v.LastModSeq, tx.Insert(&v)

2166 } else if err != nil {

2167 return 0, err

2168 }

2169 v.LastModSeq++

2170 return v.LastModSeq, tx.Update(&v)

2171}

2172

2173func (a *Account) HighestDeletedModSeq(tx *bstore.Tx) (ModSeq, error) {

2174 v := SyncState{ID: 1}

2175 err := tx.Get(&v)

2176 if err == bstore.ErrAbsent {

2177 return 0, nil

2178 }

2179 return v.HighestDeletedModSeq, err

2180}

2181

2182// WithWLock runs fn with account writelock held. Necessary for account/mailbox

2183// modification. For message delivery, a read lock is required.

2184func (a *Account) WithWLock(fn func()) {

2185 a.Lock()

2186 defer a.Unlock()

2187 fn()

2188}

2189

2190// WithRLock runs fn with account read lock held. Needed for message delivery.

2191func (a *Account) WithRLock(fn func()) {

2192 a.RLock()

2193 defer a.RUnlock()

2194 fn()

2195}

2196

2197// AddOpts influence which work MessageAdd does. Some callers can batch

2198// checks/operations efficiently. For convenience and safety, a zero AddOpts does

2199// all the checks and work.

2200type AddOpts struct {

2201 SkipCheckQuota bool

2202

2203 // If set, the message size is not added to the disk usage. Caller must do that,

2204 // e.g. for many messages at once. If used together with SkipCheckQuota, the

2205 // DiskUsage is not read for database when adding a message.

2206 SkipUpdateDiskUsage bool

2207

2208 // Do not fsync the delivered message file. Useful when copying message files from

2209 // another mailbox. The hardlink created during delivery only needs a directory

2210 // fsync.

2211 SkipSourceFileSync bool

2212

2213 // The directory in which the message file is delivered, typically with a hard

2214 // link, is not fsynced. Useful when delivering many files. A single or few

2215 // directory fsyncs are more efficient.

2216 SkipDirSync bool

2217

2218 // Do not assign thread information to a message. Useful when importing many

2219 // messages and assigning threads efficiently after importing messages.

2220 SkipThreads bool

2221

2222 // If JunkFilter is set, it is used for training. If not set, and the filter must

2223 // be trained for a message, the junk filter is opened, modified and saved to disk.

2224 JunkFilter *junk.Filter

2225

2226 SkipTraining bool

2227

2228 // If true, a preview will be generated if the Message doesn't already have one.

2229 SkipPreview bool

2230}

2231

2232// todo optimization: when moving files, we open the original, call MessageAdd() which hardlinks it and close the file gain. when passing the filename, we could just use os.Link, saves 2 syscalls.

2233

2234// MessageAdd delivers a mail message to the account.

2235//

2236// The file is hardlinked or copied, the caller must clean up the original file. If

2237// this call succeeds, but the database transaction with the change can't be

2238// committed, the caller must clean up the delivered message file identified by

2239// m.ID.

2240//

2241// If the message does not fit in the quota, an error with ErrOverQuota is returned

2242// and the mailbox and message are unchanged and the transaction can continue. For

2243// other errors, the caller must abort the transaction.

2244//

2245// The message, with msg.MsgPrefix and msgFile combined, must have a header

2246// section. The caller is responsible for adding a header separator to

2247// msg.MsgPrefix if missing from an incoming message.

2248//

2249// If UID is not set, it is assigned automatically.

2250//

2251// If the message ModSeq is zero, it is assigned automatically. If the message

2252// CreateSeq is zero, it is set to ModSeq. The mailbox ModSeq is set to the message

2253// ModSeq.

2254//

2255// If the message does not fit in the quota, an error with ErrOverQuota is returned

2256// and the mailbox and message are unchanged and the transaction can continue. For

2257// other errors, the caller must abort the transaction.

2258//

2259// If the destination mailbox has the Sent special-use flag, the message is parsed

2260// for its recipients (to/cc/bcc). Their domains are added to Recipients for use in

2261// reputation classification.

2262//

2263// Must be called with account write lock held.

2264//

2265// Caller must save the mailbox after MessageAdd returns, and broadcast changes for

2266// new the message, updated mailbox counts and possibly new mailbox keywords.

2267func (a *Account) MessageAdd(log mlog.Log, tx *bstore.Tx, mb *Mailbox, m *Message, msgFile *os.File, opts AddOpts) (rerr error) {

2268 if m.Expunged {

2269 return fmt.Errorf("cannot deliver expunged message")

2270 }

2271

2272 if !opts.SkipUpdateDiskUsage || !opts.SkipCheckQuota {

2273 du := DiskUsage{ID: 1}

2274 if err := tx.Get(&du); err != nil {

2275 return fmt.Errorf("get disk usage: %v", err)

2276 }

2277

2278 if !opts.SkipCheckQuota {

2279 maxSize := a.QuotaMessageSize()

2280 if maxSize > 0 && m.Size > maxSize-du.MessageSize {

2281 return fmt.Errorf("%w: max size %d bytes", ErrOverQuota, maxSize)

2282 }

2283 }

2284

2285 if !opts.SkipUpdateDiskUsage {

2286 du.MessageSize += m.Size

2287 if err := tx.Update(&du); err != nil {

2288 return fmt.Errorf("update disk usage: %v", err)

2289 }

2290 }

2291 }

2292

2293 m.MailboxID = mb.ID

2294 if m.MailboxOrigID == 0 {

2295 m.MailboxOrigID = mb.ID

2296 }

2297 if m.UID == 0 {

2298 m.UID = mb.UIDNext

2299 if err := mb.UIDNextAdd(1); err != nil {

2300 return fmt.Errorf("adding uid: %v", err)

2301 }

2302 }

2303 if m.ModSeq == 0 {

2304 modseq, err := a.NextModSeq(tx)

2305 if err != nil {

2306 return fmt.Errorf("assigning next modseq: %w", err)

2307 }

2308 m.ModSeq = modseq

2309 } else if m.ModSeq < mb.ModSeq {

2310 return fmt.Errorf("cannot deliver message with modseq %d < mailbox modseq %d", m.ModSeq, mb.ModSeq)

2311 }

2312 if m.CreateSeq == 0 {

2313 m.CreateSeq = m.ModSeq

2314 }

2315 mb.ModSeq = m.ModSeq

2316

2317 if m.SaveDate == nil {

2318 now := time.Now()

2319 m.SaveDate = &now

2320 }

2321 if m.Received.IsZero() {

2322 m.Received = time.Now()

2323 }

2324

2325 if len(m.Keywords) > 0 {

2326 mb.Keywords, _ = MergeKeywords(mb.Keywords, m.Keywords)

2327 }

2328

2329 conf, _ := a.Conf()

2330 m.JunkFlagsForMailbox(*mb, conf)

2331

2332 var part *message.Part

2333 if m.ParsedBuf == nil {

2334 mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile.

2335 p, err := message.EnsurePart(log.Logger, false, mr, m.Size)

2336 if err != nil {

2337 log.Infox("parsing delivered message", err, slog.String("parse", ""), slog.Int64("message", m.ID))

2338 // We continue, p is still valid.

2339 }

2340 part = &p

2341 buf, err := json.Marshal(part)

2342 if err != nil {

2343 return fmt.Errorf("marshal parsed message: %w", err)

2344 }

2345 m.ParsedBuf = buf

2346 }

2347

2348 var partTried bool

2349 getPart := func() *message.Part {

2350 if part != nil {

2351 return part

2352 }

2353 if partTried {

2354 return nil

2355 }

2356 partTried = true

2357 var p message.Part

2358 if err := json.Unmarshal(m.ParsedBuf, &p); err != nil {

2359 log.Errorx("unmarshal parsed message, continuing", err, slog.String("parse", ""))

2360 } else {

2361 mr := FileMsgReader(m.MsgPrefix, msgFile)

2362 p.SetReaderAt(mr)

2363 part = &p

2364 }

2365 return part

2366 }

2367

2368 // If we are delivering to the originally intended mailbox, no need to store the mailbox ID again.

2369 if m.MailboxDestinedID != 0 && m.MailboxDestinedID == m.MailboxOrigID {

2370 m.MailboxDestinedID = 0

2371 }

2372

2373 if m.MessageID == "" && m.SubjectBase == "" && getPart() != nil {

2374 m.PrepareThreading(log, part)

2375 }

2376

2377 if !opts.SkipPreview && m.Preview == nil {

2378 if p := getPart(); p != nil {

2379 s, err := p.Preview(log)

2380 if err != nil {

2381 return fmt.Errorf("generating preview: %v", err)

2382 }

2383 m.Preview = &s

2384 }

2385 }

2386

2387 // Assign to thread (if upgrade has completed).

2388 noThreadID := opts.SkipThreads

2389 if m.ThreadID == 0 && !opts.SkipThreads && getPart() != nil {

2390 select {

2391 case <-a.threadsCompleted:

2392 if a.threadsErr != nil {

2393 log.Info("not assigning threads for new delivery, upgrading to threads failed")

2394 noThreadID = true

2395 } else {

2396 if err := assignThread(log, tx, m, part); err != nil {

2397 return fmt.Errorf("assigning thread: %w", err)

2398 }

2399 }

2400 default:

2401 // note: since we have a write transaction to get here, we can't wait for the

2402 // thread upgrade to finish.

2403 // If we don't assign a threadid the upgrade process will do it.

2404 log.Info("not assigning threads for new delivery, upgrading to threads in progress which will assign this message")

2405 noThreadID = true

2406 }

2407 }

2408

2409 if err := tx.Insert(m); err != nil {

2410 return fmt.Errorf("inserting message: %w", err)

2411 }

2412 if !noThreadID && m.ThreadID == 0 {

2413 m.ThreadID = m.ID

2414 if err := tx.Update(m); err != nil {

2415 return fmt.Errorf("updating message for its own thread id: %w", err)

2416 }

2417 }

2418

2419 // todo: perhaps we should match the recipients based on smtp submission and a matching message-id? we now miss the addresses in bcc's if the mail client doesn't save a message that includes the bcc header in the sent mailbox.

2420 if mb.Sent && getPart() != nil && part.Envelope != nil {

2421 e := part.Envelope

2422 sent := e.Date

2423 if sent.IsZero() {

2424 sent = m.Received

2425 }

2426 if sent.IsZero() {

2427 sent = time.Now()

2428 }

2429 addrs := append(append(e.To, e.CC...), e.BCC...)

2430 for _, addr := range addrs {

2431 if addr.User == "" {

2432 // Would trigger error because Recipient.Localpart must be nonzero. todo: we could allow empty localpart in db, and filter by not using FilterNonzero.

2433 log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", slog.Any("address", addr))

2434 continue

2435 }

2436 d, err := dns.ParseDomain(addr.Host)

2437 if err != nil {

2438 log.Debugx("parsing domain in to/cc/bcc address", err, slog.Any("address", addr))

2439 continue

2440 }

2441 lp, err := smtp.ParseLocalpart(addr.User)

2442 if err != nil {

2443 log.Debugx("parsing localpart in to/cc/bcc address", err, slog.Any("address", addr))

2444 continue

2445 }

2446 mr := Recipient{

2447 MessageID: m.ID,

2448 Localpart: lp.String(),

2449 Domain: d.Name(),

2450 OrgDomain: publicsuffix.Lookup(context.TODO(), log.Logger, d).Name(),

2451 Sent: sent,

2452 }

2453 if err := tx.Insert(&mr); err != nil {

2454 return fmt.Errorf("inserting sent message recipients: %w", err)

2455 }

2456 }

2457 }

2458

2459 msgPath := a.MessagePath(m.ID)

2460 msgDir := filepath.Dir(msgPath)

2461 if a.lastMsgDir != msgDir {

2462 os.MkdirAll(msgDir, 0770)

2463 if err := moxio.SyncDir(log, msgDir); err != nil {

2464 return fmt.Errorf("sync message dir: %w", err)

2465 }

2466 a.lastMsgDir = msgDir

2467 }

2468

2469 // Sync file data to disk.

2470 if !opts.SkipSourceFileSync {

2471 if err := msgFile.Sync(); err != nil {

2472 return fmt.Errorf("fsync message file: %w", err)

2473 }

2474 }

2475

2476 if err := moxio.LinkOrCopy(log, msgPath, msgFile.Name(), &moxio.AtReader{R: msgFile}, true); err != nil {

2477 return fmt.Errorf("linking/copying message to new file: %w", err)

2478 }

2479

2480 defer func() {

2481 if rerr != nil {

2482 err := os.Remove(msgPath)

2483 log.Check(err, "removing delivered message file", slog.String("path", msgPath))

2484 }

2485 }()

2486

2487 if !opts.SkipDirSync {

2488 if err := moxio.SyncDir(log, msgDir); err != nil {

2489 return fmt.Errorf("sync directory: %w", err)

2490 }

2491 }

2492

2493 if !opts.SkipTraining && m.NeedsTraining() && a.HasJunkFilter() {

2494 jf, opened, err := a.ensureJunkFilter(context.TODO(), log, opts.JunkFilter)

2495 if err != nil {

2496 return fmt.Errorf("open junk filter: %w", err)

2497 }

2498 defer func() {

2499 if jf != nil && opened {

2500 err := jf.CloseDiscard()

2501 log.Check(err, "closing junk filter without saving")

2502 }

2503 }()

2504

2505 // todo optimize: should let us do the tx.Update of m if needed. we should at least merge it with the common case of setting a thread id. and we should try to merge that with the insert by expliciting getting the next id from bstore.

2506

2507 if err := a.RetrainMessage(context.TODO(), log, tx, jf, m); err != nil {

2508 return fmt.Errorf("training junkfilter: %w", err)

2509 }

2510

2511 if opened {

2512 err := jf.Close()

2513 jf = nil

2514 if err != nil {

2515 return fmt.Errorf("close junk filter: %w", err)

2516 }

2517 }

2518 }

2519

2520 mb.MailboxCounts.Add(m.MailboxCounts())

2521

2522 return nil

2523}

2524

2525// SetPassword saves a new password for this account. This password is used for

2526// IMAP, SMTP (submission) sessions and the HTTP account web page.

2527//

2528// Callers are responsible for checking if the account has NoCustomPassword set.

2529func (a *Account) SetPassword(log mlog.Log, password string) error {

2530 password, err := precis.OpaqueString.String(password)

2531 if err != nil {

2532 return fmt.Errorf(`password not allowed by "precis"`)

2533 }

2534

2535 if len(password) < 8 {

2536 // We actually check for bytes...

2537 return fmt.Errorf("password must be at least 8 characters long")

2538 }

2539

2540 hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)

2541 if err != nil {

2542 return fmt.Errorf("generating password hash: %w", err)

2543 }

2544

2545 err = a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

2546 if _, err := bstore.QueryTx[Password](tx).Delete(); err != nil {

2547 return fmt.Errorf("deleting existing password: %v", err)

2548 }

2549 var pw Password

2550 pw.Hash = string(hash)

2551

2552 // CRAM-MD5 calculates an HMAC-MD5, with the password as key, over a per-attempt

2553 // unique text that includes a timestamp. HMAC performs two hashes. Both times, the

2554 // first block is based on the key/password. We hash those first blocks now, and

2555 // store the hash state in the database. When we actually authenticate, we'll

2556 // complete the HMAC by hashing only the text. We cannot store crypto/hmac's hash,

2557 // because it does not expose its internal state and isn't a BinaryMarshaler.

2558 // ../rfc/2104:121

2559 pw.CRAMMD5.Ipad = md5.New()

2560 pw.CRAMMD5.Opad = md5.New()

2561 key := []byte(password)

2562 if len(key) > 64 {

2563 t := md5.Sum(key)

2564 key = t[:]

2565 }

2566 ipad := make([]byte, md5.BlockSize)

2567 opad := make([]byte, md5.BlockSize)

2568 copy(ipad, key)

2569 copy(opad, key)

2570 for i := range ipad {

2571 ipad[i] ^= 0x36

2572 opad[i] ^= 0x5c

2573 }

2574 pw.CRAMMD5.Ipad.Write(ipad)

2575 pw.CRAMMD5.Opad.Write(opad)

2576

2577 pw.SCRAMSHA1.Salt = scram.MakeRandom()

2578 pw.SCRAMSHA1.Iterations = 2 * 4096

2579 pw.SCRAMSHA1.SaltedPassword = scram.SaltPassword(sha1.New, password, pw.SCRAMSHA1.Salt, pw.SCRAMSHA1.Iterations)

2580

2581 pw.SCRAMSHA256.Salt = scram.MakeRandom()

2582 pw.SCRAMSHA256.Iterations = 4096

2583 pw.SCRAMSHA256.SaltedPassword = scram.SaltPassword(sha256.New, password, pw.SCRAMSHA256.Salt, pw.SCRAMSHA256.Iterations)

2584

2585 if err := tx.Insert(&pw); err != nil {

2586 return fmt.Errorf("inserting new password: %v", err)

2587 }

2588

2589 return sessionRemoveAll(context.TODO(), log, tx, a.Name)

2590 })

2591 if err == nil {

2592 log.Info("new password set for account", slog.String("account", a.Name))

2593 }

2594 return err

2595}

2596

2597// SessionsClear invalidates all (web) login sessions for the account.

2598func (a *Account) SessionsClear(ctx context.Context, log mlog.Log) error {

2599 return a.DB.Write(ctx, func(tx *bstore.Tx) error {

2600 return sessionRemoveAll(ctx, log, tx, a.Name)

2601 })

2602}

2603

2604// Subjectpass returns the signing key for use with subjectpass for the given

2605// email address with canonical localpart.

2606func (a *Account) Subjectpass(email string) (key string, err error) {

2607 return key, a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

2608 v := Subjectpass{Email: email}

2609 err := tx.Get(&v)

2610 if err == nil {

2611 key = v.Key

2612 return nil

2613 }

2614 if !errors.Is(err, bstore.ErrAbsent) {

2615 return fmt.Errorf("get subjectpass key from accounts database: %w", err)

2616 }

2617 key = ""

2618 const chars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"

2619 buf := make([]byte, 16)

2620 if _, err := cryptorand.Read(buf); err != nil {

2621 return err

2622 }

2623 for _, b := range buf {

2624 key += string(chars[int(b)%len(chars)])

2625 }

2626 v.Key = key

2627 return tx.Insert(&v)

2628 })

2629}

2630

2631// Ensure mailbox is present in database, adding records for the mailbox and its

2632// parents if they aren't present.

2633//

2634// If subscribe is true, any mailboxes that were created will also be subscribed to.

2635//

2636// The leaf mailbox is created with special-use flags, taking the flags away from

2637// other mailboxes, and reflecting that in the returned changes.

2638//

2639// Modseq is used, and initialized if 0, for created mailboxes.

2640//

2641// Name must be in normalized form, see CheckMailboxName.

2642//

2643// Caller must hold account wlock.

2644// Caller must propagate changes if any.

2645func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool, specialUse SpecialUse, modseq *ModSeq) (mb Mailbox, changes []Change, rerr error) {

2646 if norm.NFC.String(name) != name {

2647 return Mailbox{}, nil, fmt.Errorf("mailbox name not normalized")

2648 }

2649

2650 // Quick sanity check.

2651 if strings.EqualFold(name, "inbox") && name != "Inbox" {

2652 return Mailbox{}, nil, fmt.Errorf("bad casing for inbox")

2653 }

2654

2655 // Get mailboxes with same name or prefix (parents).

2656 elems := strings.Split(name, "/")

2657 q := bstore.QueryTx[Mailbox](tx)

2658 q.FilterEqual("Expunged", false)

2659 q.FilterFn(func(xmb Mailbox) bool {

2660 t := strings.Split(xmb.Name, "/")

2661 return len(t) <= len(elems) && slices.Equal(t, elems[:len(t)])

2662 })

2663 l, err := q.List()

2664 if err != nil {

2665 return Mailbox{}, nil, fmt.Errorf("list mailboxes: %v", err)

2666 }

2667

2668 mailboxes := map[string]Mailbox{}

2669 for _, xmb := range l {

2670 mailboxes[xmb.Name] = xmb

2671 }

2672

2673 p := ""

2674 var exists bool

2675 var parentID int64

2676 for _, elem := range elems {

2677 if p != "" {

2678 p += "/"

2679 }

2680 p += elem

2681 mb, exists = mailboxes[p]

2682 if exists {

2683 parentID = mb.ID

2684 continue

2685 }

2686 uidval, err := a.NextUIDValidity(tx)

2687 if err != nil {

2688 return Mailbox{}, nil, fmt.Errorf("next uid validity: %v", err)

2689 }

2690 if *modseq == 0 {

2691 *modseq, err = a.NextModSeq(tx)

2692 if err != nil {

2693 return Mailbox{}, nil, fmt.Errorf("next modseq: %v", err)

2694 }

2695 }

2696 mb = Mailbox{

2697 CreateSeq: *modseq,

2698 ModSeq: *modseq,

2699 ParentID: parentID,

2700 Name: p,

2701 UIDValidity: uidval,

2702 UIDNext: 1,

2703 HaveCounts: true,

2704 }

2705 err = tx.Insert(&mb)

2706 if err != nil {

2707 return Mailbox{}, nil, fmt.Errorf("creating new mailbox %q: %v", p, err)

2708 }

2709 parentID = mb.ID

2710

2711 var flags []string

2712 if subscribe {

2713 if tx.Get(&Subscription{p}) != nil {

2714 err := tx.Insert(&Subscription{p})

2715 if err != nil {

2716 return Mailbox{}, nil, fmt.Errorf("subscribing to mailbox %q: %v", p, err)

2717 }

2718 }

2719 flags = []string{`\Subscribed`}

2720 } else if err := tx.Get(&Subscription{p}); err == nil {

2721 flags = []string{`\Subscribed`}

2722 } else if err != bstore.ErrAbsent {

2723 return Mailbox{}, nil, fmt.Errorf("looking up subscription for %q: %v", p, err)

2724 }

2725

2726 changes = append(changes, ChangeAddMailbox{mb, flags})

2727 }

2728

2729 // Clear any special-use flags from existing mailboxes and assign them to this mailbox.

2730 var zeroSpecialUse SpecialUse

2731 if !exists && specialUse != zeroSpecialUse {

2732 var qerr error

2733 clearSpecialUse := func(b bool, fn func(*Mailbox) *bool) {

2734 if !b || qerr != nil {

2735 return

2736 }

2737 qs := bstore.QueryTx[Mailbox](tx)

2738 qs.FilterFn(func(xmb Mailbox) bool {

2739 return *fn(&xmb)

2740 })

2741 xmb, err := qs.Get()

2742 if err == bstore.ErrAbsent {

2743 return

2744 } else if err != nil {

2745 qerr = fmt.Errorf("looking up mailbox with special-use flag: %v", err)

2746 return

2747 }

2748 p := fn(&xmb)

2749 *p = false

2750 xmb.ModSeq = *modseq

2751 if err := tx.Update(&xmb); err != nil {

2752 qerr = fmt.Errorf("clearing special-use flag: %v", err)

2753 } else {

2754 changes = append(changes, xmb.ChangeSpecialUse())

2755 }

2756 }

2757 clearSpecialUse(specialUse.Archive, func(xmb *Mailbox) *bool { return &xmb.Archive })

2758 clearSpecialUse(specialUse.Draft, func(xmb *Mailbox) *bool { return &xmb.Draft })

2759 clearSpecialUse(specialUse.Junk, func(xmb *Mailbox) *bool { return &xmb.Junk })

2760 clearSpecialUse(specialUse.Sent, func(xmb *Mailbox) *bool { return &xmb.Sent })

2761 clearSpecialUse(specialUse.Trash, func(xmb *Mailbox) *bool { return &xmb.Trash })

2762 if qerr != nil {

2763 return Mailbox{}, nil, qerr

2764 }

2765

2766 mb.SpecialUse = specialUse

2767 mb.ModSeq = *modseq

2768 if err := tx.Update(&mb); err != nil {

2769 return Mailbox{}, nil, fmt.Errorf("setting special-use flag for new mailbox: %v", err)

2770 }

2771 changes = append(changes, mb.ChangeSpecialUse())

2772 }

2773 return mb, changes, nil

2774}

2775

2776// MailboxExists checks if mailbox exists.

2777// Caller must hold account rlock.

2778func (a *Account) MailboxExists(tx *bstore.Tx, name string) (bool, error) {

2779 q := bstore.QueryTx[Mailbox](tx)

2780 q.FilterEqual("Expunged", false)

2781 q.FilterEqual("Name", name)

2782 return q.Exists()

2783}

2784

2785// MailboxFind finds a mailbox by name, returning a nil mailbox and nil error if mailbox does not exist.

2786func (a *Account) MailboxFind(tx *bstore.Tx, name string) (*Mailbox, error) {

2787 q := bstore.QueryTx[Mailbox](tx)

2788 q.FilterEqual("Expunged", false)

2789 q.FilterEqual("Name", name)

2790 mb, err := q.Get()

2791 if err == bstore.ErrAbsent {

2792 return nil, nil

2793 }

2794 if err != nil {

2795 return nil, fmt.Errorf("looking up mailbox: %w", err)

2796 }

2797 return &mb, nil

2798}

2799

2800// SubscriptionEnsure ensures a subscription for name exists. The mailbox does not

2801// have to exist. Any parents are not automatically subscribed.

2802// Changes are returned and must be broadcasted by the caller.

2803func (a *Account) SubscriptionEnsure(tx *bstore.Tx, name string) ([]Change, error) {

2804 if err := tx.Get(&Subscription{name}); err == nil {

2805 return nil, nil

2806 }

2807

2808 if err := tx.Insert(&Subscription{name}); err != nil {

2809 return nil, fmt.Errorf("inserting subscription: %w", err)

2810 }

2811

2812 q := bstore.QueryTx[Mailbox](tx)

2813 q.FilterEqual("Expunged", false)

2814 q.FilterEqual("Name", name)

2815 _, err := q.Get()

2816 if err == nil {

2817 return []Change{ChangeAddSubscription{name, nil}}, nil

2818 } else if err != bstore.ErrAbsent {

2819 return nil, fmt.Errorf("looking up mailbox for subscription: %w", err)

2820 }

2821 return []Change{ChangeAddSubscription{name, []string{`\NonExistent`}}}, nil

2822}

2823

2824// MessageRuleset returns the first ruleset (if any) that matches the message

2825// represented by msgPrefix and msgFile, with smtp and validation fields from m.

2826func MessageRuleset(log mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset {

2827 if len(dest.Rulesets) == 0 {

2828 return nil

2829 }

2830

2831 mr := FileMsgReader(msgPrefix, msgFile) // We don't close, it would close the msgFile.

2832 p, err := message.Parse(log.Logger, false, mr)

2833 if err != nil {

2834 log.Errorx("parsing message for evaluating rulesets, continuing with headers", err, slog.String("parse", ""))

2835 // note: part is still set.

2836 }

2837 // todo optimize: only parse header if needed for rulesets. and probably reuse an earlier parsing.

2838 header, err := p.Header()

2839 if err != nil {

2840 log.Errorx("parsing message headers for evaluating rulesets, delivering to default mailbox", err, slog.String("parse", ""))

2841 // todo: reject message?

2842 return nil

2843 }

2844

2845ruleset:

2846 for _, rs := range dest.Rulesets {

2847 if rs.SMTPMailFromRegexpCompiled != nil {

2848 if !rs.SMTPMailFromRegexpCompiled.MatchString(m.MailFrom) {

2849 continue ruleset

2850 }

2851 }

2852 if rs.MsgFromRegexpCompiled != nil {

2853 if m.MsgFromLocalpart == "" && m.MsgFromDomain == "" || !rs.MsgFromRegexpCompiled.MatchString(m.MsgFromLocalpart.String()+"@"+m.MsgFromDomain) {

2854 continue ruleset

2855 }

2856 }

2857

2858 if !rs.VerifiedDNSDomain.IsZero() {

2859 d := rs.VerifiedDNSDomain.Name()

2860 suffix := "." + d

2861 matchDomain := func(s string) bool {

2862 return s == d || strings.HasSuffix(s, suffix)

2863 }

2864 var ok bool

2865 if m.EHLOValidated && matchDomain(m.EHLODomain) {

2866 ok = true

2867 }

2868 if m.MailFromValidated && matchDomain(m.MailFromDomain) {

2869 ok = true

2870 }

2871 for _, d := range m.DKIMDomains {

2872 if matchDomain(d) {

2873 ok = true

2874 break

2875 }

2876 }

2877 if !ok {

2878 continue ruleset

2879 }

2880 }

2881

2882 header:

2883 for _, t := range rs.HeadersRegexpCompiled {

2884 for k, vl := range header {

2885 k = strings.ToLower(k)

2886 if !t[0].MatchString(k) {

2887 continue

2888 }

2889 for _, v := range vl {

2890 v = strings.ToLower(strings.TrimSpace(v))

2891 if t[1].MatchString(v) {

2892 continue header

2893 }

2894 }

2895 }

2896 continue ruleset

2897 }

2898 return &rs

2899 }

2900 return nil

2901}

2902

2903// MessagePath returns the file system path of a message.

2904func (a *Account) MessagePath(messageID int64) string {

2905 return strings.Join(append([]string{a.Dir, "msg"}, messagePathElems(messageID)...), string(filepath.Separator))

2906}

2907

2908// MessageReader opens a message for reading, transparently combining the

2909// message prefix with the original incoming message.

2910func (a *Account) MessageReader(m Message) *MsgReader {

2911 return &MsgReader{prefix: m.MsgPrefix, path: a.MessagePath(m.ID), size: m.Size}

2912}

2913

2914// DeliverDestination delivers an email to dest, based on the configured rulesets.

2915//

2916// Returns ErrOverQuota when account would be over quota after adding message.

2917//

2918// Caller must hold account wlock (mailbox may be created).

2919// Message delivery, possible mailbox creation, and updated mailbox counts are

2920// broadcasted.

2921func (a *Account) DeliverDestination(log mlog.Log, dest config.Destination, m *Message, msgFile *os.File) error {

2922 var mailbox string

2923 rs := MessageRuleset(log, dest, m, m.MsgPrefix, msgFile)

2924 if rs != nil {

2925 mailbox = rs.Mailbox

2926 } else if dest.Mailbox == "" {

2927 mailbox = "Inbox"

2928 } else {

2929 mailbox = dest.Mailbox

2930 }

2931 return a.DeliverMailbox(log, mailbox, m, msgFile)

2932}

2933

2934// DeliverMailbox delivers an email to the specified mailbox.

2935//

2936// Returns ErrOverQuota when account would be over quota after adding message.

2937//

2938// Caller must hold account wlock (mailbox may be created).

2939// Message delivery, possible mailbox creation, and updated mailbox counts are

2940// broadcasted.

2941func (a *Account) DeliverMailbox(log mlog.Log, mailbox string, m *Message, msgFile *os.File) (rerr error) {

2942 var changes []Change

2943

2944 var commit bool

2945 defer func() {

2946 if !commit && m.ID != 0 {

2947 p := a.MessagePath(m.ID)

2948 err := os.Remove(p)

2949 log.Check(err, "remove delivered message file", slog.String("path", p))

2950 m.ID = 0

2951 }

2952 }()

2953

2954 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

2955 mb, chl, err := a.MailboxEnsure(tx, mailbox, true, SpecialUse{}, &m.ModSeq)

2956 if err != nil {

2957 return fmt.Errorf("ensuring mailbox: %w", err)

2958 }

2959 if m.CreateSeq == 0 {

2960 m.CreateSeq = m.ModSeq

2961 }

2962

2963 nmbkeywords := len(mb.Keywords)

2964

2965 if err := a.MessageAdd(log, tx, &mb, m, msgFile, AddOpts{}); err != nil {

2966 return err

2967 }

2968

2969 if err := tx.Update(&mb); err != nil {

2970 return fmt.Errorf("updating mailbox for delivery: %w", err)

2971 }

2972

2973 changes = append(changes, chl...)

2974 changes = append(changes, m.ChangeAddUID(mb), mb.ChangeCounts())

2975 if nmbkeywords != len(mb.Keywords) {

2976 changes = append(changes, mb.ChangeKeywords())

2977 }

2978 return nil

2979 })

2980 if err != nil {

2981 return err

2982 }

2983 commit = true

2984 BroadcastChanges(a, changes)

2985 return nil

2986}

2987

2988type RemoveOpts struct {

2989 JunkFilter *junk.Filter // If set, this filter is used for training, instead of opening and saving the junk filter.

2990}

2991

2992// MessageRemove markes messages as expunged, updates mailbox counts for the

2993// messages, sets a new modseq on the messages and mailbox, untrains the junk

2994// filter and queues the messages for erasing when the last reference has gone.

2995//

2996// Caller must save the modified mailbox to the database.

2997//

2998// The disk usage is not immediately updated. That will happen when the message

2999// is actually removed from disk.

3000//

3001// The junk filter is untrained for the messages if it was trained.

3002// Useful as optimization when messages are moved and the junk/nonjunk flags do not

3003// change (which can happen due to automatic junk/nonjunk flags for mailboxes).

3004//

3005// An empty list of messages results in an error.

3006//

3007// Caller must broadcast changes.

3008//

3009// Must be called with wlock held.

3010func (a *Account) MessageRemove(log mlog.Log, tx *bstore.Tx, modseq ModSeq, mb *Mailbox, opts RemoveOpts, l ...Message) (chremuids ChangeRemoveUIDs, chmbc ChangeMailboxCounts, rerr error) {

3011 if len(l) == 0 {

3012 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("must expunge at least one message")

3013 }

3014

3015 mb.ModSeq = modseq

3016

3017 // Remove any message recipients.

3018 anyIDs := make([]any, len(l))

3019 for i, m := range l {

3020 anyIDs[i] = m.ID

3021 }

3022 qmr := bstore.QueryTx[Recipient](tx)

3023 qmr.FilterEqual("MessageID", anyIDs...)

3024 if _, err := qmr.Delete(); err != nil {

3025 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("deleting message recipients for messages: %w", err)

3026 }

3027

3028 // Loaded lazily.

3029 jf := opts.JunkFilter

3030

3031 // Mark messages expunged.

3032 ids := make([]int64, 0, len(l))

3033 uids := make([]UID, 0, len(l))

3034 for _, m := range l {

3035 ids = append(ids, m.ID)

3036 uids = append(uids, m.UID)

3037

3038 if m.Expunged {

3039 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("message %d is already expunged", m.ID)

3040 }

3041

3042 mb.Sub(m.MailboxCounts())

3043

3044 m.ModSeq = modseq

3045 m.Expunged = true

3046 m.Junk = false

3047 m.Notjunk = false

3048

3049 if err := tx.Update(&m); err != nil {

3050 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("marking message %d expunged: %v", m.ID, err)

3051 }

3052

3053 // Ensure message gets erased in future.

3054 if err := tx.Insert(&MessageErase{m.ID, false}); err != nil {

3055 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("inserting message erase %d : %v", m.ID, err)

3056 }

3057

3058 if m.TrainedJunk == nil || !a.HasJunkFilter() {

3059 continue

3060 }

3061 // Untrain, as needed by updated flags Junk/Notjunk to false.

3062 if jf == nil {

3063 var err error

3064 jf, _, err = a.OpenJunkFilter(context.TODO(), log)

3065 if err != nil {

3066 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("open junk filter: %v", err)

3067 }

3068 defer func() {

3069 err := jf.Close()

3070 if rerr == nil {

3071 rerr = err

3072 } else {

3073 log.Check(err, "closing junk filter")

3074 }

3075 }()

3076 }

3077 if err := a.RetrainMessage(context.TODO(), log, tx, jf, &m); err != nil {

3078 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("retraining expunged messages: %w", err)

3079 }

3080 }

3081

3082 return ChangeRemoveUIDs{mb.ID, uids, modseq, ids, mb.UIDNext, mb.MessageCountIMAP(), uint32(mb.MailboxCounts.Unseen)}, mb.ChangeCounts(), nil

3083}

3084

3085// TidyRejectsMailbox removes old reject emails, and returns whether there is space for a new delivery.

3086//

3087// The changed mailbox is saved to the database.

3088//

3089// Caller most hold account wlock.

3090// Caller must broadcast changes.

3091func (a *Account) TidyRejectsMailbox(log mlog.Log, tx *bstore.Tx, mbRej *Mailbox) (changes []Change, hasSpace bool, rerr error) {

3092 // Gather old messages to expunge.

3093 old := time.Now().Add(-14 * 24 * time.Hour)

3094 qdel := bstore.QueryTx[Message](tx)

3095 qdel.FilterNonzero(Message{MailboxID: mbRej.ID})

3096 qdel.FilterEqual("Expunged", false)

3097 qdel.FilterLess("Received", old)

3098 qdel.SortAsc("UID")

3099 expunge, err := qdel.List()

3100 if err != nil {

3101 return nil, false, fmt.Errorf("listing old messages: %w", err)

3102 }

3103

3104 if len(expunge) > 0 {

3105 modseq, err := a.NextModSeq(tx)

3106 if err != nil {

3107 return nil, false, fmt.Errorf("next mod seq: %v", err)

3108 }

3109

3110 chremuids, chmbcounts, err := a.MessageRemove(log, tx, modseq, mbRej, RemoveOpts{}, expunge...)

3111 if err != nil {

3112 return nil, false, fmt.Errorf("removing messages: %w", err)

3113 }

3114 if err := tx.Update(mbRej); err != nil {

3115 return nil, false, fmt.Errorf("updating mailbox: %v", err)

3116 }

3117 changes = append(changes, chremuids, chmbcounts)

3118 }

3119

3120 // We allow up to n messages.

3121 qcount := bstore.QueryTx[Message](tx)

3122 qcount.FilterNonzero(Message{MailboxID: mbRej.ID})

3123 qcount.FilterEqual("Expunged", false)

3124 qcount.Limit(1000)

3125 n, err := qcount.Count()

3126 if err != nil {

3127 return nil, false, fmt.Errorf("counting rejects: %w", err)

3128 }

3129 hasSpace = n < 1000

3130

3131 return changes, hasSpace, nil

3132}

3133

3134// RejectsRemove removes a message from the rejects mailbox if present.

3135//

3136// Caller most hold account wlock.

3137// Changes are broadcasted.

3138func (a *Account) RejectsRemove(log mlog.Log, rejectsMailbox, messageID string) error {

3139 var changes []Change

3140

3141 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

3142 mb, err := a.MailboxFind(tx, rejectsMailbox)

3143 if err != nil {

3144 return fmt.Errorf("finding mailbox: %w", err)

3145 }

3146 if mb == nil {

3147 return nil

3148 }

3149

3150 q := bstore.QueryTx[Message](tx)

3151 q.FilterNonzero(Message{MailboxID: mb.ID, MessageID: messageID})

3152 q.FilterEqual("Expunged", false)

3153 expunge, err := q.List()

3154 if err != nil {

3155 return fmt.Errorf("listing messages to remove: %w", err)

3156 }

3157

3158 if len(expunge) == 0 {

3159 return nil

3160 }

3161

3162 modseq, err := a.NextModSeq(tx)

3163 if err != nil {

3164 return fmt.Errorf("get next mod seq: %v", err)

3165 }

3166

3167 chremuids, chmbcounts, err := a.MessageRemove(log, tx, modseq, mb, RemoveOpts{}, expunge...)

3168 if err != nil {

3169 return fmt.Errorf("removing messages: %w", err)

3170 }

3171 changes = append(changes, chremuids, chmbcounts)

3172

3173 if err := tx.Update(mb); err != nil {

3174 return fmt.Errorf("saving mailbox: %w", err)

3175 }

3176

3177 return nil

3178 })

3179 if err != nil {

3180 return err

3181 }

3182

3183 BroadcastChanges(a, changes)

3184

3185 return nil

3186}

3187

3188// AddMessageSize adjusts the DiskUsage.MessageSize by size.

3189func (a *Account) AddMessageSize(log mlog.Log, tx *bstore.Tx, size int64) error {

3190 du := DiskUsage{ID: 1}

3191 if err := tx.Get(&du); err != nil {

3192 return fmt.Errorf("get diskusage: %v", err)

3193 }

3194 du.MessageSize += size

3195 if du.MessageSize < 0 {

3196 log.Error("negative total message size", slog.Int64("delta", size), slog.Int64("newtotalsize", du.MessageSize))

3197 }

3198 if err := tx.Update(&du); err != nil {

3199 return fmt.Errorf("update total message size: %v", err)

3200 }

3201 return nil

3202}

3203

3204// QuotaMessageSize returns the effective maximum total message size for an

3205// account. Returns 0 if there is no maximum.

3206func (a *Account) QuotaMessageSize() int64 {

3207 conf, _ := a.Conf()

3208 size := conf.QuotaMessageSize

3209 if size == 0 {

3210 size = mox.Conf.Static.QuotaMessageSize

3211 }

3212 if size < 0 {

3213 size = 0

3214 }

3215 return size

3216}

3217

3218// CanAddMessageSize checks if a message of size bytes can be added, depending on

3219// total message size and configured quota for account.

3220func (a *Account) CanAddMessageSize(tx *bstore.Tx, size int64) (ok bool, maxSize int64, err error) {

3221 maxSize = a.QuotaMessageSize()

3222 if maxSize <= 0 {

3223 return true, 0, nil

3224 }

3225

3226 du := DiskUsage{ID: 1}

3227 if err := tx.Get(&du); err != nil {

3228 return false, maxSize, fmt.Errorf("get diskusage: %v", err)

3229 }

3230 return du.MessageSize+size <= maxSize, maxSize, nil

3231}

3232

3233// We keep a cache of recent successful authentications, so we don't have to bcrypt successful calls each time.

3234var authCache = struct {

3235 sync.Mutex

3236 success map[authKey]string

3237}{

3238 success: map[authKey]string{},

3239}

3240

3241type authKey struct {

3242 email, hash string

3243}

3244

3245// StartAuthCache starts a goroutine that regularly clears the auth cache.

3246func StartAuthCache() {

3247 go manageAuthCache()

3248}

3249

3250func manageAuthCache() {

3251 for {

3252 authCache.Lock()

3253 authCache.success = map[authKey]string{}

3254 authCache.Unlock()

3255 time.Sleep(15 * time.Minute)

3256 }

3257}

3258

3259// OpenEmailAuth opens an account given an email address and password.

3260//

3261// The email address may contain a catchall separator.

3262// For invalid credentials, a nil account is returned, but accName may be

3263// non-empty.

3264func OpenEmailAuth(log mlog.Log, email string, password string, checkLoginDisabled bool) (racc *Account, raccName string, rerr error) {

3265 // We check for LoginDisabled after verifying the password. Otherwise users can get

3266 // messages about the account being disabled without knowing the password.

3267 acc, accName, _, err := OpenEmail(log, email, false)

3268 if err != nil {

3269 return nil, "", err

3270 }

3271

3272 defer func() {

3273 if rerr != nil {

3274 err := acc.Close()

3275 log.Check(err, "closing account after open auth failure")

3276 acc = nil

3277 }

3278 }()

3279

3280 password, err = precis.OpaqueString.String(password)

3281 if err != nil {

3282 return nil, "", ErrUnknownCredentials

3283 }

3284

3285 pw, err := bstore.QueryDB[Password](context.TODO(), acc.DB).Get()

3286 if err != nil {

3287 if err == bstore.ErrAbsent {

3288 return nil, "", ErrUnknownCredentials

3289 }

3290 return nil, "", fmt.Errorf("looking up password: %v", err)

3291 }

3292 authCache.Lock()

3293 ok := len(password) >= 8 && authCache.success[authKey{email, pw.Hash}] == password

3294 authCache.Unlock()

3295 if !ok {

3296 if err := bcrypt.CompareHashAndPassword([]byte(pw.Hash), []byte(password)); err != nil {

3297 return nil, "", ErrUnknownCredentials

3298 }

3299 }

3300 if checkLoginDisabled {

3301 conf, aok := acc.Conf()

3302 if !aok {

3303 return nil, "", fmt.Errorf("cannot find config for account")

3304 } else if conf.LoginDisabled != "" {

3305 return nil, "", fmt.Errorf("%w: %s", ErrLoginDisabled, conf.LoginDisabled)

3306 }

3307 }

3308 authCache.Lock()

3309 authCache.success[authKey{email, pw.Hash}] = password

3310 authCache.Unlock()

3311 return acc, accName, nil

3312}

3313

3314// OpenEmail opens an account given an email address.

3315//

3316// The email address may contain a catchall separator.

3317//

3318// Returns account on success, may return non-empty account name even on error.

3319func OpenEmail(log mlog.Log, email string, checkLoginDisabled bool) (*Account, string, config.Destination, error) {

3320 addr, err := smtp.ParseAddress(email)

3321 if err != nil {

3322 return nil, "", config.Destination{}, fmt.Errorf("%w: %v", ErrUnknownCredentials, err)

3323 }

3324 accountName, _, _, dest, err := mox.LookupAddress(addr.Localpart, addr.Domain, false, false, false)

3325 if err != nil && (errors.Is(err, mox.ErrAddressNotFound) || errors.Is(err, mox.ErrDomainNotFound)) {

3326 return nil, accountName, config.Destination{}, ErrUnknownCredentials

3327 } else if err != nil {

3328 return nil, accountName, config.Destination{}, fmt.Errorf("looking up address: %v", err)

3329 }

3330 acc, err := OpenAccount(log, accountName, checkLoginDisabled)

3331 if err != nil {

3332 return nil, accountName, config.Destination{}, err

3333 }

3334 return acc, accountName, dest, nil

3335}

3336

3337// We store max 1<<shift files in each subdir of an account "msg" directory.

3338// Defaults to 1 for easy use in tests. Set to 13, for 8k message files, in main

3339// for normal operation.

3340var msgFilesPerDirShift = 1

3341var msgFilesPerDir int64 = 1 << msgFilesPerDirShift

3342

3343func MsgFilesPerDirShiftSet(shift int) {

3344 msgFilesPerDirShift = shift

3345 msgFilesPerDir = 1 << shift

3346}

3347

3348// 64 characters, must be power of 2 for MessagePath

3349const msgDirChars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ-_"

3350

3351// MessagePath returns the filename of the on-disk filename, relative to the

3352// containing directory such as <account>/msg or queue.

3353// Returns names like "AB/1".

3354func MessagePath(messageID int64) string {

3355 return strings.Join(messagePathElems(messageID), string(filepath.Separator))

3356}

3357

3358// messagePathElems returns the elems, for a single join without intermediate

3359// string allocations.

3360func messagePathElems(messageID int64) []string {

3361 v := messageID >> msgFilesPerDirShift

3362 dir := ""

3363 for {

3364 dir += string(msgDirChars[int(v)&(len(msgDirChars)-1)])

3365 v >>= 6

3366 if v == 0 {

3367 break

3368 }

3369 }

3370 return []string{dir, strconv.FormatInt(messageID, 10)}

3371}

3372

3373// Set returns a copy of f, with each flag that is true in mask set to the

3374// value from flags.

3375func (f Flags) Set(mask, flags Flags) Flags {

3376 set := func(d *bool, m, v bool) {

3377 if m {

3378 *d = v

3379 }

3380 }

3381 r := f

3382 set(&r.Seen, mask.Seen, flags.Seen)

3383 set(&r.Answered, mask.Answered, flags.Answered)

3384 set(&r.Flagged, mask.Flagged, flags.Flagged)

3385 set(&r.Forwarded, mask.Forwarded, flags.Forwarded)

3386 set(&r.Junk, mask.Junk, flags.Junk)

3387 set(&r.Notjunk, mask.Notjunk, flags.Notjunk)

3388 set(&r.Deleted, mask.Deleted, flags.Deleted)

3389 set(&r.Draft, mask.Draft, flags.Draft)

3390 set(&r.Phishing, mask.Phishing, flags.Phishing)

3391 set(&r.MDNSent, mask.MDNSent, flags.MDNSent)

3392 return r

3393}

3394

3395// Changed returns a mask of flags that have been between f and other.

3396func (f Flags) Changed(other Flags) (mask Flags) {

3397 mask.Seen = f.Seen != other.Seen

3398 mask.Answered = f.Answered != other.Answered

3399 mask.Flagged = f.Flagged != other.Flagged

3400 mask.Forwarded = f.Forwarded != other.Forwarded

3401 mask.Junk = f.Junk != other.Junk

3402 mask.Notjunk = f.Notjunk != other.Notjunk

3403 mask.Deleted = f.Deleted != other.Deleted

3404 mask.Draft = f.Draft != other.Draft

3405 mask.Phishing = f.Phishing != other.Phishing

3406 mask.MDNSent = f.MDNSent != other.MDNSent

3407 return

3408}

3409

3410// Strings returns the flags that are set in their string form.

3411func (f Flags) Strings() []string {

3412 fields := []struct {

3413 word string

3414 have bool

3415 }{

3416 {`$forwarded`, f.Forwarded},

3417 {`$junk`, f.Junk},

3418 {`$mdnsent`, f.MDNSent},

3419 {`$notjunk`, f.Notjunk},

3420 {`$phishing`, f.Phishing},

3421 {`\answered`, f.Answered},

3422 {`\deleted`, f.Deleted},

3423 {`\draft`, f.Draft},

3424 {`\flagged`, f.Flagged},

3425 {`\seen`, f.Seen},

3426 }

3427 var l []string

3428 for _, fh := range fields {

3429 if fh.have {

3430 l = append(l, fh.word)

3431 }

3432 }

3433 return l

3434}

3435

3436var systemWellKnownFlags = map[string]bool{

3437 `\answered`: true,

3438 `\flagged`: true,

3439 `\deleted`: true,

3440 `\seen`: true,

3441 `\draft`: true,

3442 `$junk`: true,

3443 `$notjunk`: true,

3444 `$forwarded`: true,

3445 `$phishing`: true,

3446 `$mdnsent`: true,

3447}

3448

3449// ParseFlagsKeywords parses a list of textual flags into system/known flags, and

3450// other keywords. Keywords are lower-cased and sorted and check for valid syntax.

3451func ParseFlagsKeywords(l []string) (flags Flags, keywords []string, rerr error) {

3452 fields := map[string]*bool{

3453 `\answered`: &flags.Answered,

3454 `\flagged`: &flags.Flagged,

3455 `\deleted`: &flags.Deleted,

3456 `\seen`: &flags.Seen,

3457 `\draft`: &flags.Draft,

3458 `$junk`: &flags.Junk,

3459 `$notjunk`: &flags.Notjunk,

3460 `$forwarded`: &flags.Forwarded,

3461 `$phishing`: &flags.Phishing,

3462 `$mdnsent`: &flags.MDNSent,

3463 }

3464 seen := map[string]bool{}

3465 for _, f := range l {

3466 f = strings.ToLower(f)

3467 if field, ok := fields[f]; ok {

3468 *field = true

3469 } else if seen[f] {

3470 if mox.Pedantic {

3471 return Flags{}, nil, fmt.Errorf("duplicate keyword %s", f)

3472 }

3473 } else {

3474 if err := CheckKeyword(f); err != nil {

3475 return Flags{}, nil, fmt.Errorf("invalid keyword %s", f)

3476 }

3477 keywords = append(keywords, f)

3478 seen[f] = true

3479 }

3480 }

3481 sort.Strings(keywords)

3482 return flags, keywords, nil

3483}

3484

3485// RemoveKeywords removes keywords from l, returning whether any modifications were

3486// made, and a slice, a new slice in case of modifications. Keywords must have been

3487// validated earlier, e.g. through ParseFlagKeywords or CheckKeyword. Should only

3488// be used with valid keywords, not with system flags like \Seen.

3489func RemoveKeywords(l, remove []string) ([]string, bool) {

3490 var copied bool

3491 var changed bool

3492 for _, k := range remove {

3493 if i := slices.Index(l, k); i >= 0 {

3494 if !copied {

3495 l = slices.Clone(l)

3496 copied = true

3497 }

3498 copy(l[i:], l[i+1:])

3499 l = l[:len(l)-1]

3500 changed = true

3501 }

3502 }

3503 return l, changed

3504}

3505

3506// MergeKeywords adds keywords from add into l, returning whether it added any

3507// keyword, and the slice with keywords, a new slice if modifications were made.

3508// Keywords are only added if they aren't already present. Should only be used with

3509// keywords, not with system flags like \Seen.

3510func MergeKeywords(l, add []string) ([]string, bool) {

3511 var copied bool

3512 var changed bool

3513 for _, k := range add {

3514 if !slices.Contains(l, k) {

3515 if !copied {

3516 l = slices.Clone(l)

3517 copied = true

3518 }

3519 l = append(l, k)

3520 changed = true

3521 }

3522 }

3523 if changed {

3524 sort.Strings(l)

3525 }

3526 return l, changed

3527}

3528

3529// CheckKeyword returns an error if kw is not a valid keyword. Kw should

3530// already be in lower-case.

3531func CheckKeyword(kw string) error {

3532 if kw == "" {

3533 return fmt.Errorf("keyword cannot be empty")

3534 }

3535 if systemWellKnownFlags[kw] {

3536 return fmt.Errorf("cannot use well-known flag as keyword")

3537 }

3538 for _, c := range kw {

3539 // ../rfc/9051:6334

3540 if c <= ' ' || c > 0x7e || c >= 'A' && c <= 'Z' || strings.ContainsRune(`(){%*"\]`, c) {

3541 return errors.New(`not a valid keyword, must be lower-case ascii without spaces and without any of these characters: (){%*"\]`)

3542 }

3543 }

3544 return nil

3545}

3546

3547// SendLimitReached checks whether sending a message to recipients would reach

3548// the limit of outgoing messages for the account. If so, the message should

3549// not be sent. If the returned numbers are >= 0, the limit was reached and the

3550// values are the configured limits.

3551//

3552// To limit damage to the internet and our reputation in case of account

3553// compromise, we limit the max number of messages sent in a 24 hour window, both

3554// total number of messages and number of first-time recipients.

3555func (a *Account) SendLimitReached(tx *bstore.Tx, recipients []smtp.Path) (msglimit, rcptlimit int, rerr error) {

3556 conf, _ := a.Conf()

3557 msgmax := conf.MaxOutgoingMessagesPerDay

3558 if msgmax == 0 {

3559 // For human senders, 1000 recipients in a day is quite a lot.

3560 msgmax = 1000

3561 }

3562 rcptmax := conf.MaxFirstTimeRecipientsPerDay

3563 if rcptmax == 0 {

3564 // Human senders may address a new human-sized list of people once in a while. In

3565 // case of a compromise, a spammer will probably try to send to many new addresses.

3566 rcptmax = 200

3567 }

3568

3569 rcpts := map[string]time.Time{}

3570 n := 0

3571 err := bstore.QueryTx[Outgoing](tx).FilterGreater("Submitted", time.Now().Add(-24*time.Hour)).ForEach(func(o Outgoing) error {

3572 n++

3573 if rcpts[o.Recipient].IsZero() || o.Submitted.Before(rcpts[o.Recipient]) {

3574 rcpts[o.Recipient] = o.Submitted

3575 }

3576 return nil

3577 })

3578 if err != nil {

3579 return -1, -1, fmt.Errorf("querying message recipients in past 24h: %w", err)

3580 }

3581 if n+len(recipients) > msgmax {

3582 return msgmax, -1, nil

3583 }

3584

3585 // Only check if max first-time recipients is reached if there are enough messages

3586 // to trigger the limit.

3587 if n+len(recipients) < rcptmax {

3588 return -1, -1, nil

3589 }

3590

3591 isFirstTime := func(rcpt string, before time.Time) (bool, error) {

3592 exists, err := bstore.QueryTx[Outgoing](tx).FilterNonzero(Outgoing{Recipient: rcpt}).FilterLess("Submitted", before).Exists()

3593 return !exists, err

3594 }

3595

3596 firsttime := 0

3597 now := time.Now()

3598 for _, r := range recipients {

3599 if first, err := isFirstTime(r.XString(true), now); err != nil {

3600 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

3601 } else if first {

3602 firsttime++

3603 }

3604 }

3605 for r, t := range rcpts {

3606 if first, err := isFirstTime(r, t); err != nil {

3607 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

3608 } else if first {

3609 firsttime++

3610 }

3611 }

3612 if firsttime > rcptmax {

3613 return -1, rcptmax, nil

3614 }

3615 return -1, -1, nil

3616}

3617

3618var ErrMailboxExpunged = errors.New("mailbox was deleted")

3619

3620// MailboxID gets a mailbox by ID.

3621//

3622// Returns bstore.ErrAbsent if the mailbox does not exist.

3623// Returns ErrMailboxExpunged if the mailbox is expunged.

3624func MailboxID(tx *bstore.Tx, id int64) (Mailbox, error) {

3625 mb := Mailbox{ID: id}

3626 err := tx.Get(&mb)

3627 if err == nil && mb.Expunged {

3628 return Mailbox{}, ErrMailboxExpunged

3629 }

3630 return mb, err

3631}

3632

3633// MailboxCreate creates a new mailbox, including any missing parent mailboxes,

3634// the total list of created mailboxes is returned in created. On success, if

3635// exists is false and rerr nil, the changes must be broadcasted by the caller.

3636//

3637// The mailbox is created with special-use flags, with those flags taken away from

3638// other mailboxes if they have them, reflected in the returned changes.

3639//

3640// Name must be in normalized form, see CheckMailboxName.

3641func (a *Account) MailboxCreate(tx *bstore.Tx, name string, specialUse SpecialUse) (nmb Mailbox, changes []Change, created []string, exists bool, rerr error) {

3642 elems := strings.Split(name, "/")

3643 var p string

3644 var modseq ModSeq

3645 for i, elem := range elems {

3646 if i > 0 {

3647 p += "/"

3648 }

3649 p += elem

3650 exists, err := a.MailboxExists(tx, p)

3651 if err != nil {

3652 return Mailbox{}, nil, nil, false, fmt.Errorf("checking if mailbox exists")

3653 }

3654 if exists {

3655 if i == len(elems)-1 {

3656 return Mailbox{}, nil, nil, true, fmt.Errorf("mailbox already exists")

3657 }

3658 continue

3659 }

3660 mb, nchanges, err := a.MailboxEnsure(tx, p, true, specialUse, &modseq)

3661 if err != nil {

3662 return Mailbox{}, nil, nil, false, fmt.Errorf("ensuring mailbox exists: %v", err)

3663 }

3664 nmb = mb

3665 changes = append(changes, nchanges...)

3666 created = append(created, p)

3667 }

3668 return nmb, changes, created, false, nil

3669}

3670

3671// MailboxRename renames mailbox mbsrc to dst, including children of mbsrc, and

3672// adds missing parents for dst.

3673//

3674// Name must be in normalized form, see CheckMailboxName, and cannot be Inbox.

3675func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc *Mailbox, dst string, modseq *ModSeq) (changes []Change, isInbox, alreadyExists bool, rerr error) {

3676 if mbsrc.Name == "Inbox" || dst == "Inbox" {

3677 return nil, true, false, fmt.Errorf("inbox cannot be renamed")

3678 }

3679

3680 // Check if destination mailbox already exists.

3681 if exists, err := a.MailboxExists(tx, dst); err != nil {

3682 return nil, false, false, fmt.Errorf("checking if destination mailbox exists: %v", err)

3683 } else if exists {

3684 return nil, false, true, fmt.Errorf("destination mailbox already exists")

3685 }

3686

3687 if *modseq == 0 {