1/*

2Package store implements storage for accounts, their mailboxes, IMAP

3subscriptions and messages, and broadcasts updates (e.g. mail delivery) to

4interested sessions (e.g. IMAP connections).

5

6Layout of storage for accounts:

7

8 <DataDir>/accounts/<name>/index.db

9 <DataDir>/accounts/<name>/msg/[a-zA-Z0-9_-]+/<id>

10

11Index.db holds tables for user information, mailboxes, and messages. Message contents

12are stored in the msg/ subdirectory, each in their own file. The on-disk message

13does not contain headers generated during an incoming SMTP transaction, such as

14Received and Authentication-Results headers. Those are in the database to

15prevent having to rewrite incoming messages (e.g. Authentication-Result for DKIM

16signatures can only be determined after having read the message). Messages must

17be read through MsgReader, which transparently adds the prefix from the

18database.

19*/

20package store

21

22// todo: make up a function naming scheme that indicates whether caller should broadcast changes.

23

24import (

25 "context"

26 "crypto/md5"

27 cryptorand "crypto/rand"

28 "crypto/sha1"

29 "crypto/sha256"

30 "encoding"

31 "encoding/json"

32 "errors"

33 "fmt"

34 "hash"

35 "io"

36 "io/fs"

37 "log/slog"

38 "os"

39 "path/filepath"

40 "reflect"

41 "runtime/debug"

42 "slices"

43 "sort"

44 "strconv"

45 "strings"

46 "sync"

47 "time"

48

49 "golang.org/x/crypto/bcrypt"

50 "golang.org/x/text/secure/precis"

51 "golang.org/x/text/unicode/norm"

52

53 "github.com/mjl-/bstore"

54

55 "github.com/mjl-/mox/config"

56 "github.com/mjl-/mox/dns"

57 "github.com/mjl-/mox/junk"

58 "github.com/mjl-/mox/message"

59 "github.com/mjl-/mox/metrics"

60 "github.com/mjl-/mox/mlog"

61 "github.com/mjl-/mox/mox-"

62 "github.com/mjl-/mox/moxio"

63 "github.com/mjl-/mox/moxvar"

64 "github.com/mjl-/mox/publicsuffix"

65 "github.com/mjl-/mox/scram"

66 "github.com/mjl-/mox/smtp"

67)

68

69// If true, each time an account is closed its database file is checked for

70// consistency. If an inconsistency is found, panic is called. Set by default

71// because of all the packages with tests, the mox main function sets it to

72// false again.

73var CheckConsistencyOnClose = true

74

75var (

76 ErrUnknownMailbox = errors.New("no such mailbox")

77 ErrUnknownCredentials = errors.New("credentials not found")

78 ErrAccountUnknown = errors.New("no such account")

79 ErrOverQuota = errors.New("account over quota")

80 ErrLoginDisabled = errors.New("login disabled for account")

81)

82

83var DefaultInitialMailboxes = config.InitialMailboxes{

84 SpecialUse: config.SpecialUseMailboxes{

85 Sent: "Sent",

86 Archive: "Archive",

87 Trash: "Trash",

88 Draft: "Drafts",

89 Junk: "Junk",

90 },

91}

92

93type SCRAM struct {

94 Salt []byte

95 Iterations int

96 SaltedPassword []byte

97}

98

99// CRAMMD5 holds HMAC ipad and opad hashes that are initialized with the first

100// block with (a derivation of) the key/password, so we don't store the password in plain

101// text.

102type CRAMMD5 struct {

103 Ipad hash.Hash

104 Opad hash.Hash

105}

106

107// BinaryMarshal is used by bstore to store the ipad/opad hash states.

108func (c CRAMMD5) MarshalBinary() ([]byte, error) {

109 if c.Ipad == nil || c.Opad == nil {

110 return nil, nil

111 }

112

113 ipad, err := c.Ipad.(encoding.BinaryMarshaler).MarshalBinary()

114 if err != nil {

115 return nil, fmt.Errorf("marshal ipad: %v", err)

116 }

117 opad, err := c.Opad.(encoding.BinaryMarshaler).MarshalBinary()

118 if err != nil {

119 return nil, fmt.Errorf("marshal opad: %v", err)

120 }

121 buf := make([]byte, 2+len(ipad)+len(opad))

122 ipadlen := uint16(len(ipad))

123 buf[0] = byte(ipadlen >> 8)

124 buf[1] = byte(ipadlen >> 0)

125 copy(buf[2:], ipad)

126 copy(buf[2+len(ipad):], opad)

127 return buf, nil

128}

129

130// BinaryUnmarshal is used by bstore to restore the ipad/opad hash states.

131func (c *CRAMMD5) UnmarshalBinary(buf []byte) error {

132 if len(buf) == 0 {

133 *c = CRAMMD5{}

134 return nil

135 }

136 if len(buf) < 2 {

137 return fmt.Errorf("short buffer")

138 }

139 ipadlen := int(uint16(buf[0])<<8 | uint16(buf[1])<<0)

140 if len(buf) < 2+ipadlen {

141 return fmt.Errorf("buffer too short for ipadlen")

142 }

143 ipad := md5.New()

144 opad := md5.New()

145 if err := ipad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2 : 2+ipadlen]); err != nil {

146 return fmt.Errorf("unmarshal ipad: %v", err)

147 }

148 if err := opad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2+ipadlen:]); err != nil {

149 return fmt.Errorf("unmarshal opad: %v", err)

150 }

151 *c = CRAMMD5{ipad, opad}

152 return nil

153}

154

155// Password holds credentials in various forms, for logging in with SMTP/IMAP.

156type Password struct {

157 Hash string // bcrypt hash for IMAP LOGIN, SASL PLAIN and HTTP basic authentication.

158 CRAMMD5 CRAMMD5 // For SASL CRAM-MD5.

159 SCRAMSHA1 SCRAM // For SASL SCRAM-SHA-1.

160 SCRAMSHA256 SCRAM // For SASL SCRAM-SHA-256.

161}

162

163// Subjectpass holds the secret key used to sign subjectpass tokens.

164type Subjectpass struct {

165 Email string // Our destination address (canonical, with catchall localpart stripped).

166 Key string

167}

168

169// NextUIDValidity is a singleton record in the database with the next UIDValidity

170// to use for the next mailbox.

171type NextUIDValidity struct {

172 ID int // Just a single record with ID 1.

173 Next uint32

174}

175

176// SyncState track ModSeqs.

177type SyncState struct {

178 ID int // Just a single record with ID 1.

179

180 // Last used, next assigned will be one higher. The first value we hand out is 2.

181 // That's because 0 (the default value for old existing messages, from before the

182 // Message.ModSeq field) is special in IMAP, so we return it as 1.

183 LastModSeq ModSeq `bstore:"nonzero"`

184

185 // Highest ModSeq of expunged record that we deleted. When a clients synchronizes

186 // and requests changes based on a modseq before this one, we don't have the

187 // history to provide information about deletions. We normally keep these expunged

188 // records around, but we may periodically truly delete them to reclaim storage

189 // space. Initially set to -1 because we don't want to match with any ModSeq in the

190 // database, which can be zero values.

191 HighestDeletedModSeq ModSeq

192}

193

194// Mailbox is collection of messages, e.g. Inbox or Sent.

195type Mailbox struct {

196 ID int64

197

198 CreateSeq ModSeq

199 ModSeq ModSeq `bstore:"index"` // Of last change, or when deleted.

200 Expunged bool

201

202 ParentID int64 `bstore:"ref Mailbox"` // Zero for top-level mailbox.

203

204 // "Inbox" is the name for the special IMAP "INBOX". Slash separated for hierarchy.

205 // Names must be unique for mailboxes that are not expunged.

206 Name string `bstore:"nonzero"`

207

208 // If UIDs are invalidated, e.g. when renaming a mailbox to a previously existing

209 // name, UIDValidity must be changed. Used by IMAP for synchronization.

210 UIDValidity uint32

211

212 // UID likely to be assigned to next message. Used by IMAP to detect messages

213 // delivered to a mailbox.

214 UIDNext UID

215

216 SpecialUse

217

218 // Keywords as used in messages. Storing a non-system keyword for a message

219 // automatically adds it to this list. Used in the IMAP FLAGS response. Only

220 // "atoms" are allowed (IMAP syntax), keywords are case-insensitive, only stored in

221 // lower case (for JMAP), sorted.

222 Keywords []string

223

224 HaveCounts bool // Deprecated. Covered by Upgrade.MailboxCounts. No longer read.

225 MailboxCounts // Statistics about messages, kept up to date whenever a change happens.

226}

227

228// Annotation is a per-mailbox or global (per-account) annotation for the IMAP

229// metadata extension, currently always a private annotation.

230type Annotation struct {

231 ID int64

232

233 CreateSeq ModSeq

234 ModSeq ModSeq `bstore:"index"`

235 Expunged bool

236

237 // Can be zero, indicates global (per-account) annotation.

238 MailboxID int64 `bstore:"ref Mailbox,index MailboxID+Key"`

239

240 // "Entry name", always starts with "/private/" or "/shared/". Stored lower-case,

241 // comparisons must be done case-insensitively.

242 Key string `bstore:"nonzero"`

243

244 IsString bool // If true, the value is a string instead of bytes.

245 Value []byte

246}

247

248// Change returns a broadcastable change for the annotation.

249func (a Annotation) Change(mailboxName string) ChangeAnnotation {

250 return ChangeAnnotation{a.MailboxID, mailboxName, a.Key, a.ModSeq}

251}

252

253// MailboxCounts tracks statistics about messages for a mailbox.

254type MailboxCounts struct {

255 Total int64 // Total number of messages, excluding \Deleted. For JMAP.

256 Deleted int64 // Number of messages with \Deleted flag. Used for IMAP message count that includes messages with \Deleted.

257 Unread int64 // Messages without \Seen, excluding those with \Deleted, for JMAP.

258 Unseen int64 // Messages without \Seen, including those with \Deleted, for IMAP.

259 Size int64 // Number of bytes for all messages.

260}

261

262func (mc MailboxCounts) String() string {

263 return fmt.Sprintf("%d total, %d deleted, %d unread, %d unseen, size %d bytes", mc.Total, mc.Deleted, mc.Unread, mc.Unseen, mc.Size)

264}

265

266// Add increases mailbox counts mc with those of delta.

267func (mc *MailboxCounts) Add(delta MailboxCounts) {

268 mc.Total += delta.Total

269 mc.Deleted += delta.Deleted

270 mc.Unread += delta.Unread

271 mc.Unseen += delta.Unseen

272 mc.Size += delta.Size

273}

274

275// Add decreases mailbox counts mc with those of delta.

276func (mc *MailboxCounts) Sub(delta MailboxCounts) {

277 mc.Total -= delta.Total

278 mc.Deleted -= delta.Deleted

279 mc.Unread -= delta.Unread

280 mc.Unseen -= delta.Unseen

281 mc.Size -= delta.Size

282}

283

284// SpecialUse identifies a specific role for a mailbox, used by clients to

285// understand where messages should go.

286type SpecialUse struct {

287 Archive bool

288 Draft bool // "Drafts"

289 Junk bool

290 Sent bool

291 Trash bool

292}

293

294// CalculateCounts calculates the full current counts for messages in the mailbox.

295func (mb *Mailbox) CalculateCounts(tx *bstore.Tx) (mc MailboxCounts, err error) {

296 q := bstore.QueryTx[Message](tx)

297 q.FilterNonzero(Message{MailboxID: mb.ID})

298 q.FilterEqual("Expunged", false)

299 err = q.ForEach(func(m Message) error {

300 mc.Add(m.MailboxCounts())

301 return nil

302 })

303 return

304}

305

306// ChangeSpecialUse returns a change for special-use flags, for broadcasting to

307// other connections.

308func (mb Mailbox) ChangeSpecialUse() ChangeMailboxSpecialUse {

309 return ChangeMailboxSpecialUse{mb.ID, mb.Name, mb.SpecialUse, mb.ModSeq}

310}

311

312// ChangeKeywords returns a change with new keywords for a mailbox (e.g. after

313// setting a new keyword on a message in the mailbox), for broadcasting to other

314// connections.

315func (mb Mailbox) ChangeKeywords() ChangeMailboxKeywords {

316 return ChangeMailboxKeywords{mb.ID, mb.Name, mb.Keywords}

317}

318

319func (mb Mailbox) ChangeAddMailbox(flags []string) ChangeAddMailbox {

320 return ChangeAddMailbox{Mailbox: mb, Flags: flags}

321}

322

323func (mb Mailbox) ChangeRemoveMailbox() ChangeRemoveMailbox {

324 return ChangeRemoveMailbox{mb.ID, mb.Name, mb.ModSeq}

325}

326

327// KeywordsChanged returns whether the keywords in a mailbox have changed.

328func (mb Mailbox) KeywordsChanged(origmb Mailbox) bool {

329 if len(mb.Keywords) != len(origmb.Keywords) {

330 return true

331 }

332 // Keywords are stored sorted.

333 for i, kw := range mb.Keywords {

334 if origmb.Keywords[i] != kw {

335 return true

336 }

337 }

338 return false

339}

340

341// CountsChange returns a change with mailbox counts.

342func (mb Mailbox) ChangeCounts() ChangeMailboxCounts {

343 return ChangeMailboxCounts{mb.ID, mb.Name, mb.MailboxCounts}

344}

345

346// Subscriptions are separate from existence of mailboxes.

347type Subscription struct {

348 Name string

349}

350

351// Flags for a mail message.

352type Flags struct {

353 Seen bool

354 Answered bool

355 Flagged bool

356 Forwarded bool

357 Junk bool

358 Notjunk bool

359 Deleted bool

360 Draft bool

361 Phishing bool

362 MDNSent bool

363}

364

365// FlagsAll is all flags set, for use as mask.

366var FlagsAll = Flags{true, true, true, true, true, true, true, true, true, true}

367

368// Validation of "message From" domain.

369type Validation uint8

370

371const (

372 ValidationUnknown Validation = 0

373 ValidationStrict Validation = 1 // Like DMARC, with strict policies.

374 ValidationDMARC Validation = 2 // Actual DMARC policy.

375 ValidationRelaxed Validation = 3 // Like DMARC, with relaxed policies.

376 ValidationPass Validation = 4 // For SPF.

377 ValidationNeutral Validation = 5 // For SPF.

378 ValidationTemperror Validation = 6

379 ValidationPermerror Validation = 7

380 ValidationFail Validation = 8

381 ValidationSoftfail Validation = 9 // For SPF.

382 ValidationNone Validation = 10 // E.g. No records.

383)

384

385// Message stored in database and per-message file on disk.

386//

387// Contents are always the combined data from MsgPrefix and the on-disk file named

388// based on ID.

389//

390// Messages always have a header section, even if empty. Incoming messages without

391// header section must get an empty header section added before inserting.

392type Message struct {

393 // ID of the message, determines path to on-disk message file. Set when adding to a

394 // mailbox. When a message is moved to another mailbox, the mailbox ID is changed,

395 // but for synchronization purposes, a new Message record is inserted (which gets a

396 // new ID) with the Expunged field set and the MailboxID and UID copied.

397 ID int64

398

399 // UID, for IMAP. Set when adding to mailbox. Strictly increasing values, per

400 // mailbox. The UID of a message can never change (though messages can be copied),

401 // and the contents of a message/UID also never changes.

402 UID UID `bstore:"nonzero"`

403

404 MailboxID int64 `bstore:"nonzero,unique MailboxID+UID,index MailboxID+Received,index MailboxID+ModSeq,ref Mailbox"`

405

406 // Modification sequence, for faster syncing with IMAP QRESYNC and JMAP.

407 // ModSeq is the last modification. CreateSeq is the Seq the message was inserted,

408 // always <= ModSeq. If Expunged is set, the message has been removed and should not

409 // be returned to the user. In this case, ModSeq is the Seq where the message is

410 // removed, and will never be changed again.

411 // We have an index on both ModSeq (for JMAP that synchronizes per account) and

412 // MailboxID+ModSeq (for IMAP that synchronizes per mailbox).

413 // The index on CreateSeq helps efficiently finding created messages for JMAP.

414 // The value of ModSeq is special for IMAP. Messages that existed before ModSeq was

415 // added have 0 as value. But modseq 0 in IMAP is special, so we return it as 1. If

416 // we get modseq 1 from a client, the IMAP server will translate it to 0. When we

417 // return modseq to clients, we turn 0 into 1.

418 ModSeq ModSeq `bstore:"index"`

419 CreateSeq ModSeq `bstore:"index"`

420 Expunged bool

421

422 // If set, this message was delivered to a Rejects mailbox. When it is moved to a

423 // different mailbox, its MailboxOrigID is set to the destination mailbox and this

424 // flag cleared.

425 IsReject bool

426

427 // If set, this is a forwarded message (through a ruleset with IsForward). This

428 // causes fields used during junk analysis to be moved to their Orig variants, and

429 // masked IP fields cleared, so they aren't used in junk classifications for

430 // incoming messages. This ensures the forwarded messages don't cause negative

431 // reputation for the forwarding mail server, which may also be sending regular

432 // messages.

433 IsForward bool

434

435 // MailboxOrigID is the mailbox the message was originally delivered to. Typically

436 // Inbox or Rejects, but can also be a mailbox configured in a Ruleset, or

437 // Postmaster, TLS/DMARC reporting addresses. MailboxOrigID is not changed when the

438 // message is moved to another mailbox, e.g. Archive/Trash/Junk. Used for

439 // per-mailbox reputation.

440 //

441 // MailboxDestinedID is normally 0, but when a message is delivered to the Rejects

442 // mailbox, it is set to the intended mailbox according to delivery rules,

443 // typically that of Inbox. When such a message is moved out of Rejects, the

444 // MailboxOrigID is corrected by setting it to MailboxDestinedID. This ensures the

445 // message is used for reputation calculation for future deliveries to that

446 // mailbox.

447 //

448 // These are not bstore references to prevent having to update all messages in a

449 // mailbox when the original mailbox is removed. Use of these fields requires

450 // checking if the mailbox still exists.

451 MailboxOrigID int64

452 MailboxDestinedID int64

453

454 // Received indicates time of receival over SMTP, or of IMAP APPEND.

455 Received time.Time `bstore:"default now,index"`

456

457 // SaveDate is the time of copy/move/save to a mailbox, used with IMAP SAVEDATE

458 // extension. Must be updated each time a message is copied/moved to another

459 // mailbox. Can be nil for messages from before this functionality was introduced.

460 SaveDate *time.Time `bstore:"default now"`

461

462 // Full IP address of remote SMTP server. Empty if not delivered over SMTP. The

463 // masked IPs are used to classify incoming messages. They are left empty for

464 // messages matching a ruleset for forwarded messages.

465 RemoteIP string

466 RemoteIPMasked1 string `bstore:"index RemoteIPMasked1+Received"` // For IPv4 /32, for IPv6 /64, for reputation.

467 RemoteIPMasked2 string `bstore:"index RemoteIPMasked2+Received"` // For IPv4 /26, for IPv6 /48.

468 RemoteIPMasked3 string `bstore:"index RemoteIPMasked3+Received"` // For IPv4 /21, for IPv6 /32.

469

470 // Only set if present and not an IP address. Unicode string. Empty for forwarded

471 // messages.

472 EHLODomain string `bstore:"index EHLODomain+Received"`

473 MailFrom string // With localpart and domain. Can be empty.

474 MailFromLocalpart smtp.Localpart // SMTP "MAIL FROM", can be empty.

475 // Only set if it is a domain, not an IP. Unicode string. Empty for forwarded

476 // messages, but see OrigMailFromDomain.

477 MailFromDomain string `bstore:"index MailFromDomain+Received"`

478 RcptToLocalpart smtp.Localpart // SMTP "RCPT TO", can be empty.

479 RcptToDomain string // Unicode string.

480

481 // Parsed "From" message header, used for reputation along with domain validation.

482 MsgFromLocalpart smtp.Localpart

483 MsgFromDomain string `bstore:"index MsgFromDomain+Received"` // Unicode string.

484 MsgFromOrgDomain string `bstore:"index MsgFromOrgDomain+Received"` // Unicode string.

485

486 // Simplified statements of the Validation fields below, used for incoming messages

487 // to check reputation.

488 EHLOValidated bool

489 MailFromValidated bool

490 MsgFromValidated bool

491

492 EHLOValidation Validation // Validation can also take reverse IP lookup into account, not only SPF.

493 MailFromValidation Validation // Can have SPF-specific validations like ValidationSoftfail.

494 MsgFromValidation Validation // Desirable validations: Strict, DMARC, Relaxed. Will not be just Pass.

495

496 // Domains with verified DKIM signatures. Unicode string. For forwarded messages, a

497 // DKIM domain that matched a ruleset's verified domain is left out, but included

498 // in OrigDKIMDomains.

499 DKIMDomains []string `bstore:"index DKIMDomains+Received"`

500

501 // For forwarded messages,

502 OrigEHLODomain string

503 OrigDKIMDomains []string

504

505 // Canonicalized Message-Id, always lower-case and normalized quoting, without

506 // <>'s. Empty if missing. Used for matching message threads, and to prevent

507 // duplicate reject delivery.

508 MessageID string `bstore:"index"`

509 // lower-case: ../rfc/5256:495

510

511 // For matching threads in case there is no References/In-Reply-To header. It is

512 // lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed.

513 SubjectBase string `bstore:"index"`

514 // ../rfc/5256:90

515

516 // Hash of message. For rejects delivery in case there is no Message-ID, only set

517 // when delivered as reject.

518 MessageHash []byte

519

520 // ID of message starting this thread.

521 ThreadID int64 `bstore:"index"`

522 // IDs of parent messages, from closest parent to the root message. Parent messages

523 // may be in a different mailbox, or may no longer exist. ThreadParentIDs must

524 // never contain the message id itself (a cycle), and parent messages must

525 // reference the same ancestors. Moving a message to another mailbox keeps the

526 // message ID and changes the MailboxID (and UID) of the message, leaving threading

527 // parent ids intact.

528 ThreadParentIDs []int64

529 // ThreadMissingLink is true if there is no match with a direct parent. E.g. first

530 // ID in ThreadParentIDs is not the direct ancestor (an intermediate message may

531 // have been deleted), or subject-based matching was done.

532 ThreadMissingLink bool

533 // If set, newly delivered child messages are automatically marked as read. This

534 // field is copied to new child messages. Changes are propagated to the webmail

535 // client.

536 ThreadMuted bool

537 // If set, this (sub)thread is collapsed in the webmail client, for threading mode

538 // "on" (mode "unread" ignores it). This field is copied to new child message.

539 // Changes are propagated to the webmail client.

540 ThreadCollapsed bool

541

542 // If received message was known to match a mailing list rule (with modified junk

543 // filtering).

544 IsMailingList bool

545

546 // If this message is a DSN, generated by us or received. For DSNs, we don't look

547 // at the subject when matching threads.

548 DSN bool

549

550 ReceivedTLSVersion uint16 // 0 if unknown, 1 if plaintext/no TLS, otherwise TLS cipher suite.

551 ReceivedTLSCipherSuite uint16

552 ReceivedRequireTLS bool // Whether RequireTLS was known to be used for incoming delivery.

553

554 Flags

555 // For keywords other than system flags or the basic well-known $-flags. Only in

556 // "atom" syntax (IMAP), they are case-insensitive, always stored in lower-case

557 // (for JMAP), sorted.

558 Keywords []string `bstore:"index"`

559 Size int64

560 TrainedJunk *bool // If nil, no training done yet. Otherwise, true is trained as junk, false trained as nonjunk.

561 MsgPrefix []byte // Typically holds received headers and/or header separator.

562

563 // If non-nil, a preview of the message based on text and/or html parts of the

564 // message. Used in the webmail and IMAP PREVIEW extension. If non-nil, it is empty

565 // if no preview could be created, or the message has not textual content or

566 // couldn't be parsed.

567 // Previews are typically created when delivering a message, but not when importing

568 // messages, for speed. Previews are generated on first request (in the webmail, or

569 // through the IMAP fetch attribute "PREVIEW" (without "LAZY")), and stored with

570 // the message at that time.

571 // The preview is at most 256 characters (can be more bytes), with detected quoted

572 // text replaced with "[...]". Previews typically end with a newline, callers may

573 // want to strip whitespace.

574 Preview *string

575

576 // ParsedBuf message structure. Currently saved as JSON of message.Part because bstore

577 // cannot yet store recursive types. Created when first needed, and saved in the

578 // database.

579 // todo: once replaced with non-json storage, remove date fixup in ../message/part.go.

580 ParsedBuf []byte

581}

582

583// MailboxCounts returns the delta to counts this message means for its

584// mailbox.

585func (m Message) MailboxCounts() (mc MailboxCounts) {

586 if m.Expunged {

587 return

588 }

589 if m.Deleted {

590 mc.Deleted++

591 } else {

592 mc.Total++

593 }

594 if !m.Seen {

595 mc.Unseen++

596 if !m.Deleted {

597 mc.Unread++

598 }

599 }

600 mc.Size += m.Size

601 return

602}

603

604func (m Message) ChangeAddUID() ChangeAddUID {

605 return ChangeAddUID{m.MailboxID, m.UID, m.ModSeq, m.Flags, m.Keywords}

606}

607

608func (m Message) ChangeFlags(orig Flags) ChangeFlags {

609 mask := m.Flags.Changed(orig)

610 return ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, ModSeq: m.ModSeq, Mask: mask, Flags: m.Flags, Keywords: m.Keywords}

611}

612

613func (m Message) ChangeThread() ChangeThread {

614 return ChangeThread{[]int64{m.ID}, m.ThreadMuted, m.ThreadCollapsed}

615}

616

617// ModSeq represents a modseq as stored in the database. ModSeq 0 in the

618// database is sent to the client as 1, because modseq 0 is special in IMAP.

619// ModSeq coming from the client are of type int64.

620type ModSeq int64

621

622func (ms ModSeq) Client() int64 {

623 if ms == 0 {

624 return 1

625 }

626 return int64(ms)

627}

628

629// ModSeqFromClient converts a modseq from a client to a modseq for internal

630// use, e.g. in a database query.

631// ModSeq 1 is turned into 0 (the Go zero value for ModSeq).

632func ModSeqFromClient(modseq int64) ModSeq {

633 if modseq == 1 {

634 return 0

635 }

636 return ModSeq(modseq)

637}

638

639// Erase clears fields from a Message that are no longer needed after actually

640// removing the message file from the file system, after all references to the

641// message have gone away. Only the fields necessary for synchronisation are kept.

642func (m *Message) erase() {

643 if !m.Expunged {

644 panic("erase called on non-expunged message")

645 }

646 *m = Message{

647 ID: m.ID,

648 UID: m.UID,

649 MailboxID: m.MailboxID,

650 CreateSeq: m.CreateSeq,

651 ModSeq: m.ModSeq,

652 Expunged: true,

653 ThreadID: m.ThreadID,

654 }

655}

656

657// PrepareThreading sets MessageID, SubjectBase and DSN (used in threading) based

658// on the part.

659func (m *Message) PrepareThreading(log mlog.Log, part *message.Part) {

660 m.DSN = part.IsDSN()

661

662 if part.Envelope == nil {

663 return

664 }

665 messageID, raw, err := message.MessageIDCanonical(part.Envelope.MessageID)

666 if err != nil {

667 log.Debugx("parsing message-id, ignoring", err, slog.String("messageid", part.Envelope.MessageID))

668 } else if raw {

669 log.Debug("could not parse message-id as address, continuing with raw value", slog.String("messageid", part.Envelope.MessageID))

670 }

671 m.MessageID = messageID

672 m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false)

673}

674

675// LoadPart returns a message.Part by reading from m.ParsedBuf.

676func (m Message) LoadPart(r io.ReaderAt) (message.Part, error) {

677 if m.ParsedBuf == nil {

678 return message.Part{}, fmt.Errorf("message not parsed")

679 }

680 var p message.Part

681 err := json.Unmarshal(m.ParsedBuf, &p)

682 if err != nil {

683 return p, fmt.Errorf("unmarshal message part")

684 }

685 p.SetReaderAt(r)

686 return p, nil

687}

688

689// NeedsTraining returns whether message needs a training update, based on

690// TrainedJunk (current training status) and new Junk/Notjunk flags.

691func (m Message) NeedsTraining() bool {

692 needs, _, _, _, _ := m.needsTraining()

693 return needs

694}

695

696func (m Message) needsTraining() (needs, untrain, untrainJunk, train, trainJunk bool) {

697 untrain = m.TrainedJunk != nil

698 untrainJunk = untrain && *m.TrainedJunk

699 train = m.Junk != m.Notjunk

700 trainJunk = m.Junk

701 needs = untrain != train || untrain && train && untrainJunk != trainJunk

702 return

703}

704

705// JunkFlagsForMailbox sets Junk and Notjunk flags based on mailbox name if configured. Often

706// used when delivering/moving/copying messages to a mailbox. Mail clients are not

707// very helpful with setting junk/notjunk flags. But clients can move/copy messages

708// to other mailboxes. So we set flags when clients move a message.

709func (m *Message) JunkFlagsForMailbox(mb Mailbox, conf config.Account) {

710 if mb.Junk {

711 m.Junk = true

712 m.Notjunk = false

713 return

714 }

715

716 if !conf.AutomaticJunkFlags.Enabled {

717 return

718 }

719

720 lmailbox := strings.ToLower(mb.Name)

721

722 if conf.JunkMailbox != nil && conf.JunkMailbox.MatchString(lmailbox) {

723 m.Junk = true

724 m.Notjunk = false

725 } else if conf.NeutralMailbox != nil && conf.NeutralMailbox.MatchString(lmailbox) {

726 m.Junk = false

727 m.Notjunk = false

728 } else if conf.NotJunkMailbox != nil && conf.NotJunkMailbox.MatchString(lmailbox) {

729 m.Junk = false

730 m.Notjunk = true

731 } else if conf.JunkMailbox == nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox != nil {

732 m.Junk = true

733 m.Notjunk = false

734 } else if conf.JunkMailbox != nil && conf.NeutralMailbox == nil && conf.NotJunkMailbox != nil {

735 m.Junk = false

736 m.Notjunk = false

737 } else if conf.JunkMailbox != nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox == nil {

738 m.Junk = false

739 m.Notjunk = true

740 }

741}

742

743// Recipient represents the recipient of a message. It is tracked to allow

744// first-time incoming replies from users this account has sent messages to. When a

745// mailbox is added to the Sent mailbox the message is parsed and recipients are

746// inserted as recipient. Recipients are never removed other than for removing the

747// message. On move/copy of a message, recipients aren't modified either. For IMAP,

748// this assumes a client simply appends messages to the Sent mailbox (as opposed to

749// copying messages from some place).

750type Recipient struct {

751 ID int64

752 MessageID int64 `bstore:"nonzero,ref Message"` // Ref gives it its own index, useful for fast removal as well.

753 Localpart string `bstore:"nonzero"` // Encoded localpart.

754 Domain string `bstore:"nonzero,index Domain+Localpart"` // Unicode string.

755 OrgDomain string `bstore:"nonzero,index"` // Unicode string.

756 Sent time.Time `bstore:"nonzero"`

757}

758

759// Outgoing is a message submitted for delivery from the queue. Used to enforce

760// maximum outgoing messages.

761type Outgoing struct {

762 ID int64

763 Recipient string `bstore:"nonzero,index"` // Canonical international address with utf8 domain.

764 Submitted time.Time `bstore:"nonzero,default now"`

765}

766

767// RecipientDomainTLS stores TLS capabilities of a recipient domain as encountered

768// during most recent connection (delivery attempt).

769type RecipientDomainTLS struct {

770 Domain string // Unicode.

771 Updated time.Time `bstore:"default now"`

772 STARTTLS bool // Supports STARTTLS.

773 RequireTLS bool // Supports RequireTLS SMTP extension.

774}

775

776// DiskUsage tracks quota use.

777type DiskUsage struct {

778 ID int64 // Always one record with ID 1.

779 MessageSize int64 // Sum of all messages, for quota accounting.

780}

781

782// SessionToken and CSRFToken are types to prevent mixing them up.

783// Base64 raw url encoded.

784type SessionToken string

785type CSRFToken string

786

787// LoginSession represents a login session. We keep a limited number of sessions

788// for a user, removing the oldest session when a new one is created.

789type LoginSession struct {

790 ID int64

791 Created time.Time `bstore:"nonzero,default now"` // Of original login.

792 Expires time.Time `bstore:"nonzero"` // Extended each time it is used.

793 SessionTokenBinary [16]byte `bstore:"nonzero"` // Stored in cookie, like "webmailsession" or "webaccountsession".

794 CSRFTokenBinary [16]byte // For API requests, in "x-mox-csrf" header.

795 AccountName string `bstore:"nonzero"`

796 LoginAddress string `bstore:"nonzero"`

797

798 // Set when loading from database.

799 sessionToken SessionToken

800 csrfToken CSRFToken

801}

802

803// Quoting is a setting for how to quote in replies/forwards.

804type Quoting string

805

806const (

807 Default Quoting = "" // Bottom-quote if text is selected, top-quote otherwise.

808 Bottom Quoting = "bottom"

809 Top Quoting = "top"

810)

811

812// Settings are webmail client settings.

813type Settings struct {

814 ID uint8 // Singleton ID 1.

815

816 Signature string

817 Quoting Quoting

818

819 // Whether to show the bars underneath the address input fields indicating

820 // starttls/dnssec/dane/mtasts/requiretls support by address.

821 ShowAddressSecurity bool

822

823 // Show HTML version of message by default, instead of plain text.

824 ShowHTML bool

825

826 // If true, don't show shortcuts in webmail after mouse interaction.

827 NoShowShortcuts bool

828

829 // Additional headers to display in message view. E.g. Delivered-To, User-Agent, X-Mox-Reason.

830 ShowHeaders []string

831}

832

833// ViewMode how a message should be viewed: its text parts, html parts, or html

834// with loading external resources.

835type ViewMode string

836

837const (

838 ModeText ViewMode = "text"

839 ModeHTML ViewMode = "html"

840 ModeHTMLExt ViewMode = "htmlext" // HTML with external resources.

841)

842

843// FromAddressSettings are webmail client settings per "From" address.

844type FromAddressSettings struct {

845 FromAddress string // Unicode.

846 ViewMode ViewMode

847}

848

849// RulesetNoListID records a user "no" response to the question of

850// creating/removing a ruleset after moving a message with list-id header from/to

851// the inbox.

852type RulesetNoListID struct {

853 ID int64

854 RcptToAddress string `bstore:"nonzero"`

855 ListID string `bstore:"nonzero"`

856 ToInbox bool // Otherwise from Inbox to other mailbox.

857}

858

859// RulesetNoMsgFrom records a user "no" response to the question of

860// creating/moveing a ruleset after moving a mesage with message "from" address

861// from/to the inbox.

862type RulesetNoMsgFrom struct {

863 ID int64

864 RcptToAddress string `bstore:"nonzero"`

865 MsgFromAddress string `bstore:"nonzero"` // Unicode.

866 ToInbox bool // Otherwise from Inbox to other mailbox.

867}

868

869// RulesetNoMailbox represents a "never from/to this mailbox" response to the

870// question of adding/removing a ruleset after moving a message.

871type RulesetNoMailbox struct {

872 ID int64

873

874 // The mailbox from/to which the move has happened.

875 // Not a references, if mailbox is deleted, an entry becomes ineffective.

876 MailboxID int64 `bstore:"nonzero"`

877 ToMailbox bool // Whether MailboxID is the destination of the move (instead of source).

878}

879

880// MessageErase represents the need to remove a message file from disk, and clear

881// message fields from the database, but only when the last reference to the

882// message is gone (all IMAP sessions need to have applied the changes indicating

883// message removal).

884type MessageErase struct {

885 ID int64 // Same ID as Message.ID.

886

887 // Whether to subtract the size from the total disk usage. Useful for moving

888 // messages, which involves duplicating the message temporarily, while there are

889 // still references in the old mailbox, but which isn't counted as using twice the

890 // disk space..

891 SkipUpdateDiskUsage bool

892}

893

894// Types stored in DB.

895var DBTypes = []any{

896 NextUIDValidity{},

897 Message{},

898 Recipient{},

899 Mailbox{},

900 Subscription{},

901 Outgoing{},

902 Password{},

903 Subjectpass{},

904 SyncState{},

905 Upgrade{},

906 RecipientDomainTLS{},

907 DiskUsage{},

908 LoginSession{},

909 Settings{},

910 FromAddressSettings{},

911 RulesetNoListID{},

912 RulesetNoMsgFrom{},

913 RulesetNoMailbox{},

914 Annotation{},

915 MessageErase{},

916}

917

918// Account holds the information about a user, includings mailboxes, messages, imap subscriptions.

919type Account struct {

920 Name string // Name, according to configuration.

921 Dir string // Directory where account files, including the database, bloom filter, and mail messages, are stored for this account.

922 DBPath string // Path to database with mailboxes, messages, etc.

923 DB *bstore.DB // Open database connection.

924

925 // Channel that is closed if/when account has/gets "threads" accounting (see

926 // Upgrade.Threads).

927 threadsCompleted chan struct{}

928 // If threads upgrade completed with error, this is set. Used for warning during

929 // delivery, or aborting when importing.

930 threadsErr error

931

932 // Message directory of last delivery. Used to check we don't have to make that

933 // directory when delivering.

934 lastMsgDir string

935

936 // If set, consistency checks won't fail on message ModSeq/CreateSeq being zero.

937 skipMessageZeroSeqCheck bool

938

939 // Write lock must be held when modifying account/mailbox/message/flags/annotations

940 // if the change needs to be synchronized with client connections by broadcasting

941 // the changes. Changes that are not protocol-visible do not require a lock, the

942 // database transactions isolate activity, though locking may be necessary to

943 // protect in-memory-only access.

944 //

945 // Read lock for reading mailboxes/messages as a consistent snapsnot (i.e. not

946 // concurrent changes). For longer transactions, e.g. when reading many messages,

947 // the lock can be released while continuing to read from the transaction.

948 //

949 // When making changes to mailboxes/messages, changes must be broadcasted before

950 // releasing the lock to ensure proper UID ordering.

951 sync.RWMutex

952

953 // Reference count, while >0, this account is alive and shared. Protected by

954 // openAccounts, not by account wlock.

955 nused int

956 removed bool // Marked for removal. Last close removes the account directory.

957 closed chan struct{} // Closed when last reference is gone.

958}

959

960type Upgrade struct {

961 ID byte

962 Threads byte // 0: None, 1: Adding MessageID's completed, 2: Adding ThreadID's completed.

963 MailboxModSeq bool // Whether mailboxes have been assigned modseqs.

964 MailboxParentID bool // Setting ParentID on mailboxes.

965 MailboxCounts bool // Global flag about whether we have mailbox flags. Instead of previous per-mailbox boolean.

966}

967

968// upgradeInit is the value to for new account database, that don't need any upgrading.

969var upgradeInit = Upgrade{

970 ID: 1, // Singleton.

971 Threads: 2,

972 MailboxModSeq: true,

973 MailboxParentID: true,

974 MailboxCounts: true,

975}

976

977// InitialUIDValidity returns a UIDValidity used for initializing an account.

978// It can be replaced during tests with a predictable value.

979var InitialUIDValidity = func() uint32 {

980 return uint32(time.Now().Unix() >> 1) // A 2-second resolution will get us far enough beyond 2038.

981}

982

983var openAccounts = struct {

984 sync.Mutex

985 names map[string]*Account

986}{

987 names: map[string]*Account{},

988}

989

990func closeAccount(acc *Account) (rerr error) {

991 // If we need to remove the account files, we do so without the accounts lock.

992 remove := false

993 defer func() {

994 if remove {

995 log := mlog.New("store", nil)

996 err := removeAccount(log, acc.Name)

997 if rerr == nil {

998 rerr = err

999 }

1000 close(acc.closed)

1001 }

1002 }()

1003

1004 openAccounts.Lock()

1005 defer openAccounts.Unlock()

1006 acc.nused--

1007 if acc.nused > 0 {

1008 return

1009 }

1010 remove = acc.removed

1011

1012 defer func() {

1013 err := acc.DB.Close()

1014 acc.DB = nil

1015 delete(openAccounts.names, acc.Name)

1016 if !remove {

1017 close(acc.closed)

1018 }

1019

1020 if rerr == nil {

1021 rerr = err

1022 }

1023 }()

1024

1025 // Verify there are no more pending MessageErase records.

1026 l, err := bstore.QueryDB[MessageErase](context.TODO(), acc.DB).List()

1027 if err != nil {

1028 return fmt.Errorf("listing messageerase records: %v", err)

1029 } else if len(l) > 0 {

1030 return fmt.Errorf("messageerase records still present after last account reference is gone: %v", l)

1031 }

1032

1033 return nil

1034}

1035

1036// removeAccount moves the account directory for an account away and removes

1037// all files, and removes the AccountRemove struct from the database.

1038func removeAccount(log mlog.Log, accountName string) error {

1039 log = log.With(slog.String("account", accountName))

1040 log.Info("removing account directory and files")

1041

1042 // First move the account directory away.

1043 odir := filepath.Join(mox.DataDirPath("accounts"), accountName)

1044 tmpdir := filepath.Join(mox.DataDirPath("tmp"), "oldaccount-"+accountName)

1045 if err := os.Rename(odir, tmpdir); err != nil {

1046 return fmt.Errorf("moving account data directory %q out of the way to %q (account not removed): %v", odir, tmpdir, err)

1047 }

1048

1049 var errs []error

1050

1051 // Commit removal to database.

1052 err := AuthDB.Write(context.Background(), func(tx *bstore.Tx) error {

1053 if err := tx.Delete(&AccountRemove{accountName}); err != nil {

1054 return fmt.Errorf("deleting account removal request: %v", err)

1055 }

1056 if err := tlsPublicKeyRemoveForAccount(tx, accountName); err != nil {

1057 return fmt.Errorf("removing tls public keys for account: %v", err)

1058 }

1059

1060 if err := loginAttemptRemoveAccount(tx, accountName); err != nil {

1061 return fmt.Errorf("removing historic login attempts for account: %v", err)

1062 }

1063 return nil

1064 })

1065 if err != nil {

1066 errs = append(errs, fmt.Errorf("remove account from database: %w", err))

1067 }

1068

1069 // Remove the account directory and its message and other files.

1070 if err := os.RemoveAll(tmpdir); err != nil {

1071 errs = append(errs, fmt.Errorf("removing account data directory %q that was moved to %q: %v", odir, tmpdir, err))

1072 }

1073

1074 return errors.Join(errs...)

1075}

1076

1077// OpenAccount opens an account by name.

1078//

1079// No additional data path prefix or ".db" suffix should be added to the name.

1080// A single shared account exists per name.

1081func OpenAccount(log mlog.Log, name string, checkLoginDisabled bool) (*Account, error) {

1082 openAccounts.Lock()

1083 defer openAccounts.Unlock()

1084 if acc, ok := openAccounts.names[name]; ok {

1085 if acc.removed {

1086 return nil, fmt.Errorf("account has been removed")

1087 }

1088

1089 acc.nused++

1090 return acc, nil

1091 }

1092

1093 if a, ok := mox.Conf.Account(name); !ok {

1094 return nil, ErrAccountUnknown

1095 } else if checkLoginDisabled && a.LoginDisabled != "" {

1096 return nil, fmt.Errorf("%w: %s", ErrLoginDisabled, a.LoginDisabled)

1097 }

1098

1099 acc, err := openAccount(log, name)

1100 if err != nil {

1101 return nil, err

1102 }

1103 openAccounts.names[name] = acc

1104 return acc, nil

1105}

1106

1107// openAccount opens an existing account, or creates it if it is missing.

1108// Called with openAccounts lock held.

1109func openAccount(log mlog.Log, name string) (a *Account, rerr error) {

1110 dir := filepath.Join(mox.DataDirPath("accounts"), name)

1111 return OpenAccountDB(log, dir, name)

1112}

1113

1114// OpenAccountDB opens an account database file and returns an initialized account

1115// or error. Only exported for use by subcommands that verify the database file.

1116// Almost all account opens must go through OpenAccount/OpenEmail/OpenEmailAuth.

1117func OpenAccountDB(log mlog.Log, accountDir, accountName string) (a *Account, rerr error) {

1118 dbpath := filepath.Join(accountDir, "index.db")

1119

1120 // Create account if it doesn't exist yet.

1121 isNew := false

1122 if _, err := os.Stat(dbpath); err != nil && os.IsNotExist(err) {

1123 isNew = true

1124 os.MkdirAll(accountDir, 0770)

1125 }

1126

1127 opts := bstore.Options{Timeout: 5 * time.Second, Perm: 0660, RegisterLogger: moxvar.RegisterLogger(dbpath, log.Logger)}

1128 db, err := bstore.Open(context.TODO(), dbpath, &opts, DBTypes...)

1129 if err != nil {

1130 return nil, err

1131 }

1132

1133 defer func() {

1134 if rerr != nil {

1135 err := db.Close()

1136 log.Check(err, "closing database file after error")

1137 if isNew {

1138 err := os.Remove(dbpath)

1139 log.Check(err, "removing new database file after error")

1140 }

1141 }

1142 }()

1143

1144 acc := &Account{

1145 Name: accountName,

1146 Dir: accountDir,

1147 DBPath: dbpath,

1148 DB: db,

1149 nused: 1,

1150 closed: make(chan struct{}),

1151 threadsCompleted: make(chan struct{}),

1152 }

1153

1154 if isNew {

1155 if err := initAccount(db); err != nil {

1156 return nil, fmt.Errorf("initializing account: %v", err)

1157 }

1158

1159 close(acc.threadsCompleted)

1160 return acc, nil

1161 }

1162

1163 // Ensure singletons are present, like DiskUsage and Settings.

1164 // Process pending MessageErase records. Check that next the message ID assigned by

1165 // the database does not already have a file on disk, or increase the sequence so

1166 // it doesn't.

1167 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

1168 if tx.Get(&Settings{ID: 1}) == bstore.ErrAbsent {

1169 if err := tx.Insert(&Settings{ID: 1, ShowAddressSecurity: true}); err != nil {

1170 return err

1171 }

1172 }

1173

1174 du := DiskUsage{ID: 1}

1175 err = tx.Get(&du)

1176 if err == bstore.ErrAbsent {

1177 // No DiskUsage record yet, calculate total size and insert.

1178 err := bstore.QueryTx[Mailbox](tx).FilterEqual("Expunged", false).ForEach(func(mb Mailbox) error {

1179 du.MessageSize += mb.Size

1180 return nil

1181 })

1182 if err != nil {

1183 return err

1184 }

1185 if err := tx.Insert(&du); err != nil {

1186 return err

1187 }

1188 } else if err != nil {

1189 return err

1190 }

1191

1192 var erase []MessageErase

1193 if _, err := bstore.QueryTx[MessageErase](tx).Gather(&erase).Delete(); err != nil {

1194 return fmt.Errorf("fetching messages to erase: %w", err)

1195 }

1196 if len(erase) > 0 {

1197 log.Debug("deleting message files from message erase records", slog.Int("count", len(erase)))

1198 }

1199 var duChanged bool

1200 for _, me := range erase {

1201 // Clear the fields from the message not needed for synchronization.

1202 m := Message{ID: me.ID}

1203 if err := tx.Get(&m); err != nil {

1204 return fmt.Errorf("get message %d to expunge: %w", me.ID, err)

1205 } else if !m.Expunged {

1206 return fmt.Errorf("message %d to erase is not expunged", m.ID)

1207 }

1208

1209 // We remove before we update/commit the database, so we are sure we don't leave

1210 // files behind in case of an error/crash.

1211 p := acc.MessagePath(me.ID)

1212 err := os.Remove(p)

1213 log.Check(err, "removing message file for expunged message", slog.String("path", p))

1214

1215 if !me.SkipUpdateDiskUsage {

1216 du.MessageSize -= m.Size

1217 duChanged = true

1218 }

1219

1220 m.erase()

1221 if err := tx.Update(&m); err != nil {

1222 return fmt.Errorf("save erase of message %d in database: %w", m.ID, err)

1223 }

1224 }

1225

1226 if duChanged {

1227 if err := tx.Update(&du); err != nil {

1228 return fmt.Errorf("saving disk usage after erasing messages: %w", err)

1229 }

1230 }

1231

1232 // Ensure the message directories don't have a higher message ID than occurs in our

1233 // database. If so, increase the next ID used for inserting a message to prevent

1234 // clash during delivery.

1235 last, err := bstore.QueryTx[Message](tx).SortDesc("ID").Limit(1).Get()

1236 if err != nil && err != bstore.ErrAbsent {

1237 return fmt.Errorf("querying last message: %v", err)

1238 }

1239

1240 // We look in the directory where the message is stored (the id can be 0, which is fine).

1241 maxDBID := last.ID

1242 p := acc.MessagePath(maxDBID)

1243 dir := filepath.Dir(p)

1244 maxFSID := maxDBID

1245 // We also try looking for the next directories that would be created for messages,

1246 // until one doesn't exist anymore. We never delete these directories.

1247 for {

1248 np := acc.MessagePath(maxFSID + msgFilesPerDir)

1249 ndir := filepath.Dir(np)

1250 if _, err := os.Stat(ndir); err == nil {

1251 maxFSID = (maxFSID + msgFilesPerDir) &^ (msgFilesPerDir - 1) // First ID for dir.

1252 dir = ndir

1253 } else if errors.Is(err, fs.ErrNotExist) {

1254 break

1255 } else {

1256 return fmt.Errorf("stat next message directory %q: %v", ndir, err)

1257 }

1258 }

1259 // Find highest numbered file within the directory.

1260 entries, err := os.ReadDir(dir)

1261 if err != nil && !errors.Is(err, fs.ErrNotExist) {

1262 return fmt.Errorf("read message directory %q: %v", dir, err)

1263 }

1264 dirFirstID := maxFSID &^ (msgFilesPerDir - 1)

1265 for _, e := range entries {

1266 id, err := strconv.ParseInt(e.Name(), 10, 64)

1267 if err == nil && (id < dirFirstID || id >= dirFirstID+msgFilesPerDir) {

1268 err = fmt.Errorf("directory %s has message id %d outside of range [%d - %d), ignoring", dir, id, dirFirstID, dirFirstID+msgFilesPerDir)

1269 }

1270 if err != nil {

1271 p := filepath.Join(dir, e.Name())

1272 log.Errorx("unrecognized file in message directory, parsing filename as number", err, slog.String("path", p))

1273 } else {

1274 maxFSID = max(maxFSID, id)

1275 }

1276 }

1277 // Warn if we need to increase the message ID in the database.

1278 var mailboxID int64

1279 if maxFSID > maxDBID {

1280 log.Warn("unexpected message file with higher message id than highest id in database, moving database id sequence forward to prevent clashes during future deliveries", slog.Int64("maxdbmsgid", maxDBID), slog.Int64("maxfilemsgid", maxFSID))

1281

1282 mb, err := bstore.QueryTx[Mailbox](tx).Limit(1).Get()

1283 if err != nil {

1284 return fmt.Errorf("get a mailbox: %v", err)

1285 }

1286 mailboxID = mb.ID

1287 }

1288 for maxFSID > maxDBID {

1289 // Set fields that must be non-zero.

1290 m := Message{

1291 UID: ^UID(0),

1292 MailboxID: mailboxID,

1293 }

1294 // Insert and delete to increase the sequence, silly but effective.

1295 if err := tx.Insert(&m); err != nil {

1296 return fmt.Errorf("inserting message to increase id: %v", err)

1297 }

1298 if err := tx.Delete(&m); err != nil {

1299 return fmt.Errorf("deleting message after increasing id: %v", err)

1300 }

1301 maxDBID = m.ID

1302 }

1303

1304 return nil

1305 })

1306 if err != nil {

1307 return nil, fmt.Errorf("calculating counts for mailbox, inserting settings, expunging messages: %v", err)

1308 }

1309

1310 up := Upgrade{ID: 1}

1311 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

1312 err := tx.Get(&up)

1313 if err == bstore.ErrAbsent {

1314 if err := tx.Insert(&up); err != nil {

1315 return fmt.Errorf("inserting initial upgrade record: %v", err)

1316 }

1317 err = nil

1318 }

1319 return err

1320 })

1321 if err != nil {

1322 return nil, fmt.Errorf("checking message threading: %v", err)

1323 }

1324

1325 // Ensure all mailboxes have a modseq based on highest modseq message in each

1326 // mailbox, and a createseq.

1327 if !up.MailboxModSeq {

1328 log.Debug("upgrade: adding modseq to each mailbox")

1329 err := acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1330 var modseq ModSeq

1331

1332 mbl, err := bstore.QueryTx[Mailbox](tx).FilterEqual("Expunged", false).List()

1333 if err != nil {

1334 return fmt.Errorf("listing mailboxes: %v", err)

1335 }

1336 for _, mb := range mbl {

1337 // Get current highest modseq of message in account.

1338 qms := bstore.QueryTx[Message](tx)

1339 qms.FilterNonzero(Message{MailboxID: mb.ID})

1340 qms.SortDesc("ModSeq")

1341 qms.Limit(1)

1342 m, err := qms.Get()

1343 if err == nil {

1344 mb.ModSeq = ModSeq(m.ModSeq.Client())

1345 } else if err == bstore.ErrAbsent {

1346 if modseq == 0 {

1347 modseq, err = acc.NextModSeq(tx)

1348 if err != nil {

1349 return fmt.Errorf("get next mod seq for mailbox without messages: %v", err)

1350 }

1351 }

1352 mb.ModSeq = modseq

1353 } else {

1354 return fmt.Errorf("looking up highest modseq for mailbox: %v", err)

1355 }

1356 mb.CreateSeq = 1

1357 if err := tx.Update(&mb); err != nil {

1358 return fmt.Errorf("updating mailbox with modseq: %v", err)

1359 }

1360 }

1361

1362 up.MailboxModSeq = true

1363 if err := tx.Update(&up); err != nil {

1364 return fmt.Errorf("marking upgrade done: %v", err)

1365 }

1366

1367 return nil

1368 })

1369 if err != nil {

1370 return nil, fmt.Errorf("upgrade: adding modseq to each mailbox: %v", err)

1371 }

1372 }

1373

1374 // Add ParentID to mailboxes.

1375 if !up.MailboxParentID {

1376 log.Debug("upgrade: setting parentid on each mailbox")

1377

1378 err := acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1379 mbl, err := bstore.QueryTx[Mailbox](tx).FilterEqual("Expunged", false).SortAsc("Name").List()

1380 if err != nil {

1381 return fmt.Errorf("listing mailboxes: %w", err)

1382 }

1383

1384 names := map[string]Mailbox{}

1385 for _, mb := range mbl {

1386 names[mb.Name] = mb

1387 }

1388

1389 var modseq ModSeq

1390

1391 // Ensure a parent mailbox for name exists, creating it if needed, including any

1392 // grandparents, up to the top.

1393 var ensureParentMailboxID func(name string) (int64, error)

1394 ensureParentMailboxID = func(name string) (int64, error) {

1395 parentName := mox.ParentMailboxName(name)

1396 if parentName == "" {

1397 return 0, nil

1398 }

1399 parent := names[parentName]

1400 if parent.ID != 0 {

1401 return parent.ID, nil

1402 }

1403

1404 parentParentID, err := ensureParentMailboxID(parentName)

1405 if err != nil {

1406 return 0, fmt.Errorf("creating parent mailbox %q: %w", parentName, err)

1407 }

1408

1409 if modseq == 0 {

1410 modseq, err = a.NextModSeq(tx)

1411 if err != nil {

1412 return 0, fmt.Errorf("get next modseq: %w", err)

1413 }

1414 }

1415

1416 uidvalidity, err := a.NextUIDValidity(tx)

1417 if err != nil {

1418 return 0, fmt.Errorf("next uid validity: %w", err)

1419 }

1420

1421 parent = Mailbox{

1422 CreateSeq: modseq,

1423 ModSeq: modseq,

1424 ParentID: parentParentID,

1425 Name: parentName,

1426 UIDValidity: uidvalidity,

1427 UIDNext: 1,

1428 SpecialUse: SpecialUse{},

1429 HaveCounts: true,

1430 }

1431 if err := tx.Insert(&parent); err != nil {

1432 return 0, fmt.Errorf("creating parent mailbox: %w", err)

1433 }

1434 return parent.ID, nil

1435 }

1436

1437 for _, mb := range mbl {

1438 parentID, err := ensureParentMailboxID(mb.Name)

1439 if err != nil {

1440 return fmt.Errorf("creating missing parent mailbox for mailbox %q: %w", mb.Name, err)

1441 }

1442 mb.ParentID = parentID

1443 if err := tx.Update(&mb); err != nil {

1444 return fmt.Errorf("update mailbox with parentid: %w", err)

1445 }

1446 }

1447

1448 up.MailboxParentID = true

1449 if err := tx.Update(&up); err != nil {

1450 return fmt.Errorf("marking upgrade done: %w", err)

1451 }

1452 return nil

1453 })

1454 if err != nil {

1455 return nil, fmt.Errorf("upgrade: setting parentid on each mailbox: %w", err)

1456 }

1457 }

1458

1459 if !up.MailboxCounts {

1460 log.Debug("upgrade: ensuring all mailboxes have message counts")

1461

1462 err := acc.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1463 err := bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error {

1464 mc, err := mb.CalculateCounts(tx)

1465 if err != nil {

1466 return err

1467 }

1468 mb.HaveCounts = true

1469 mb.MailboxCounts = mc

1470 return tx.Update(&mb)

1471 })

1472 if err != nil {

1473 return err

1474 }

1475

1476 up.MailboxCounts = true

1477 if err := tx.Update(&up); err != nil {

1478 return fmt.Errorf("marking upgrade done: %w", err)

1479 }

1480 return nil

1481 })

1482 if err != nil {

1483 return nil, fmt.Errorf("upgrade: ensuring message counts on all mailboxes")

1484 }

1485 }

1486

1487 if up.Threads == 2 {

1488 close(acc.threadsCompleted)

1489 return acc, nil

1490 }

1491

1492 // Increase account use before holding on to account in background.

1493 // Caller holds the lock. The goroutine below decreases nused by calling

1494 // closeAccount.

1495 acc.nused++

1496

1497 // Ensure all messages have a MessageID and SubjectBase, which are needed when

1498 // matching threads.

1499 // Then assign messages to threads, in the same way we do during imports.

1500 log.Info("upgrading account for threading, in background", slog.String("account", acc.Name))

1501 go func() {

1502 defer func() {

1503 err := closeAccount(acc)

1504 log.Check(err, "closing use of account after upgrading account storage for threads", slog.String("account", a.Name))

1505

1506 // Mark that upgrade has finished, possibly error is indicated in threadsErr.

1507 close(acc.threadsCompleted)

1508 }()

1509

1510 defer func() {

1511 x := recover() // Should not happen, but don't take program down if it does.

1512 if x != nil {

1513 log.Error("upgradeThreads panic", slog.Any("err", x))

1514 debug.PrintStack()

1515 metrics.PanicInc(metrics.Upgradethreads)

1516 acc.threadsErr = fmt.Errorf("panic during upgradeThreads: %v", x)

1517 }

1518 }()

1519

1520 err := upgradeThreads(mox.Shutdown, log, acc, up)

1521 if err != nil {

1522 a.threadsErr = err

1523 log.Errorx("upgrading account for threading, aborted", err, slog.String("account", a.Name))

1524 } else {

1525 log.Info("upgrading account for threading, completed", slog.String("account", a.Name))

1526 }

1527 }()

1528 return acc, nil

1529}

1530

1531// ThreadingWait blocks until the one-time account threading upgrade for the

1532// account has completed, and returns an error if not successful.

1533//

1534// To be used before starting an import of messages.

1535func (a *Account) ThreadingWait(log mlog.Log) error {

1536 select {

1537 case <-a.threadsCompleted:

1538 return a.threadsErr

1539 default:

1540 }

1541 log.Debug("waiting for account upgrade to complete")

1542

1543 <-a.threadsCompleted

1544 return a.threadsErr

1545}

1546

1547func initAccount(db *bstore.DB) error {

1548 return db.Write(context.TODO(), func(tx *bstore.Tx) error {

1549 uidvalidity := InitialUIDValidity()

1550

1551 if err := tx.Insert(&upgradeInit); err != nil {

1552 return err

1553 }

1554 if err := tx.Insert(&DiskUsage{ID: 1}); err != nil {

1555 return err

1556 }

1557 if err := tx.Insert(&Settings{ID: 1}); err != nil {

1558 return err

1559 }

1560

1561 modseq, err := nextModSeq(tx)

1562 if err != nil {

1563 return fmt.Errorf("get next modseq: %v", err)

1564 }

1565

1566 if len(mox.Conf.Static.DefaultMailboxes) > 0 {

1567 // Deprecated in favor of InitialMailboxes.

1568 defaultMailboxes := mox.Conf.Static.DefaultMailboxes

1569 mailboxes := []string{"Inbox"}

1570 for _, name := range defaultMailboxes {

1571 if strings.EqualFold(name, "Inbox") {

1572 continue

1573 }

1574 mailboxes = append(mailboxes, name)

1575 }

1576 for _, name := range mailboxes {

1577 mb := Mailbox{

1578 CreateSeq: modseq,

1579 ModSeq: modseq,

1580 ParentID: 0,

1581 Name: name,

1582 UIDValidity: uidvalidity,

1583 UIDNext: 1,

1584 HaveCounts: true,

1585 }

1586 if strings.HasPrefix(name, "Archive") {

1587 mb.Archive = true

1588 } else if strings.HasPrefix(name, "Drafts") {

1589 mb.Draft = true

1590 } else if strings.HasPrefix(name, "Junk") {

1591 mb.Junk = true

1592 } else if strings.HasPrefix(name, "Sent") {

1593 mb.Sent = true

1594 } else if strings.HasPrefix(name, "Trash") {

1595 mb.Trash = true

1596 }

1597 if err := tx.Insert(&mb); err != nil {

1598 return fmt.Errorf("creating mailbox: %w", err)

1599 }

1600 if err := tx.Insert(&Subscription{name}); err != nil {

1601 return fmt.Errorf("adding subscription: %w", err)

1602 }

1603 }

1604 } else {

1605 mailboxes := mox.Conf.Static.InitialMailboxes

1606 var zerouse config.SpecialUseMailboxes

1607 if mailboxes.SpecialUse == zerouse && len(mailboxes.Regular) == 0 {

1608 mailboxes = DefaultInitialMailboxes

1609 }

1610

1611 add := func(name string, use SpecialUse) error {

1612 mb := Mailbox{

1613 CreateSeq: modseq,

1614 ModSeq: modseq,

1615 ParentID: 0,

1616 Name: name,

1617 UIDValidity: uidvalidity,

1618 UIDNext: 1,

1619 SpecialUse: use,

1620 HaveCounts: true,

1621 }

1622 if err := tx.Insert(&mb); err != nil {

1623 return fmt.Errorf("creating mailbox: %w", err)

1624 }

1625 if err := tx.Insert(&Subscription{name}); err != nil {

1626 return fmt.Errorf("adding subscription: %w", err)

1627 }

1628 return nil

1629 }

1630 addSpecialOpt := func(nameOpt string, use SpecialUse) error {

1631 if nameOpt == "" {

1632 return nil

1633 }

1634 return add(nameOpt, use)

1635 }

1636 l := []struct {

1637 nameOpt string

1638 use SpecialUse

1639 }{

1640 {"Inbox", SpecialUse{}},

1641 {mailboxes.SpecialUse.Archive, SpecialUse{Archive: true}},

1642 {mailboxes.SpecialUse.Draft, SpecialUse{Draft: true}},

1643 {mailboxes.SpecialUse.Junk, SpecialUse{Junk: true}},

1644 {mailboxes.SpecialUse.Sent, SpecialUse{Sent: true}},

1645 {mailboxes.SpecialUse.Trash, SpecialUse{Trash: true}},

1646 }

1647 for _, e := range l {

1648 if err := addSpecialOpt(e.nameOpt, e.use); err != nil {

1649 return err

1650 }

1651 }

1652 for _, name := range mailboxes.Regular {

1653 if err := add(name, SpecialUse{}); err != nil {

1654 return err

1655 }

1656 }

1657 }

1658

1659 uidvalidity++

1660 if err := tx.Insert(&NextUIDValidity{1, uidvalidity}); err != nil {

1661 return fmt.Errorf("inserting nextuidvalidity: %w", err)

1662 }

1663 return nil

1664 })

1665}

1666

1667// Remove schedules an account for removal. New opens will fail. When the last

1668// reference is closed, the account files are removed.

1669func (a *Account) Remove(ctx context.Context) error {

1670 openAccounts.Lock()

1671 defer openAccounts.Unlock()

1672

1673 if err := AuthDB.Insert(ctx, &AccountRemove{AccountName: a.Name}); err != nil {

1674 return fmt.Errorf("inserting account removal: %w", err)

1675 }

1676 a.removed = true

1677

1678 return nil

1679}

1680

1681// WaitClosed waits until the last reference to this account is gone and the

1682// account is closed. Used during tests, to ensure the consistency checks run after

1683// expunged messages have been erased.

1684func (a *Account) WaitClosed() {

1685 <-a.closed

1686}

1687

1688// Close reduces the reference count, and closes the database connection when

1689// it was the last user.

1690func (a *Account) Close() error {

1691 if CheckConsistencyOnClose {

1692 xerr := a.CheckConsistency()

1693 err := closeAccount(a)

1694 if xerr != nil {

1695 panic(xerr)

1696 }

1697 return err

1698 }

1699 return closeAccount(a)

1700}

1701

1702// SetSkipMessageModSeqZeroCheck skips consistency checks for Message.ModSeq and

1703// Message.CreateSeq being zero.

1704func (a *Account) SetSkipMessageModSeqZeroCheck(skip bool) {

1705 a.Lock()

1706 defer a.Unlock()

1707 a.skipMessageZeroSeqCheck = true

1708}

1709

1710// CheckConsistency checks the consistency of the database and returns a non-nil

1711// error for these cases:

1712//

1713// - Missing or unexpected on-disk message files.

1714// - Mismatch between message size and length of MsgPrefix and on-disk file.

1715// - Incorrect mailbox counts.

1716// - Incorrect total message size.

1717// - Message with UID >= mailbox uid next.

1718// - Mailbox uidvalidity >= account uid validity.

1719// - Mailbox ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq, and Modseq >= highest message ModSeq.

1720// - Mailbox must have a live parent ID if they are live themselves, live names must be unique.

1721// - Message ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq.

1722// - All messages have a nonzero ThreadID, and no cycles in ThreadParentID, and parent messages the same ThreadParentIDs tail.

1723// - Annotations must have ModSeq > 0, CreateSeq > 0, ModSeq >= CreateSeq and live keys must be unique per mailbox.

1724// - Recalculate junk filter (words and counts) and check they are the same.

1725func (a *Account) CheckConsistency() error {

1726 a.Lock()

1727 defer a.Unlock()

1728

1729 var uidErrors []string // With a limit, could be many.

1730 var modseqErrors []string // With limit.

1731 var fileErrors []string // With limit.

1732 var threadidErrors []string // With limit.

1733 var threadParentErrors []string // With limit.

1734 var threadAncestorErrors []string // With limit.

1735 var errmsgs []string

1736

1737 ctx := context.Background()

1738 log := mlog.New("store", nil)

1739

1740 err := a.DB.Read(ctx, func(tx *bstore.Tx) error {

1741 nuv := NextUIDValidity{ID: 1}

1742 err := tx.Get(&nuv)

1743 if err != nil {

1744 return fmt.Errorf("fetching next uid validity: %v", err)

1745 }

1746

1747 mailboxes := map[int64]Mailbox{} // Also expunged mailboxes.

1748 mailboxNames := map[string]Mailbox{} // Only live names.

1749 err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {

1750 mailboxes[mb.ID] = mb

1751 if !mb.Expunged {

1752 if xmb, ok := mailboxNames[mb.Name]; ok {

1753 errmsg := fmt.Sprintf("mailbox %q exists as id %d and id %d", mb.Name, mb.ID, xmb.ID)

1754 errmsgs = append(errmsgs, errmsg)

1755 }

1756 mailboxNames[mb.Name] = mb

1757 }

1758

1759 if mb.UIDValidity >= nuv.Next {

1760 errmsg := fmt.Sprintf("mailbox %q (id %d) has uidvalidity %d >= account next uidvalidity %d", mb.Name, mb.ID, mb.UIDValidity, nuv.Next)

1761 errmsgs = append(errmsgs, errmsg)

1762 }

1763

1764 if mb.ModSeq == 0 || mb.CreateSeq == 0 || mb.CreateSeq > mb.ModSeq {

1765 errmsg := fmt.Sprintf("mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and createseq <= modseq", mb.Name, mb.ID, mb.ModSeq, mb.CreateSeq)

1766 errmsgs = append(errmsgs, errmsg)

1767 return nil

1768 }

1769 m, err := bstore.QueryTx[Message](tx).FilterNonzero(Message{MailboxID: mb.ID}).SortDesc("ModSeq").Limit(1).Get()

1770 if err == bstore.ErrAbsent {

1771 return nil

1772 } else if err != nil {

1773 return fmt.Errorf("get message with highest modseq for mailbox: %v", err)

1774 } else if mb.ModSeq < m.ModSeq {

1775 errmsg := fmt.Sprintf("mailbox %q (id %d) has modseq %d < highest message modseq is %d", mb.Name, mb.ID, mb.ModSeq, m.ModSeq)

1776 errmsgs = append(errmsgs, errmsg)

1777 }

1778 return nil

1779 })

1780 if err != nil {

1781 return fmt.Errorf("checking mailboxes: %v", err)

1782 }

1783

1784 // Check ParentID and name of parent.

1785 for _, mb := range mailboxNames {

1786 if mox.ParentMailboxName(mb.Name) == "" {

1787 if mb.ParentID == 0 {

1788 continue

1789 }

1790 errmsg := fmt.Sprintf("mailbox %q (id %d) is a root mailbox but has parentid %d", mb.Name, mb.ID, mb.ParentID)

1791 errmsgs = append(errmsgs, errmsg)

1792 } else if mb.ParentID == 0 {

1793 errmsg := fmt.Sprintf("mailbox %q (id %d) is not a root mailbox but has a zero parentid", mb.Name, mb.ID)

1794 errmsgs = append(errmsgs, errmsg)

1795 } else if mox.ParentMailboxName(mb.Name) != mailboxes[mb.ParentID].Name {

1796 errmsg := fmt.Sprintf("mailbox %q (id %d) has parent mailbox id %d with name %q, but parent name should be %q", mb.Name, mb.ID, mb.ParentID, mailboxes[mb.ParentID].Name, mox.ParentMailboxName(mb.Name))

1797 errmsgs = append(errmsgs, errmsg)

1798 }

1799 }

1800

1801 type annotation struct {

1802 mailboxID int64 // Can be 0.

1803 key string

1804 }

1805 annotations := map[annotation]struct{}{}

1806 err = bstore.QueryTx[Annotation](tx).ForEach(func(a Annotation) error {

1807 if !a.Expunged {

1808 k := annotation{a.MailboxID, a.Key}

1809 if _, ok := annotations[k]; ok {

1810 errmsg := fmt.Sprintf("duplicate live annotation key %q for mailbox id %d", a.Key, a.MailboxID)

1811 errmsgs = append(errmsgs, errmsg)

1812 }

1813 annotations[k] = struct{}{}

1814 }

1815 if a.ModSeq == 0 || a.CreateSeq == 0 || a.CreateSeq > a.ModSeq {

1816 errmsg := fmt.Sprintf("annotation %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and modseq >= createseq", a.ID, mailboxes[a.MailboxID].Name, a.MailboxID, a.ModSeq, a.CreateSeq)

1817 errmsgs = append(errmsgs, errmsg)

1818 } else if a.MailboxID > 0 && mailboxes[a.MailboxID].ModSeq < a.ModSeq {

1819 errmsg := fmt.Sprintf("annotation %d in mailbox %q (id %d) has invalid modseq %d > mailbox modseq %d", a.ID, mailboxes[a.MailboxID].Name, a.MailboxID, a.ModSeq, mailboxes[a.MailboxID].ModSeq)

1820 errmsgs = append(errmsgs, errmsg)

1821 }

1822 return nil

1823 })

1824 if err != nil {

1825 return fmt.Errorf("checking mailbox annotations: %v", err)

1826 }

1827

1828 // All message id's from database. For checking for unexpected files afterwards.

1829 messageIDs := map[int64]struct{}{}

1830 eraseMessageIDs := map[int64]bool{} // Value indicates whether to skip updating disk usage.

1831

1832 // If configured, we'll be building up the junk filter for the messages, to compare

1833 // against the on-disk junk filter.

1834 var jf *junk.Filter

1835 conf, _ := a.Conf()

1836 if conf.JunkFilter != nil {

1837 random := make([]byte, 16)

1838 if _, err := cryptorand.Read(random); err != nil {

1839 return fmt.Errorf("reading random: %v", err)

1840 }

1841 dbpath := filepath.Join(mox.DataDirPath("tmp"), fmt.Sprintf("junkfilter-check-%x.db", random))

1842 bloompath := filepath.Join(mox.DataDirPath("tmp"), fmt.Sprintf("junkfilter-check-%x.bloom", random))

1843 os.MkdirAll(filepath.Dir(dbpath), 0700)

1844 defer func() {

1845 err := os.Remove(bloompath)

1846 log.Check(err, "removing temp bloom file")

1847 err = os.Remove(dbpath)

1848 log.Check(err, "removing temp junk filter database file")

1849 }()

1850 jf, err = junk.NewFilter(ctx, log, conf.JunkFilter.Params, dbpath, bloompath)

1851 if err != nil {

1852 return fmt.Errorf("new junk filter: %v", err)

1853 }

1854 defer func() {

1855 err := jf.Close()

1856 log.Check(err, "closing junk filter")

1857 }()

1858 }

1859 var ntrained int

1860

1861 // Get IDs of erase messages not yet removed, they'll have a message file.

1862 err = bstore.QueryTx[MessageErase](tx).ForEach(func(me MessageErase) error {

1863 eraseMessageIDs[me.ID] = me.SkipUpdateDiskUsage

1864 return nil

1865 })

1866 if err != nil {

1867 return fmt.Errorf("listing message erase records")

1868 }

1869

1870 counts := map[int64]MailboxCounts{}

1871 var totalExpungedSize int64

1872 err = bstore.QueryTx[Message](tx).ForEach(func(m Message) error {

1873 mc := counts[m.MailboxID]

1874 mc.Add(m.MailboxCounts())

1875 counts[m.MailboxID] = mc

1876

1877 mb := mailboxes[m.MailboxID]

1878

1879 if (!a.skipMessageZeroSeqCheck && (m.ModSeq == 0 || m.CreateSeq == 0) || m.CreateSeq > m.ModSeq) && len(modseqErrors) < 20 {

1880 modseqerr := fmt.Sprintf("message %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and createseq <= modseq", m.ID, mb.Name, mb.ID, m.ModSeq, m.CreateSeq)

1881 modseqErrors = append(modseqErrors, modseqerr)

1882 }

1883 if m.UID >= mb.UIDNext && len(uidErrors) < 20 {

1884 uiderr := fmt.Sprintf("message %d in mailbox %q (id %d) has uid %d >= mailbox uidnext %d", m.ID, mb.Name, mb.ID, m.UID, mb.UIDNext)

1885 uidErrors = append(uidErrors, uiderr)

1886 }

1887 if m.Expunged {

1888 if skip := eraseMessageIDs[m.ID]; !skip {

1889 totalExpungedSize += m.Size

1890 }

1891 return nil

1892 }

1893

1894 messageIDs[m.ID] = struct{}{}

1895 p := a.MessagePath(m.ID)

1896 st, err := os.Stat(p)

1897 if err != nil {

1898 existserr := fmt.Sprintf("message %d in mailbox %q (id %d) on-disk file %s: %v", m.ID, mb.Name, mb.ID, p, err)

1899 fileErrors = append(fileErrors, existserr)

1900 } else if len(fileErrors) < 20 && m.Size != int64(len(m.MsgPrefix))+st.Size() {

1901 sizeerr := fmt.Sprintf("message %d in mailbox %q (id %d) has size %d != len msgprefix %d + on-disk file size %d = %d", m.ID, mb.Name, mb.ID, m.Size, len(m.MsgPrefix), st.Size(), int64(len(m.MsgPrefix))+st.Size())

1902 fileErrors = append(fileErrors, sizeerr)

1903 }

1904

1905 if m.ThreadID <= 0 && len(threadidErrors) < 20 {

1906 err := fmt.Sprintf("message %d in mailbox %q (id %d) has threadid 0", m.ID, mb.Name, mb.ID)

1907 threadidErrors = append(threadidErrors, err)

1908 }

1909 if slices.Contains(m.ThreadParentIDs, m.ID) && len(threadParentErrors) < 20 {

1910 err := fmt.Sprintf("message %d in mailbox %q (id %d) references itself in threadparentids", m.ID, mb.Name, mb.ID)

1911 threadParentErrors = append(threadParentErrors, err)

1912 }

1913 for i, pid := range m.ThreadParentIDs {

1914 am := Message{ID: pid}

1915 if err := tx.Get(&am); err == bstore.ErrAbsent || err == nil && am.Expunged {

1916 continue

1917 } else if err != nil {

1918 return fmt.Errorf("get ancestor message: %v", err)

1919 } else if !slices.Equal(m.ThreadParentIDs[i+1:], am.ThreadParentIDs) && len(threadAncestorErrors) < 20 {

1920 err := fmt.Sprintf("message %d, thread %d has ancestor ids %v, and ancestor at index %d with id %d should have the same tail but has %v\n", m.ID, m.ThreadID, m.ThreadParentIDs, i, am.ID, am.ThreadParentIDs)

1921 threadAncestorErrors = append(threadAncestorErrors, err)

1922 } else {

1923 break

1924 }

1925 }

1926

1927 if jf != nil {

1928 if m.Junk != m.Notjunk {

1929 ntrained++

1930 if _, err := a.TrainMessage(ctx, log, jf, m.Notjunk, m); err != nil {

1931 return fmt.Errorf("train message: %v", err)

1932 }

1933 // We are not setting m.TrainedJunk, we were only recalculating the words.

1934 }

1935 }

1936

1937 return nil

1938 })

1939 if err != nil {

1940 return fmt.Errorf("reading messages: %v", err)

1941 }

1942

1943 msgdir := filepath.Join(a.Dir, "msg")

1944 err = filepath.WalkDir(msgdir, func(path string, entry fs.DirEntry, err error) error {

1945 if err != nil {

1946 if path == msgdir && errors.Is(err, fs.ErrNotExist) {

1947 return nil

1948 }

1949 return err

1950 }

1951 if entry.IsDir() {

1952 return nil

1953 }

1954 id, err := strconv.ParseInt(filepath.Base(path), 10, 64)

1955 if err != nil {

1956 return fmt.Errorf("parsing message id from path %q: %v", path, err)

1957 }

1958 _, mok := messageIDs[id]

1959 _, meok := eraseMessageIDs[id]

1960 if !mok && !meok {

1961 return fmt.Errorf("unexpected message file %q", path)

1962 }

1963 return nil

1964 })

1965 if err != nil {

1966 return fmt.Errorf("walking message dir: %v", err)

1967 }

1968

1969 var totalMailboxSize int64

1970 for _, mb := range mailboxNames {

1971 totalMailboxSize += mb.Size

1972 if mb.MailboxCounts != counts[mb.ID] {

1973 mbcounterr := fmt.Sprintf("mailbox %q (id %d) has wrong counts %s, should be %s", mb.Name, mb.ID, mb.MailboxCounts, counts[mb.ID])

1974 errmsgs = append(errmsgs, mbcounterr)

1975 }

1976 }

1977

1978 du := DiskUsage{ID: 1}

1979 if err := tx.Get(&du); err != nil {

1980 return fmt.Errorf("get diskusage")

1981 }

1982 if du.MessageSize != totalMailboxSize+totalExpungedSize {

1983 errmsg := fmt.Sprintf("total disk usage message size in database is %d != sum of mailbox message sizes %d + sum unerased expunged message sizes %d", du.MessageSize, totalMailboxSize, totalExpungedSize)

1984 errmsgs = append(errmsgs, errmsg)

1985 }

1986

1987 // Compare on-disk junk filter with our recalculated filter.

1988 if jf != nil {

1989 load := func(f *junk.Filter) (map[junk.Wordscore]struct{}, error) {

1990 words := map[junk.Wordscore]struct{}{}

1991 err := bstore.QueryDB[junk.Wordscore](ctx, f.DB()).ForEach(func(w junk.Wordscore) error {

1992 if w.Ham != 0 || w.Spam != 0 {

1993 words[w] = struct{}{}

1994 }

1995 return nil

1996 })

1997 if err != nil {

1998 return nil, fmt.Errorf("read junk filter wordscores: %v", err)

1999 }

2000 return words, nil

2001 }

2002 if err := jf.Save(); err != nil {

2003 return fmt.Errorf("save recalculated junk filter: %v", err)

2004 }

2005 wordsExp, err := load(jf)

2006 if err != nil {

2007 return fmt.Errorf("read recalculated junk filter: %v", err)

2008 }

2009

2010 ajf, _, err := a.OpenJunkFilter(ctx, log)

2011 if err != nil {

2012 return fmt.Errorf("open account junk filter: %v", err)

2013 }

2014 defer func() {

2015 err := ajf.Close()

2016 log.Check(err, "closing junk filter")

2017 }()

2018 wordsGot, err := load(ajf)

2019 if err != nil {

2020 return fmt.Errorf("read account junk filter: %v", err)

2021 }

2022

2023 if !reflect.DeepEqual(wordsGot, wordsExp) {

2024 errmsg := fmt.Sprintf("unexpected values in junk filter, trained %d of %d\ngot:\n%v\nexpected:\n%v", ntrained, len(messageIDs), wordsGot, wordsExp)

2025 errmsgs = append(errmsgs, errmsg)

2026 }

2027 }

2028

2029 return nil

2030 })

2031 if err != nil {

2032 return err

2033 }

2034 errmsgs = append(errmsgs, uidErrors...)

2035 errmsgs = append(errmsgs, modseqErrors...)

2036 errmsgs = append(errmsgs, fileErrors...)

2037 errmsgs = append(errmsgs, threadidErrors...)

2038 errmsgs = append(errmsgs, threadParentErrors...)

2039 errmsgs = append(errmsgs, threadAncestorErrors...)

2040 if len(errmsgs) > 0 {

2041 return fmt.Errorf("%s", strings.Join(errmsgs, "; "))

2042 }

2043 return nil

2044}

2045

2046// Conf returns the configuration for this account if it still exists. During

2047// an SMTP session, a configuration update may drop an account.

2048func (a *Account) Conf() (config.Account, bool) {

2049 return mox.Conf.Account(a.Name)

2050}

2051

2052// NextUIDValidity returns the next new/unique uidvalidity to use for this account.

2053func (a *Account) NextUIDValidity(tx *bstore.Tx) (uint32, error) {

2054 nuv := NextUIDValidity{ID: 1}

2055 if err := tx.Get(&nuv); err != nil {

2056 return 0, err

2057 }

2058 v := nuv.Next

2059 nuv.Next++

2060 if err := tx.Update(&nuv); err != nil {

2061 return 0, err

2062 }

2063 return v, nil

2064}

2065

2066// NextModSeq returns the next modification sequence, which is global per account,

2067// over all types.

2068func (a *Account) NextModSeq(tx *bstore.Tx) (ModSeq, error) {

2069 return nextModSeq(tx)

2070}

2071

2072func nextModSeq(tx *bstore.Tx) (ModSeq, error) {

2073 v := SyncState{ID: 1}

2074 if err := tx.Get(&v); err == bstore.ErrAbsent {

2075 // We start assigning from modseq 2. Modseq 0 is not usable, so returned as 1, so

2076 // already used.

2077 // HighestDeletedModSeq is -1 so comparison against the default ModSeq zero value

2078 // makes sense.

2079 v = SyncState{1, 2, -1}

2080 return v.LastModSeq, tx.Insert(&v)

2081 } else if err != nil {

2082 return 0, err

2083 }

2084 v.LastModSeq++

2085 return v.LastModSeq, tx.Update(&v)

2086}

2087

2088func (a *Account) HighestDeletedModSeq(tx *bstore.Tx) (ModSeq, error) {

2089 v := SyncState{ID: 1}

2090 err := tx.Get(&v)

2091 if err == bstore.ErrAbsent {

2092 return 0, nil

2093 }

2094 return v.HighestDeletedModSeq, err

2095}

2096

2097// WithWLock runs fn with account writelock held. Necessary for account/mailbox

2098// modification. For message delivery, a read lock is required.

2099func (a *Account) WithWLock(fn func()) {

2100 a.Lock()

2101 defer a.Unlock()

2102 fn()

2103}

2104

2105// WithRLock runs fn with account read lock held. Needed for message delivery.

2106func (a *Account) WithRLock(fn func()) {

2107 a.RLock()

2108 defer a.RUnlock()

2109 fn()

2110}

2111

2112// AddOpts influence which work MessageAdd does. Some callers can batch

2113// checks/operations efficiently. For convenience and safety, a zero AddOpts does

2114// all the checks and work.

2115type AddOpts struct {

2116 SkipCheckQuota bool

2117

2118 // If set, the message size is not added to the disk usage. Caller must do that,

2119 // e.g. for many messages at once. If used together with SkipCheckQuota, the

2120 // DiskUsage is not read for database when adding a message.

2121 SkipUpdateDiskUsage bool

2122

2123 // Do not fsync the delivered message file. Useful when copying message files from

2124 // another mailbox. The hardlink created during delivery only needs a directory

2125 // fsync.

2126 SkipSourceFileSync bool

2127

2128 // The directory in which the message file is delivered, typically with a hard

2129 // link, is not fsynced. Useful when delivering many files. A single or few

2130 // directory fsyncs are more efficient.

2131 SkipDirSync bool

2132

2133 // Do not assign thread information to a message. Useful when importing many

2134 // messages and assigning threads efficiently after importing messages.

2135 SkipThreads bool

2136

2137 // If JunkFilter is set, it is used for training. If not set, and the filter must

2138 // be trained for a message, the junk filter is opened, modified and saved to disk.

2139 JunkFilter *junk.Filter

2140

2141 SkipTraining bool

2142

2143 // If true, a preview will be generated if the Message doesn't already have one.

2144 SkipPreview bool

2145}

2146

2147// todo optimization: when moving files, we open the original, call MessageAdd() which hardlinks it and close the file gain. when passing the filename, we could just use os.Link, saves 2 syscalls.

2148

2149// MessageAdd delivers a mail message to the account.

2150//

2151// The file is hardlinked or copied, the caller must clean up the original file. If

2152// this call succeeds, but the database transaction with the change can't be

2153// committed, the caller must clean up the delivered message file identified by

2154// m.ID.

2155//

2156// If the message does not fit in the quota, an error with ErrOverQuota is returned

2157// and the mailbox and message are unchanged and the transaction can continue. For

2158// other errors, the caller must abort the transaction.

2159//

2160// The message, with msg.MsgPrefix and msgFile combined, must have a header

2161// section. The caller is responsible for adding a header separator to

2162// msg.MsgPrefix if missing from an incoming message.

2163//

2164// If UID is not set, it is assigned automatically.

2165//

2166// If the message ModSeq is zero, it is assigned automatically. If the message

2167// CreateSeq is zero, it is set to ModSeq. The mailbox ModSeq is set to the message

2168// ModSeq.

2169//

2170// If the message does not fit in the quota, an error with ErrOverQuota is returned

2171// and the mailbox and message are unchanged and the transaction can continue. For

2172// other errors, the caller must abort the transaction.

2173//

2174// If the destination mailbox has the Sent special-use flag, the message is parsed

2175// for its recipients (to/cc/bcc). Their domains are added to Recipients for use in

2176// reputation classification.

2177//

2178// Must be called with account write lock held.

2179//

2180// Caller must save the mailbox after MessageAdd returns, and broadcast changes for

2181// new the message, updated mailbox counts and possibly new mailbox keywords.

2182func (a *Account) MessageAdd(log mlog.Log, tx *bstore.Tx, mb *Mailbox, m *Message, msgFile *os.File, opts AddOpts) (rerr error) {

2183 if m.Expunged {

2184 return fmt.Errorf("cannot deliver expunged message")

2185 }

2186

2187 if !opts.SkipUpdateDiskUsage || !opts.SkipCheckQuota {

2188 du := DiskUsage{ID: 1}

2189 if err := tx.Get(&du); err != nil {

2190 return fmt.Errorf("get disk usage: %v", err)

2191 }

2192

2193 if !opts.SkipCheckQuota {

2194 maxSize := a.QuotaMessageSize()

2195 if maxSize > 0 && m.Size > maxSize-du.MessageSize {

2196 return fmt.Errorf("%w: max size %d bytes", ErrOverQuota, maxSize)

2197 }

2198 }

2199

2200 if !opts.SkipUpdateDiskUsage {

2201 du.MessageSize += m.Size

2202 if err := tx.Update(&du); err != nil {

2203 return fmt.Errorf("update disk usage: %v", err)

2204 }

2205 }

2206 }

2207

2208 m.MailboxID = mb.ID

2209 if m.MailboxOrigID == 0 {

2210 m.MailboxOrigID = mb.ID

2211 }

2212 if m.UID == 0 {

2213 m.UID = mb.UIDNext

2214 mb.UIDNext++

2215 }

2216 if m.ModSeq == 0 {

2217 modseq, err := a.NextModSeq(tx)

2218 if err != nil {

2219 return fmt.Errorf("assigning next modseq: %w", err)

2220 }

2221 m.ModSeq = modseq

2222 } else if m.ModSeq < mb.ModSeq {

2223 return fmt.Errorf("cannot deliver message with modseq %d < mailbox modseq %d", m.ModSeq, mb.ModSeq)

2224 }

2225 if m.CreateSeq == 0 {

2226 m.CreateSeq = m.ModSeq

2227 }

2228 mb.ModSeq = m.ModSeq

2229

2230 if m.SaveDate == nil {

2231 now := time.Now()

2232 m.SaveDate = &now

2233 }

2234 if m.Received.IsZero() {

2235 m.Received = time.Now()

2236 }

2237

2238 if len(m.Keywords) > 0 {

2239 mb.Keywords, _ = MergeKeywords(mb.Keywords, m.Keywords)

2240 }

2241

2242 conf, _ := a.Conf()

2243 m.JunkFlagsForMailbox(*mb, conf)

2244

2245 var part *message.Part

2246 if m.ParsedBuf == nil {

2247 mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile.

2248 p, err := message.EnsurePart(log.Logger, false, mr, m.Size)

2249 if err != nil {

2250 log.Infox("parsing delivered message", err, slog.String("parse", ""), slog.Int64("message", m.ID))

2251 // We continue, p is still valid.

2252 }

2253 part = &p

2254 buf, err := json.Marshal(part)

2255 if err != nil {

2256 return fmt.Errorf("marshal parsed message: %w", err)

2257 }

2258 m.ParsedBuf = buf

2259 }

2260

2261 var partTried bool

2262 getPart := func() *message.Part {

2263 if part != nil {

2264 return part

2265 }

2266 if partTried {

2267 return nil

2268 }

2269 partTried = true

2270 var p message.Part

2271 if err := json.Unmarshal(m.ParsedBuf, &p); err != nil {

2272 log.Errorx("unmarshal parsed message, continuing", err, slog.String("parse", ""))

2273 } else {

2274 mr := FileMsgReader(m.MsgPrefix, msgFile)

2275 p.SetReaderAt(mr)

2276 part = &p

2277 }

2278 return part

2279 }

2280

2281 // If we are delivering to the originally intended mailbox, no need to store the mailbox ID again.

2282 if m.MailboxDestinedID != 0 && m.MailboxDestinedID == m.MailboxOrigID {

2283 m.MailboxDestinedID = 0

2284 }

2285

2286 if m.MessageID == "" && m.SubjectBase == "" && getPart() != nil {

2287 m.PrepareThreading(log, part)

2288 }

2289

2290 if !opts.SkipPreview && m.Preview == nil {

2291 if p := getPart(); p != nil {

2292 s, err := p.Preview(log)

2293 if err != nil {

2294 return fmt.Errorf("generating preview: %v", err)

2295 }

2296 m.Preview = &s

2297 }

2298 }

2299

2300 // Assign to thread (if upgrade has completed).

2301 noThreadID := opts.SkipThreads

2302 if m.ThreadID == 0 && !opts.SkipThreads && getPart() != nil {

2303 select {

2304 case <-a.threadsCompleted:

2305 if a.threadsErr != nil {

2306 log.Info("not assigning threads for new delivery, upgrading to threads failed")

2307 noThreadID = true

2308 } else {

2309 if err := assignThread(log, tx, m, part); err != nil {

2310 return fmt.Errorf("assigning thread: %w", err)

2311 }

2312 }

2313 default:

2314 // note: since we have a write transaction to get here, we can't wait for the

2315 // thread upgrade to finish.

2316 // If we don't assign a threadid the upgrade process will do it.

2317 log.Info("not assigning threads for new delivery, upgrading to threads in progress which will assign this message")

2318 noThreadID = true

2319 }

2320 }

2321

2322 if err := tx.Insert(m); err != nil {

2323 return fmt.Errorf("inserting message: %w", err)

2324 }

2325 if !noThreadID && m.ThreadID == 0 {

2326 m.ThreadID = m.ID

2327 if err := tx.Update(m); err != nil {

2328 return fmt.Errorf("updating message for its own thread id: %w", err)

2329 }

2330 }

2331

2332 // todo: perhaps we should match the recipients based on smtp submission and a matching message-id? we now miss the addresses in bcc's if the mail client doesn't save a message that includes the bcc header in the sent mailbox.

2333 if mb.Sent && getPart() != nil && part.Envelope != nil {

2334 e := part.Envelope

2335 sent := e.Date

2336 if sent.IsZero() {

2337 sent = m.Received

2338 }

2339 if sent.IsZero() {

2340 sent = time.Now()

2341 }

2342 addrs := append(append(e.To, e.CC...), e.BCC...)

2343 for _, addr := range addrs {

2344 if addr.User == "" {

2345 // Would trigger error because Recipient.Localpart must be nonzero. todo: we could allow empty localpart in db, and filter by not using FilterNonzero.

2346 log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", slog.Any("address", addr))

2347 continue

2348 }

2349 d, err := dns.ParseDomain(addr.Host)

2350 if err != nil {

2351 log.Debugx("parsing domain in to/cc/bcc address", err, slog.Any("address", addr))

2352 continue

2353 }

2354 lp, err := smtp.ParseLocalpart(addr.User)

2355 if err != nil {

2356 log.Debugx("parsing localpart in to/cc/bcc address", err, slog.Any("address", addr))

2357 continue

2358 }

2359 mr := Recipient{

2360 MessageID: m.ID,

2361 Localpart: lp.String(),

2362 Domain: d.Name(),

2363 OrgDomain: publicsuffix.Lookup(context.TODO(), log.Logger, d).Name(),

2364 Sent: sent,

2365 }

2366 if err := tx.Insert(&mr); err != nil {

2367 return fmt.Errorf("inserting sent message recipients: %w", err)

2368 }

2369 }

2370 }

2371

2372 msgPath := a.MessagePath(m.ID)

2373 msgDir := filepath.Dir(msgPath)

2374 if a.lastMsgDir != msgDir {

2375 os.MkdirAll(msgDir, 0770)

2376 if err := moxio.SyncDir(log, msgDir); err != nil {

2377 return fmt.Errorf("sync message dir: %w", err)

2378 }

2379 a.lastMsgDir = msgDir

2380 }

2381

2382 // Sync file data to disk.

2383 if !opts.SkipSourceFileSync {

2384 if err := msgFile.Sync(); err != nil {

2385 return fmt.Errorf("fsync message file: %w", err)

2386 }

2387 }

2388

2389 if err := moxio.LinkOrCopy(log, msgPath, msgFile.Name(), &moxio.AtReader{R: msgFile}, true); err != nil {

2390 return fmt.Errorf("linking/copying message to new file: %w", err)

2391 }

2392

2393 defer func() {

2394 if rerr != nil {

2395 err := os.Remove(msgPath)

2396 log.Check(err, "removing delivered message file", slog.String("path", msgPath))

2397 }

2398 }()

2399

2400 if !opts.SkipDirSync {

2401 if err := moxio.SyncDir(log, msgDir); err != nil {

2402 return fmt.Errorf("sync directory: %w", err)

2403 }

2404 }

2405

2406 if !opts.SkipTraining && m.NeedsTraining() && a.HasJunkFilter() {

2407 jf, opened, err := a.ensureJunkFilter(context.TODO(), log, opts.JunkFilter)

2408 if err != nil {

2409 return fmt.Errorf("open junk filter: %w", err)

2410 }

2411 defer func() {

2412 if jf != nil && opened {

2413 err := jf.CloseDiscard()

2414 log.Check(err, "closing junk filter without saving")

2415 }

2416 }()

2417

2418 // todo optimize: should let us do the tx.Update of m if needed. we should at least merge it with the common case of setting a thread id. and we should try to merge that with the insert by expliciting getting the next id from bstore.

2419

2420 if err := a.RetrainMessage(context.TODO(), log, tx, jf, m); err != nil {

2421 return fmt.Errorf("training junkfilter: %w", err)

2422 }

2423

2424 if opened {

2425 err := jf.Close()

2426 jf = nil

2427 if err != nil {

2428 return fmt.Errorf("close junk filter: %w", err)

2429 }

2430 }

2431 }

2432

2433 mb.MailboxCounts.Add(m.MailboxCounts())

2434

2435 return nil

2436}

2437

2438// SetPassword saves a new password for this account. This password is used for

2439// IMAP, SMTP (submission) sessions and the HTTP account web page.

2440//

2441// Callers are responsible for checking if the account has NoCustomPassword set.

2442func (a *Account) SetPassword(log mlog.Log, password string) error {

2443 password, err := precis.OpaqueString.String(password)

2444 if err != nil {

2445 return fmt.Errorf(`password not allowed by "precis"`)

2446 }

2447

2448 if len(password) < 8 {

2449 // We actually check for bytes...

2450 return fmt.Errorf("password must be at least 8 characters long")

2451 }

2452

2453 hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)

2454 if err != nil {

2455 return fmt.Errorf("generating password hash: %w", err)

2456 }

2457

2458 err = a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

2459 if _, err := bstore.QueryTx[Password](tx).Delete(); err != nil {

2460 return fmt.Errorf("deleting existing password: %v", err)

2461 }

2462 var pw Password

2463 pw.Hash = string(hash)

2464

2465 // CRAM-MD5 calculates an HMAC-MD5, with the password as key, over a per-attempt

2466 // unique text that includes a timestamp. HMAC performs two hashes. Both times, the

2467 // first block is based on the key/password. We hash those first blocks now, and

2468 // store the hash state in the database. When we actually authenticate, we'll

2469 // complete the HMAC by hashing only the text. We cannot store crypto/hmac's hash,

2470 // because it does not expose its internal state and isn't a BinaryMarshaler.

2471 // ../rfc/2104:121

2472 pw.CRAMMD5.Ipad = md5.New()

2473 pw.CRAMMD5.Opad = md5.New()

2474 key := []byte(password)

2475 if len(key) > 64 {

2476 t := md5.Sum(key)

2477 key = t[:]

2478 }

2479 ipad := make([]byte, md5.BlockSize)

2480 opad := make([]byte, md5.BlockSize)

2481 copy(ipad, key)

2482 copy(opad, key)

2483 for i := range ipad {

2484 ipad[i] ^= 0x36

2485 opad[i] ^= 0x5c

2486 }

2487 pw.CRAMMD5.Ipad.Write(ipad)

2488 pw.CRAMMD5.Opad.Write(opad)

2489

2490 pw.SCRAMSHA1.Salt = scram.MakeRandom()

2491 pw.SCRAMSHA1.Iterations = 2 * 4096

2492 pw.SCRAMSHA1.SaltedPassword = scram.SaltPassword(sha1.New, password, pw.SCRAMSHA1.Salt, pw.SCRAMSHA1.Iterations)

2493

2494 pw.SCRAMSHA256.Salt = scram.MakeRandom()

2495 pw.SCRAMSHA256.Iterations = 4096

2496 pw.SCRAMSHA256.SaltedPassword = scram.SaltPassword(sha256.New, password, pw.SCRAMSHA256.Salt, pw.SCRAMSHA256.Iterations)

2497

2498 if err := tx.Insert(&pw); err != nil {

2499 return fmt.Errorf("inserting new password: %v", err)

2500 }

2501

2502 return sessionRemoveAll(context.TODO(), log, tx, a.Name)

2503 })

2504 if err == nil {

2505 log.Info("new password set for account", slog.String("account", a.Name))

2506 }

2507 return err

2508}

2509

2510// SessionsClear invalidates all (web) login sessions for the account.

2511func (a *Account) SessionsClear(ctx context.Context, log mlog.Log) error {

2512 return a.DB.Write(ctx, func(tx *bstore.Tx) error {

2513 return sessionRemoveAll(ctx, log, tx, a.Name)

2514 })

2515}

2516

2517// Subjectpass returns the signing key for use with subjectpass for the given

2518// email address with canonical localpart.

2519func (a *Account) Subjectpass(email string) (key string, err error) {

2520 return key, a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

2521 v := Subjectpass{Email: email}

2522 err := tx.Get(&v)

2523 if err == nil {

2524 key = v.Key

2525 return nil

2526 }

2527 if !errors.Is(err, bstore.ErrAbsent) {

2528 return fmt.Errorf("get subjectpass key from accounts database: %w", err)

2529 }

2530 key = ""

2531 const chars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"

2532 buf := make([]byte, 16)

2533 if _, err := cryptorand.Read(buf); err != nil {

2534 return err

2535 }

2536 for _, b := range buf {

2537 key += string(chars[int(b)%len(chars)])

2538 }

2539 v.Key = key

2540 return tx.Insert(&v)

2541 })

2542}

2543

2544// Ensure mailbox is present in database, adding records for the mailbox and its

2545// parents if they aren't present.

2546//

2547// If subscribe is true, any mailboxes that were created will also be subscribed to.

2548//

2549// The leaf mailbox is created with special-use flags, taking the flags away from

2550// other mailboxes, and reflecting that in the returned changes.

2551//

2552// Modseq is used, and initialized if 0, for created mailboxes.

2553//

2554// Name must be in normalized form, see CheckMailboxName.

2555//

2556// Caller must hold account wlock.

2557// Caller must propagate changes if any.

2558func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool, specialUse SpecialUse, modseq *ModSeq) (mb Mailbox, changes []Change, rerr error) {

2559 if norm.NFC.String(name) != name {

2560 return Mailbox{}, nil, fmt.Errorf("mailbox name not normalized")

2561 }

2562

2563 // Quick sanity check.

2564 if strings.EqualFold(name, "inbox") && name != "Inbox" {

2565 return Mailbox{}, nil, fmt.Errorf("bad casing for inbox")

2566 }

2567

2568 // Get mailboxes with same name or prefix (parents).

2569 elems := strings.Split(name, "/")

2570 q := bstore.QueryTx[Mailbox](tx)

2571 q.FilterEqual("Expunged", false)

2572 q.FilterFn(func(xmb Mailbox) bool {

2573 t := strings.Split(xmb.Name, "/")

2574 return len(t) <= len(elems) && slices.Equal(t, elems[:len(t)])

2575 })

2576 l, err := q.List()

2577 if err != nil {

2578 return Mailbox{}, nil, fmt.Errorf("list mailboxes: %v", err)

2579 }

2580

2581 mailboxes := map[string]Mailbox{}

2582 for _, xmb := range l {

2583 mailboxes[xmb.Name] = xmb

2584 }

2585

2586 p := ""

2587 var exists bool

2588 var parentID int64

2589 for _, elem := range elems {

2590 if p != "" {

2591 p += "/"

2592 }

2593 p += elem

2594 mb, exists = mailboxes[p]

2595 if exists {

2596 parentID = mb.ID

2597 continue

2598 }

2599 uidval, err := a.NextUIDValidity(tx)

2600 if err != nil {

2601 return Mailbox{}, nil, fmt.Errorf("next uid validity: %v", err)

2602 }

2603 if *modseq == 0 {

2604 *modseq, err = a.NextModSeq(tx)

2605 if err != nil {

2606 return Mailbox{}, nil, fmt.Errorf("next modseq: %v", err)

2607 }

2608 }

2609 mb = Mailbox{

2610 CreateSeq: *modseq,

2611 ModSeq: *modseq,

2612 ParentID: parentID,

2613 Name: p,

2614 UIDValidity: uidval,

2615 UIDNext: 1,

2616 HaveCounts: true,

2617 }

2618 err = tx.Insert(&mb)

2619 if err != nil {

2620 return Mailbox{}, nil, fmt.Errorf("creating new mailbox %q: %v", p, err)

2621 }

2622 parentID = mb.ID

2623

2624 var flags []string

2625 if subscribe {

2626 if tx.Get(&Subscription{p}) != nil {

2627 err := tx.Insert(&Subscription{p})

2628 if err != nil {

2629 return Mailbox{}, nil, fmt.Errorf("subscribing to mailbox %q: %v", p, err)

2630 }

2631 }

2632 flags = []string{`\Subscribed`}

2633 } else if err := tx.Get(&Subscription{p}); err == nil {

2634 flags = []string{`\Subscribed`}

2635 } else if err != bstore.ErrAbsent {

2636 return Mailbox{}, nil, fmt.Errorf("looking up subscription for %q: %v", p, err)

2637 }

2638

2639 changes = append(changes, ChangeAddMailbox{mb, flags})

2640 }

2641

2642 // Clear any special-use flags from existing mailboxes and assign them to this mailbox.

2643 var zeroSpecialUse SpecialUse

2644 if !exists && specialUse != zeroSpecialUse {

2645 var qerr error

2646 clearSpecialUse := func(b bool, fn func(*Mailbox) *bool) {

2647 if !b || qerr != nil {

2648 return

2649 }

2650 qs := bstore.QueryTx[Mailbox](tx)

2651 qs.FilterFn(func(xmb Mailbox) bool {

2652 return *fn(&xmb)

2653 })

2654 xmb, err := qs.Get()

2655 if err == bstore.ErrAbsent {

2656 return

2657 } else if err != nil {

2658 qerr = fmt.Errorf("looking up mailbox with special-use flag: %v", err)

2659 return

2660 }

2661 p := fn(&xmb)

2662 *p = false

2663 xmb.ModSeq = *modseq

2664 if err := tx.Update(&xmb); err != nil {

2665 qerr = fmt.Errorf("clearing special-use flag: %v", err)

2666 } else {

2667 changes = append(changes, xmb.ChangeSpecialUse())

2668 }

2669 }

2670 clearSpecialUse(specialUse.Archive, func(xmb *Mailbox) *bool { return &xmb.Archive })

2671 clearSpecialUse(specialUse.Draft, func(xmb *Mailbox) *bool { return &xmb.Draft })

2672 clearSpecialUse(specialUse.Junk, func(xmb *Mailbox) *bool { return &xmb.Junk })

2673 clearSpecialUse(specialUse.Sent, func(xmb *Mailbox) *bool { return &xmb.Sent })

2674 clearSpecialUse(specialUse.Trash, func(xmb *Mailbox) *bool { return &xmb.Trash })

2675 if qerr != nil {

2676 return Mailbox{}, nil, qerr

2677 }

2678

2679 mb.SpecialUse = specialUse

2680 mb.ModSeq = *modseq

2681 if err := tx.Update(&mb); err != nil {

2682 return Mailbox{}, nil, fmt.Errorf("setting special-use flag for new mailbox: %v", err)

2683 }

2684 changes = append(changes, mb.ChangeSpecialUse())

2685 }

2686 return mb, changes, nil

2687}

2688

2689// MailboxExists checks if mailbox exists.

2690// Caller must hold account rlock.

2691func (a *Account) MailboxExists(tx *bstore.Tx, name string) (bool, error) {

2692 q := bstore.QueryTx[Mailbox](tx)

2693 q.FilterEqual("Expunged", false)

2694 q.FilterEqual("Name", name)

2695 return q.Exists()

2696}

2697

2698// MailboxFind finds a mailbox by name, returning a nil mailbox and nil error if mailbox does not exist.

2699func (a *Account) MailboxFind(tx *bstore.Tx, name string) (*Mailbox, error) {

2700 q := bstore.QueryTx[Mailbox](tx)

2701 q.FilterEqual("Expunged", false)

2702 q.FilterEqual("Name", name)

2703 mb, err := q.Get()

2704 if err == bstore.ErrAbsent {

2705 return nil, nil

2706 }

2707 if err != nil {

2708 return nil, fmt.Errorf("looking up mailbox: %w", err)

2709 }

2710 return &mb, nil

2711}

2712

2713// SubscriptionEnsure ensures a subscription for name exists. The mailbox does not

2714// have to exist. Any parents are not automatically subscribed.

2715// Changes are returned and must be broadcasted by the caller.

2716func (a *Account) SubscriptionEnsure(tx *bstore.Tx, name string) ([]Change, error) {

2717 if err := tx.Get(&Subscription{name}); err == nil {

2718 return nil, nil

2719 }

2720

2721 if err := tx.Insert(&Subscription{name}); err != nil {

2722 return nil, fmt.Errorf("inserting subscription: %w", err)

2723 }

2724

2725 q := bstore.QueryTx[Mailbox](tx)

2726 q.FilterEqual("Expunged", false)

2727 q.FilterEqual("Name", name)

2728 _, err := q.Get()

2729 if err == nil {

2730 return []Change{ChangeAddSubscription{name, nil}}, nil

2731 } else if err != bstore.ErrAbsent {

2732 return nil, fmt.Errorf("looking up mailbox for subscription: %w", err)

2733 }

2734 return []Change{ChangeAddSubscription{name, []string{`\NonExistent`}}}, nil

2735}

2736

2737// MessageRuleset returns the first ruleset (if any) that matches the message

2738// represented by msgPrefix and msgFile, with smtp and validation fields from m.

2739func MessageRuleset(log mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset {

2740 if len(dest.Rulesets) == 0 {

2741 return nil

2742 }

2743

2744 mr := FileMsgReader(msgPrefix, msgFile) // We don't close, it would close the msgFile.

2745 p, err := message.Parse(log.Logger, false, mr)

2746 if err != nil {

2747 log.Errorx("parsing message for evaluating rulesets, continuing with headers", err, slog.String("parse", ""))

2748 // note: part is still set.

2749 }

2750 // todo optimize: only parse header if needed for rulesets. and probably reuse an earlier parsing.

2751 header, err := p.Header()

2752 if err != nil {

2753 log.Errorx("parsing message headers for evaluating rulesets, delivering to default mailbox", err, slog.String("parse", ""))

2754 // todo: reject message?

2755 return nil

2756 }

2757

2758ruleset:

2759 for _, rs := range dest.Rulesets {

2760 if rs.SMTPMailFromRegexpCompiled != nil {

2761 if !rs.SMTPMailFromRegexpCompiled.MatchString(m.MailFrom) {

2762 continue ruleset

2763 }

2764 }

2765 if rs.MsgFromRegexpCompiled != nil {

2766 if m.MsgFromLocalpart == "" && m.MsgFromDomain == "" || !rs.MsgFromRegexpCompiled.MatchString(m.MsgFromLocalpart.String()+"@"+m.MsgFromDomain) {

2767 continue ruleset

2768 }

2769 }

2770

2771 if !rs.VerifiedDNSDomain.IsZero() {

2772 d := rs.VerifiedDNSDomain.Name()

2773 suffix := "." + d

2774 matchDomain := func(s string) bool {

2775 return s == d || strings.HasSuffix(s, suffix)

2776 }

2777 var ok bool

2778 if m.EHLOValidated && matchDomain(m.EHLODomain) {

2779 ok = true

2780 }

2781 if m.MailFromValidated && matchDomain(m.MailFromDomain) {

2782 ok = true

2783 }

2784 for _, d := range m.DKIMDomains {

2785 if matchDomain(d) {

2786 ok = true

2787 break

2788 }

2789 }

2790 if !ok {

2791 continue ruleset

2792 }

2793 }

2794

2795 header:

2796 for _, t := range rs.HeadersRegexpCompiled {

2797 for k, vl := range header {

2798 k = strings.ToLower(k)

2799 if !t[0].MatchString(k) {

2800 continue

2801 }

2802 for _, v := range vl {

2803 v = strings.ToLower(strings.TrimSpace(v))

2804 if t[1].MatchString(v) {

2805 continue header

2806 }

2807 }

2808 }

2809 continue ruleset

2810 }

2811 return &rs

2812 }

2813 return nil

2814}

2815

2816// MessagePath returns the file system path of a message.

2817func (a *Account) MessagePath(messageID int64) string {

2818 return strings.Join(append([]string{a.Dir, "msg"}, messagePathElems(messageID)...), string(filepath.Separator))

2819}

2820

2821// MessageReader opens a message for reading, transparently combining the

2822// message prefix with the original incoming message.

2823func (a *Account) MessageReader(m Message) *MsgReader {

2824 return &MsgReader{prefix: m.MsgPrefix, path: a.MessagePath(m.ID), size: m.Size}

2825}

2826

2827// DeliverDestination delivers an email to dest, based on the configured rulesets.

2828//

2829// Returns ErrOverQuota when account would be over quota after adding message.

2830//

2831// Caller must hold account wlock (mailbox may be created).

2832// Message delivery, possible mailbox creation, and updated mailbox counts are

2833// broadcasted.

2834func (a *Account) DeliverDestination(log mlog.Log, dest config.Destination, m *Message, msgFile *os.File) error {

2835 var mailbox string

2836 rs := MessageRuleset(log, dest, m, m.MsgPrefix, msgFile)

2837 if rs != nil {

2838 mailbox = rs.Mailbox

2839 } else if dest.Mailbox == "" {

2840 mailbox = "Inbox"

2841 } else {

2842 mailbox = dest.Mailbox

2843 }

2844 return a.DeliverMailbox(log, mailbox, m, msgFile)

2845}

2846

2847// DeliverMailbox delivers an email to the specified mailbox.

2848//

2849// Returns ErrOverQuota when account would be over quota after adding message.

2850//

2851// Caller must hold account wlock (mailbox may be created).

2852// Message delivery, possible mailbox creation, and updated mailbox counts are

2853// broadcasted.

2854func (a *Account) DeliverMailbox(log mlog.Log, mailbox string, m *Message, msgFile *os.File) (rerr error) {

2855 var changes []Change

2856

2857 var commit bool

2858 defer func() {

2859 if !commit && m.ID != 0 {

2860 p := a.MessagePath(m.ID)

2861 err := os.Remove(p)

2862 log.Check(err, "remove delivered message file", slog.String("path", p))

2863 m.ID = 0

2864 }

2865 }()

2866

2867 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

2868 mb, chl, err := a.MailboxEnsure(tx, mailbox, true, SpecialUse{}, &m.ModSeq)

2869 if err != nil {

2870 return fmt.Errorf("ensuring mailbox: %w", err)

2871 }

2872 if m.CreateSeq == 0 {

2873 m.CreateSeq = m.ModSeq

2874 }

2875

2876 nmbkeywords := len(mb.Keywords)

2877

2878 if err := a.MessageAdd(log, tx, &mb, m, msgFile, AddOpts{}); err != nil {

2879 return err

2880 }

2881

2882 if err := tx.Update(&mb); err != nil {

2883 return fmt.Errorf("updating mailbox for delivery: %w", err)

2884 }

2885

2886 changes = append(changes, chl...)

2887 changes = append(changes, m.ChangeAddUID(), mb.ChangeCounts())

2888 if nmbkeywords != len(mb.Keywords) {

2889 changes = append(changes, mb.ChangeKeywords())

2890 }

2891 return nil

2892 })

2893 if err != nil {

2894 return err

2895 }

2896 commit = true

2897 BroadcastChanges(a, changes)

2898 return nil

2899}

2900

2901type RemoveOpts struct {

2902 JunkFilter *junk.Filter // If set, this filter is used for training, instead of opening and saving the junk filter.

2903}

2904

2905// MessageRemove markes messages as expunged, updates mailbox counts for the

2906// messages, sets a new modseq on the messages and mailbox, untrains the junk

2907// filter and queues the messages for erasing when the last reference has gone.

2908//

2909// Caller must save the modified mailbox to the database.

2910//

2911// The disk usage is not immediately updated. That will happen when the message

2912// is actually removed from disk.

2913//

2914// The junk filter is untrained for the messages if it was trained.

2915// Useful as optimization when messages are moved and the junk/nonjunk flags do not

2916// change (which can happen due to automatic junk/nonjunk flags for mailboxes).

2917//

2918// An empty list of messages results in an error.

2919//

2920// Caller must broadcast changes.

2921//

2922// Must be called with wlock held.

2923func (a *Account) MessageRemove(log mlog.Log, tx *bstore.Tx, modseq ModSeq, mb *Mailbox, opts RemoveOpts, l ...Message) (chremuids ChangeRemoveUIDs, chmbc ChangeMailboxCounts, rerr error) {

2924 if len(l) == 0 {

2925 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("must expunge at least one message")

2926 }

2927

2928 mb.ModSeq = modseq

2929

2930 // Remove any message recipients.

2931 anyIDs := make([]any, len(l))

2932 for i, m := range l {

2933 anyIDs[i] = m.ID

2934 }

2935 qmr := bstore.QueryTx[Recipient](tx)

2936 qmr.FilterEqual("MessageID", anyIDs...)

2937 if _, err := qmr.Delete(); err != nil {

2938 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("deleting message recipients for messages: %w", err)

2939 }

2940

2941 // Loaded lazily.

2942 jf := opts.JunkFilter

2943

2944 // Mark messages expunged.

2945 ids := make([]int64, 0, len(l))

2946 uids := make([]UID, 0, len(l))

2947 for _, m := range l {

2948 ids = append(ids, m.ID)

2949 uids = append(uids, m.UID)

2950

2951 if m.Expunged {

2952 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("message %d is already expunged", m.ID)

2953 }

2954

2955 mb.Sub(m.MailboxCounts())

2956

2957 m.ModSeq = modseq

2958 m.Expunged = true

2959 m.Junk = false

2960 m.Notjunk = false

2961

2962 if err := tx.Update(&m); err != nil {

2963 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("marking message %d expunged: %v", m.ID, err)

2964 }

2965

2966 // Ensure message gets erased in future.

2967 if err := tx.Insert(&MessageErase{m.ID, false}); err != nil {

2968 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("inserting message erase %d : %v", m.ID, err)

2969 }

2970

2971 if m.TrainedJunk == nil || !a.HasJunkFilter() {

2972 continue

2973 }

2974 // Untrain, as needed by updated flags Junk/Notjunk to false.

2975 if jf == nil {

2976 var err error

2977 jf, _, err = a.OpenJunkFilter(context.TODO(), log)

2978 if err != nil {

2979 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("open junk filter: %v", err)

2980 }

2981 defer func() {

2982 err := jf.Close()

2983 if rerr == nil {

2984 rerr = err

2985 } else {

2986 log.Check(err, "closing junk filter")

2987 }

2988 }()

2989 }

2990 if err := a.RetrainMessage(context.TODO(), log, tx, jf, &m); err != nil {

2991 return ChangeRemoveUIDs{}, ChangeMailboxCounts{}, fmt.Errorf("retraining expunged messages: %w", err)

2992 }

2993 }

2994

2995 return ChangeRemoveUIDs{mb.ID, uids, modseq, ids}, mb.ChangeCounts(), nil

2996}

2997

2998// TidyRejectsMailbox removes old reject emails, and returns whether there is space for a new delivery.

2999//

3000// The changed mailbox is saved to the database.

3001//

3002// Caller most hold account wlock.

3003// Caller must broadcast changes.

3004func (a *Account) TidyRejectsMailbox(log mlog.Log, tx *bstore.Tx, mbRej *Mailbox) (changes []Change, hasSpace bool, rerr error) {

3005 // Gather old messages to expunge.

3006 old := time.Now().Add(-14 * 24 * time.Hour)

3007 qdel := bstore.QueryTx[Message](tx)

3008 qdel.FilterNonzero(Message{MailboxID: mbRej.ID})

3009 qdel.FilterEqual("Expunged", false)

3010 qdel.FilterLess("Received", old)

3011 qdel.SortAsc("UID")

3012 expunge, err := qdel.List()

3013 if err != nil {

3014 return nil, false, fmt.Errorf("listing old messages: %w", err)

3015 }

3016

3017 if len(expunge) > 0 {

3018 modseq, err := a.NextModSeq(tx)

3019 if err != nil {

3020 return nil, false, fmt.Errorf("next mod seq: %v", err)

3021 }

3022

3023 chremuids, chmbcounts, err := a.MessageRemove(log, tx, modseq, mbRej, RemoveOpts{}, expunge...)

3024 if err != nil {

3025 return nil, false, fmt.Errorf("removing messages: %w", err)

3026 }

3027 if err := tx.Update(mbRej); err != nil {

3028 return nil, false, fmt.Errorf("updating mailbox: %v", err)

3029 }

3030 changes = append(changes, chremuids, chmbcounts)

3031 }

3032

3033 // We allow up to n messages.

3034 qcount := bstore.QueryTx[Message](tx)

3035 qcount.FilterNonzero(Message{MailboxID: mbRej.ID})

3036 qcount.FilterEqual("Expunged", false)

3037 qcount.Limit(1000)

3038 n, err := qcount.Count()

3039 if err != nil {

3040 return nil, false, fmt.Errorf("counting rejects: %w", err)

3041 }

3042 hasSpace = n < 1000

3043

3044 return changes, hasSpace, nil

3045}

3046

3047// RejectsRemove removes a message from the rejects mailbox if present.

3048//

3049// Caller most hold account wlock.

3050// Changes are broadcasted.

3051func (a *Account) RejectsRemove(log mlog.Log, rejectsMailbox, messageID string) error {

3052 var changes []Change

3053

3054 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

3055 mb, err := a.MailboxFind(tx, rejectsMailbox)

3056 if err != nil {

3057 return fmt.Errorf("finding mailbox: %w", err)

3058 }

3059 if mb == nil {

3060 return nil

3061 }

3062

3063 q := bstore.QueryTx[Message](tx)

3064 q.FilterNonzero(Message{MailboxID: mb.ID, MessageID: messageID})

3065 q.FilterEqual("Expunged", false)

3066 expunge, err := q.List()

3067 if err != nil {

3068 return fmt.Errorf("listing messages to remove: %w", err)

3069 }

3070

3071 if len(expunge) == 0 {

3072 return nil

3073 }

3074

3075 modseq, err := a.NextModSeq(tx)

3076 if err != nil {

3077 return fmt.Errorf("get next mod seq: %v", err)

3078 }

3079

3080 chremuids, chmbcounts, err := a.MessageRemove(log, tx, modseq, mb, RemoveOpts{}, expunge...)

3081 if err != nil {

3082 return fmt.Errorf("removing messages: %w", err)

3083 }

3084 changes = append(changes, chremuids, chmbcounts)

3085

3086 if err := tx.Update(mb); err != nil {

3087 return fmt.Errorf("saving mailbox: %w", err)

3088 }

3089

3090 return nil

3091 })

3092 if err != nil {

3093 return err

3094 }

3095

3096 BroadcastChanges(a, changes)

3097

3098 return nil

3099}

3100

3101// AddMessageSize adjusts the DiskUsage.MessageSize by size.

3102func (a *Account) AddMessageSize(log mlog.Log, tx *bstore.Tx, size int64) error {

3103 du := DiskUsage{ID: 1}

3104 if err := tx.Get(&du); err != nil {

3105 return fmt.Errorf("get diskusage: %v", err)

3106 }

3107 du.MessageSize += size

3108 if du.MessageSize < 0 {

3109 log.Error("negative total message size", slog.Int64("delta", size), slog.Int64("newtotalsize", du.MessageSize))

3110 }

3111 if err := tx.Update(&du); err != nil {

3112 return fmt.Errorf("update total message size: %v", err)

3113 }

3114 return nil

3115}

3116

3117// QuotaMessageSize returns the effective maximum total message size for an

3118// account. Returns 0 if there is no maximum.

3119func (a *Account) QuotaMessageSize() int64 {

3120 conf, _ := a.Conf()

3121 size := conf.QuotaMessageSize

3122 if size == 0 {

3123 size = mox.Conf.Static.QuotaMessageSize

3124 }

3125 if size < 0 {

3126 size = 0

3127 }

3128 return size

3129}

3130

3131// CanAddMessageSize checks if a message of size bytes can be added, depending on

3132// total message size and configured quota for account.

3133func (a *Account) CanAddMessageSize(tx *bstore.Tx, size int64) (ok bool, maxSize int64, err error) {

3134 maxSize = a.QuotaMessageSize()

3135 if maxSize <= 0 {

3136 return true, 0, nil

3137 }

3138

3139 du := DiskUsage{ID: 1}

3140 if err := tx.Get(&du); err != nil {

3141 return false, maxSize, fmt.Errorf("get diskusage: %v", err)

3142 }

3143 return du.MessageSize+size <= maxSize, maxSize, nil

3144}

3145

3146// We keep a cache of recent successful authentications, so we don't have to bcrypt successful calls each time.

3147var authCache = struct {

3148 sync.Mutex

3149 success map[authKey]string

3150}{

3151 success: map[authKey]string{},

3152}

3153

3154type authKey struct {

3155 email, hash string

3156}

3157

3158// StartAuthCache starts a goroutine that regularly clears the auth cache.

3159func StartAuthCache() {

3160 go manageAuthCache()

3161}

3162

3163func manageAuthCache() {

3164 for {

3165 authCache.Lock()

3166 authCache.success = map[authKey]string{}

3167 authCache.Unlock()

3168 time.Sleep(15 * time.Minute)

3169 }

3170}

3171

3172// OpenEmailAuth opens an account given an email address and password.

3173//

3174// The email address may contain a catchall separator.

3175// For invalid credentials, a nil account is returned, but accName may be

3176// non-empty.

3177func OpenEmailAuth(log mlog.Log, email string, password string, checkLoginDisabled bool) (racc *Account, raccName string, rerr error) {

3178 // We check for LoginDisabled after verifying the password. Otherwise users can get

3179 // messages about the account being disabled without knowing the password.

3180 acc, accName, _, err := OpenEmail(log, email, false)

3181 if err != nil {

3182 return nil, "", err

3183 }

3184

3185 defer func() {

3186 if rerr != nil {

3187 err := acc.Close()

3188 log.Check(err, "closing account after open auth failure")

3189 acc = nil

3190 }

3191 }()

3192

3193 password, err = precis.OpaqueString.String(password)

3194 if err != nil {

3195 return nil, "", ErrUnknownCredentials

3196 }

3197

3198 pw, err := bstore.QueryDB[Password](context.TODO(), acc.DB).Get()

3199 if err != nil {

3200 if err == bstore.ErrAbsent {

3201 return nil, "", ErrUnknownCredentials

3202 }

3203 return nil, "", fmt.Errorf("looking up password: %v", err)

3204 }

3205 authCache.Lock()

3206 ok := len(password) >= 8 && authCache.success[authKey{email, pw.Hash}] == password

3207 authCache.Unlock()

3208 if !ok {

3209 if err := bcrypt.CompareHashAndPassword([]byte(pw.Hash), []byte(password)); err != nil {

3210 return nil, "", ErrUnknownCredentials

3211 }

3212 }

3213 if checkLoginDisabled {

3214 conf, aok := acc.Conf()

3215 if !aok {

3216 return nil, "", fmt.Errorf("cannot find config for account")

3217 } else if conf.LoginDisabled != "" {

3218 return nil, "", fmt.Errorf("%w: %s", ErrLoginDisabled, conf.LoginDisabled)

3219 }

3220 }

3221 authCache.Lock()

3222 authCache.success[authKey{email, pw.Hash}] = password

3223 authCache.Unlock()

3224 return acc, accName, nil

3225}

3226

3227// OpenEmail opens an account given an email address.

3228//

3229// The email address may contain a catchall separator.

3230//

3231// Returns account on success, may return non-empty account name even on error.

3232func OpenEmail(log mlog.Log, email string, checkLoginDisabled bool) (*Account, string, config.Destination, error) {

3233 addr, err := smtp.ParseAddress(email)

3234 if err != nil {

3235 return nil, "", config.Destination{}, fmt.Errorf("%w: %v", ErrUnknownCredentials, err)

3236 }

3237 accountName, _, _, dest, err := mox.LookupAddress(addr.Localpart, addr.Domain, false, false, false)

3238 if err != nil && (errors.Is(err, mox.ErrAddressNotFound) || errors.Is(err, mox.ErrDomainNotFound)) {

3239 return nil, accountName, config.Destination{}, ErrUnknownCredentials

3240 } else if err != nil {

3241 return nil, accountName, config.Destination{}, fmt.Errorf("looking up address: %v", err)

3242 }

3243 acc, err := OpenAccount(log, accountName, checkLoginDisabled)

3244 if err != nil {

3245 return nil, accountName, config.Destination{}, err

3246 }

3247 return acc, accountName, dest, nil

3248}

3249

3250// We store max 1<<shift files in each subdir of an account "msg" directory.

3251// Defaults to 1 for easy use in tests. Set to 13, for 8k message files, in main

3252// for normal operation.

3253var msgFilesPerDirShift = 1

3254var msgFilesPerDir int64 = 1 << msgFilesPerDirShift

3255

3256func MsgFilesPerDirShiftSet(shift int) {

3257 msgFilesPerDirShift = shift

3258 msgFilesPerDir = 1 << shift

3259}

3260

3261// 64 characters, must be power of 2 for MessagePath

3262const msgDirChars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ-_"

3263

3264// MessagePath returns the filename of the on-disk filename, relative to the

3265// containing directory such as <account>/msg or queue.

3266// Returns names like "AB/1".

3267func MessagePath(messageID int64) string {

3268 return strings.Join(messagePathElems(messageID), string(filepath.Separator))

3269}

3270

3271// messagePathElems returns the elems, for a single join without intermediate

3272// string allocations.

3273func messagePathElems(messageID int64) []string {

3274 v := messageID >> msgFilesPerDirShift

3275 dir := ""

3276 for {

3277 dir += string(msgDirChars[int(v)&(len(msgDirChars)-1)])

3278 v >>= 6

3279 if v == 0 {

3280 break

3281 }

3282 }

3283 return []string{dir, strconv.FormatInt(messageID, 10)}

3284}

3285

3286// Set returns a copy of f, with each flag that is true in mask set to the

3287// value from flags.

3288func (f Flags) Set(mask, flags Flags) Flags {

3289 set := func(d *bool, m, v bool) {

3290 if m {

3291 *d = v

3292 }

3293 }

3294 r := f

3295 set(&r.Seen, mask.Seen, flags.Seen)

3296 set(&r.Answered, mask.Answered, flags.Answered)

3297 set(&r.Flagged, mask.Flagged, flags.Flagged)

3298 set(&r.Forwarded, mask.Forwarded, flags.Forwarded)

3299 set(&r.Junk, mask.Junk, flags.Junk)

3300 set(&r.Notjunk, mask.Notjunk, flags.Notjunk)

3301 set(&r.Deleted, mask.Deleted, flags.Deleted)

3302 set(&r.Draft, mask.Draft, flags.Draft)

3303 set(&r.Phishing, mask.Phishing, flags.Phishing)

3304 set(&r.MDNSent, mask.MDNSent, flags.MDNSent)

3305 return r

3306}

3307

3308// Changed returns a mask of flags that have been between f and other.

3309func (f Flags) Changed(other Flags) (mask Flags) {

3310 mask.Seen = f.Seen != other.Seen

3311 mask.Answered = f.Answered != other.Answered

3312 mask.Flagged = f.Flagged != other.Flagged

3313 mask.Forwarded = f.Forwarded != other.Forwarded

3314 mask.Junk = f.Junk != other.Junk

3315 mask.Notjunk = f.Notjunk != other.Notjunk

3316 mask.Deleted = f.Deleted != other.Deleted

3317 mask.Draft = f.Draft != other.Draft

3318 mask.Phishing = f.Phishing != other.Phishing

3319 mask.MDNSent = f.MDNSent != other.MDNSent

3320 return

3321}

3322

3323// Strings returns the flags that are set in their string form.

3324func (f Flags) Strings() []string {

3325 fields := []struct {

3326 word string

3327 have bool

3328 }{

3329 {`$forwarded`, f.Forwarded},

3330 {`$junk`, f.Junk},

3331 {`$mdnsent`, f.MDNSent},

3332 {`$notjunk`, f.Notjunk},

3333 {`$phishing`, f.Phishing},

3334 {`\answered`, f.Answered},

3335 {`\deleted`, f.Deleted},

3336 {`\draft`, f.Draft},

3337 {`\flagged`, f.Flagged},

3338 {`\seen`, f.Seen},

3339 }

3340 var l []string

3341 for _, fh := range fields {

3342 if fh.have {

3343 l = append(l, fh.word)

3344 }

3345 }

3346 return l

3347}

3348

3349var systemWellKnownFlags = map[string]bool{

3350 `\answered`: true,

3351 `\flagged`: true,

3352 `\deleted`: true,

3353 `\seen`: true,

3354 `\draft`: true,

3355 `$junk`: true,

3356 `$notjunk`: true,

3357 `$forwarded`: true,

3358 `$phishing`: true,

3359 `$mdnsent`: true,

3360}

3361

3362// ParseFlagsKeywords parses a list of textual flags into system/known flags, and

3363// other keywords. Keywords are lower-cased and sorted and check for valid syntax.

3364func ParseFlagsKeywords(l []string) (flags Flags, keywords []string, rerr error) {

3365 fields := map[string]*bool{

3366 `\answered`: &flags.Answered,

3367 `\flagged`: &flags.Flagged,

3368 `\deleted`: &flags.Deleted,

3369 `\seen`: &flags.Seen,

3370 `\draft`: &flags.Draft,

3371 `$junk`: &flags.Junk,

3372 `$notjunk`: &flags.Notjunk,

3373 `$forwarded`: &flags.Forwarded,

3374 `$phishing`: &flags.Phishing,

3375 `$mdnsent`: &flags.MDNSent,

3376 }

3377 seen := map[string]bool{}

3378 for _, f := range l {

3379 f = strings.ToLower(f)

3380 if field, ok := fields[f]; ok {

3381 *field = true

3382 } else if seen[f] {

3383 if mox.Pedantic {

3384 return Flags{}, nil, fmt.Errorf("duplicate keyword %s", f)

3385 }

3386 } else {

3387 if err := CheckKeyword(f); err != nil {

3388 return Flags{}, nil, fmt.Errorf("invalid keyword %s", f)

3389 }

3390 keywords = append(keywords, f)

3391 seen[f] = true

3392 }

3393 }

3394 sort.Strings(keywords)

3395 return flags, keywords, nil

3396}

3397

3398// RemoveKeywords removes keywords from l, returning whether any modifications were

3399// made, and a slice, a new slice in case of modifications. Keywords must have been

3400// validated earlier, e.g. through ParseFlagKeywords or CheckKeyword. Should only

3401// be used with valid keywords, not with system flags like \Seen.

3402func RemoveKeywords(l, remove []string) ([]string, bool) {

3403 var copied bool

3404 var changed bool

3405 for _, k := range remove {

3406 if i := slices.Index(l, k); i >= 0 {

3407 if !copied {

3408 l = slices.Clone(l)

3409 copied = true

3410 }

3411 copy(l[i:], l[i+1:])

3412 l = l[:len(l)-1]

3413 changed = true

3414 }

3415 }

3416 return l, changed

3417}

3418

3419// MergeKeywords adds keywords from add into l, returning whether it added any

3420// keyword, and the slice with keywords, a new slice if modifications were made.

3421// Keywords are only added if they aren't already present. Should only be used with

3422// keywords, not with system flags like \Seen.

3423func MergeKeywords(l, add []string) ([]string, bool) {

3424 var copied bool

3425 var changed bool

3426 for _, k := range add {

3427 if !slices.Contains(l, k) {

3428 if !copied {

3429 l = slices.Clone(l)

3430 copied = true

3431 }

3432 l = append(l, k)

3433 changed = true

3434 }

3435 }

3436 if changed {

3437 sort.Strings(l)

3438 }

3439 return l, changed

3440}

3441

3442// CheckKeyword returns an error if kw is not a valid keyword. Kw should

3443// already be in lower-case.

3444func CheckKeyword(kw string) error {

3445 if kw == "" {

3446 return fmt.Errorf("keyword cannot be empty")

3447 }

3448 if systemWellKnownFlags[kw] {

3449 return fmt.Errorf("cannot use well-known flag as keyword")

3450 }

3451 for _, c := range kw {

3452 // ../rfc/9051:6334

3453 if c <= ' ' || c > 0x7e || c >= 'A' && c <= 'Z' || strings.ContainsRune(`(){%*"\]`, c) {

3454 return errors.New(`not a valid keyword, must be lower-case ascii without spaces and without any of these characters: (){%*"\]`)

3455 }

3456 }

3457 return nil

3458}

3459

3460// SendLimitReached checks whether sending a message to recipients would reach

3461// the limit of outgoing messages for the account. If so, the message should

3462// not be sent. If the returned numbers are >= 0, the limit was reached and the

3463// values are the configured limits.

3464//

3465// To limit damage to the internet and our reputation in case of account

3466// compromise, we limit the max number of messages sent in a 24 hour window, both

3467// total number of messages and number of first-time recipients.

3468func (a *Account) SendLimitReached(tx *bstore.Tx, recipients []smtp.Path) (msglimit, rcptlimit int, rerr error) {

3469 conf, _ := a.Conf()

3470 msgmax := conf.MaxOutgoingMessagesPerDay

3471 if msgmax == 0 {

3472 // For human senders, 1000 recipients in a day is quite a lot.

3473 msgmax = 1000

3474 }

3475 rcptmax := conf.MaxFirstTimeRecipientsPerDay

3476 if rcptmax == 0 {

3477 // Human senders may address a new human-sized list of people once in a while. In

3478 // case of a compromise, a spammer will probably try to send to many new addresses.

3479 rcptmax = 200

3480 }

3481

3482 rcpts := map[string]time.Time{}

3483 n := 0

3484 err := bstore.QueryTx[Outgoing](tx).FilterGreater("Submitted", time.Now().Add(-24*time.Hour)).ForEach(func(o Outgoing) error {

3485 n++

3486 if rcpts[o.Recipient].IsZero() || o.Submitted.Before(rcpts[o.Recipient]) {

3487 rcpts[o.Recipient] = o.Submitted

3488 }

3489 return nil

3490 })

3491 if err != nil {

3492 return -1, -1, fmt.Errorf("querying message recipients in past 24h: %w", err)

3493 }

3494 if n+len(recipients) > msgmax {

3495 return msgmax, -1, nil

3496 }

3497

3498 // Only check if max first-time recipients is reached if there are enough messages

3499 // to trigger the limit.

3500 if n+len(recipients) < rcptmax {

3501 return -1, -1, nil

3502 }

3503

3504 isFirstTime := func(rcpt string, before time.Time) (bool, error) {

3505 exists, err := bstore.QueryTx[Outgoing](tx).FilterNonzero(Outgoing{Recipient: rcpt}).FilterLess("Submitted", before).Exists()

3506 return !exists, err

3507 }

3508

3509 firsttime := 0

3510 now := time.Now()

3511 for _, r := range recipients {

3512 if first, err := isFirstTime(r.XString(true), now); err != nil {

3513 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

3514 } else if first {

3515 firsttime++

3516 }

3517 }

3518 for r, t := range rcpts {

3519 if first, err := isFirstTime(r, t); err != nil {

3520 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

3521 } else if first {

3522 firsttime++

3523 }

3524 }

3525 if firsttime > rcptmax {

3526 return -1, rcptmax, nil

3527 }

3528 return -1, -1, nil

3529}

3530

3531var ErrMailboxExpunged = errors.New("mailbox was deleted")

3532

3533// MailboxID gets a mailbox by ID.

3534//

3535// Returns bstore.ErrAbsent if the mailbox does not exist.

3536// Returns ErrMailboxExpunged if the mailbox is expunged.

3537func MailboxID(tx *bstore.Tx, id int64) (Mailbox, error) {

3538 mb := Mailbox{ID: id}

3539 err := tx.Get(&mb)

3540 if err == nil && mb.Expunged {

3541 return Mailbox{}, ErrMailboxExpunged

3542 }

3543 return mb, err

3544}

3545

3546// MailboxCreate creates a new mailbox, including any missing parent mailboxes,

3547// the total list of created mailboxes is returned in created. On success, if

3548// exists is false and rerr nil, the changes must be broadcasted by the caller.

3549//

3550// The mailbox is created with special-use flags, with those flags taken away from

3551// other mailboxes if they have them, reflected in the returned changes.

3552//

3553// Name must be in normalized form, see CheckMailboxName.

3554func (a *Account) MailboxCreate(tx *bstore.Tx, name string, specialUse SpecialUse) (nmb Mailbox, changes []Change, created []string, exists bool, rerr error) {

3555 elems := strings.Split(name, "/")

3556 var p string

3557 var modseq ModSeq

3558 for i, elem := range elems {

3559 if i > 0 {

3560 p += "/"

3561 }

3562 p += elem

3563 exists, err := a.MailboxExists(tx, p)

3564 if err != nil {

3565 return Mailbox{}, nil, nil, false, fmt.Errorf("checking if mailbox exists")

3566 }

3567 if exists {

3568 if i == len(elems)-1 {

3569 return Mailbox{}, nil, nil, true, fmt.Errorf("mailbox already exists")

3570 }

3571 continue

3572 }

3573 mb, nchanges, err := a.MailboxEnsure(tx, p, true, specialUse, &modseq)

3574 if err != nil {

3575 return Mailbox{}, nil, nil, false, fmt.Errorf("ensuring mailbox exists: %v", err)

3576 }

3577 nmb = mb

3578 changes = append(changes, nchanges...)

3579 created = append(created, p)

3580 }

3581 return nmb, changes, created, false, nil

3582}

3583

3584// MailboxRename renames mailbox mbsrc to dst, including children of mbsrc, and

3585// adds missing parents for dst.

3586//

3587// Name must be in normalized form, see CheckMailboxName, and cannot be Inbox.

3588func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc *Mailbox, dst string, modseq *ModSeq) (changes []Change, isInbox, alreadyExists bool, rerr error) {

3589 if mbsrc.Name == "Inbox" || dst == "Inbox" {

3590 return nil, true, false, fmt.Errorf("inbox cannot be renamed")

3591 }

3592

3593 // Check if destination mailbox already exists.

3594 if exists, err := a.MailboxExists(tx, dst); err != nil {

3595 return nil, false, false, fmt.Errorf("checking if destination mailbox exists: %v", err)

3596 } else if exists {

3597 return nil, false, true, fmt.Errorf("destination mailbox already exists")

3598 }

3599

3600 if *modseq == 0 {

3601 var err error

3602 *modseq, err = a.NextModSeq(tx)

3603 if err != nil {

3604 return nil, false, false, fmt.Errorf("get next modseq: %v", err)

3605 }

3606 }

3607

3608 origName := mbsrc.Name

3609

3610 // Move children to their new name.

3611 srcPrefix := mbsrc.Name + "/"

3612 q := bstore.QueryTx[Mailbox](tx)

3613 q.FilterEqual("Expunged", false)

3614 q.FilterFn(func(mb Mailbox) bool {

3615 return strings.HasPrefix(mb.Name, srcPrefix)

3616 })

3617 q.SortDesc("Name") // From leaf towards dst.

3618 kids, err := q.List()

3619 if err != nil {

3620 return nil, false, false, fmt.Errorf("listing child mailboxes")

3621 }

3622

3623 // Rename children, from leaf towards dst (because sorted reverse by name).

3624 for _, mb := range kids {

3625 nname := dst + "/" + mb.Name[len(mbsrc.Name)+1:]

3626 var flags []string

3627 if err := tx.Get(&Subscription{nname}); err == nil {

3628 flags = []string{`\Subscribed`}

3629 } else if err != bstore.ErrAbsent {

3630 return nil, false, false, fmt.Errorf("look up subscription for new name of child %q: %v", nname, err)

3631 }

3632 // Leaf is first.

3633 changes = append(changes, ChangeRenameMailbox{mb.ID, mb.Name, nname, flags, *modseq})

3634

3635 mb.Name = nname

3636 mb.ModSeq = *modseq

3637 if err := tx.Update(&mb); err != nil {

3638 return nil, false, false, fmt.Errorf("rename child mailbox %q: %v", mb.Name, err)

3639 }

3640 }

3641

3642 // Move name out of the way. We may have to create it again, as our new parent.

3643 var flags []string

3644 if err := tx.Get(&Subscription{dst}); err == nil {

3645 flags = []string{`\Subscribed`}

3646 } else if err != bstore.ErrAbsent {

3647 return nil, false, false, fmt.Errorf("look up subscription for new name %q: %v", dst, err)

3648 }

3649 changes = append(changes, ChangeRenameMailbox{mbsrc.ID, mbsrc.Name, dst, flags, *modseq})

3650 mbsrc.ModSeq = *modseq

3651 mbsrc.Name = dst

3652 if err := tx.Update(mbsrc); err != nil {

3653 return nil, false, false, fmt.Errorf("rename mailbox: %v", err)

3654 }

3655

3656 // Add any missing parents for the new name. A mailbox may have been renamed from

3657 // a/b to a/b/x/y, and we'll have to add a new "a" and a/b.

3658 t := strings.Split(dst, "/")

3659 t = t[:len(t)-1]

3660 var parent Mailbox

3661 var parentChanges []Change

3662 for i := range t {

3663 s := strings.Join(t[:i+1], "/")

3664 q := bstore.QueryTx[Mailbox](tx)

3665 q.FilterEqual("Expunged", false)

3666 q.FilterNonzero(Mailbox{Name: s})

3667 pmb, err := q.Get()

3668 if err == nil {

3669 parent = pmb

3670 continue

3671 } else if err != bstore.ErrAbsent {

3672 return nil, false, false, fmt.Errorf("lookup destination parent mailbox %q: %v", s, err)

3673 }

3674

3675 uidval, err := a.NextUIDValidity(tx)

3676 if err != nil {

3677 return nil, false, false, fmt.Errorf("next uid validity: %v", err)

3678 }

3679 parent = Mailbox{

3680 CreateSeq: *modseq,

3681 ModSeq: *modseq,

3682 ParentID: parent.ID,

3683 Name: s,

3684 UIDValidity: uidval,

3685 UIDNext: 1,

3686 HaveCounts: true,

3687 }

3688 if err := tx.Insert(&parent); err != nil {

3689 return nil, false, false, fmt.Errorf("inserting destination parent mailbox %q: %v", s, err)

3690 }

3691

3692 var flags []string

3693 if err := tx.Get(&Subscription{parent.Name}); err == nil {

3694 flags = []string{`\Subscribed`}

3695 } else if err != bstore.ErrAbsent {

3696 return nil, false, false, fmt.Errorf("look up subscription for new parent %q: %v", parent.Name, err)

3697 }

3698 parentChanges = append(parentChanges, ChangeAddMailbox{parent, flags})

3699 }

3700

3701 mbsrc.ParentID = parent.ID

3702 if err := tx.Update(mbsrc); err != nil {

3703 return nil, false, false, fmt.Errorf("set parent id on rename mailbox: %v", err)

3704 }

3705

3706 // If we were moved from a/b to a/b/x, we mention the creation of a/b after we mentioned the rename.

3707 if strings.HasPrefix(dst, origName+"/") {

3708 changes = append(changes, parentChanges...)

3709 } else {

3710 changes = slices.Concat(parentChanges, changes)

3711 }

3712

3713 return changes, false, false, nil

3714}

3715

3716// MailboxDelete marks a mailbox as deleted, including its annotations. If it has

3717// children, the return value indicates that and an error is returned.

3718//

3719// Caller should broadcast the changes (deleting all messages in the mailbox and

3720// deleting the mailbox itself).

3721func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx, mb *Mailbox) (changes []Change, hasChildren bool, rerr error) {

3722 // Look for existence of child mailboxes. There is a lot of text in the IMAP RFCs about

3723 // NoInferior and NoSelect. We just require only leaf mailboxes are deleted.

3724 qmb := bstore.QueryTx[Mailbox](tx)

3725 qmb.FilterEqual("Expunged", false)

3726 mbprefix := mb.Name + "/"

3727 qmb.FilterFn(func(xmb Mailbox) bool {

3728 return strings.HasPrefix(xmb.Name, mbprefix)

3729 })

3730 if childExists, err := qmb.Exists(); err != nil {

3731 return nil, false, fmt.Errorf("checking if mailbox has child: %v", err)

3732 } else if childExists {

3733 return nil, true, fmt.Errorf("mailbox has a child, only leaf mailboxes can be deleted")

3734 }

3735

3736 modseq, err := a.NextModSeq(tx)

3737 if err != nil {

3738 return nil, false, fmt.Errorf("get next modseq: %v", err)

3739 }

3740

3741 qm := bstore.QueryTx[Message](tx)

3742 qm.FilterNonzero(Message{MailboxID: mb.ID})

3743 qm.FilterEqual("Expunged", false)

3744 qm.SortAsc("UID")

3745 l, err := qm.List()

3746 if err != nil {

3747 return nil, false, fmt.Errorf("listing messages in mailbox to remove; %v", err)

3748 }

3749

3750 if len(l) > 0 {

3751 chrem, _, err := a.MessageRemove(log, tx, modseq, mb, RemoveOpts{}, l...)

3752 if err != nil {

3753 return nil, false, fmt.Errorf("marking messages removed: %v", err)

3754 }

3755 changes = append(changes, chrem)

3756 }

3757

3758 // Marking metadata annotations deleted. ../rfc/5464:373

3759 qa := bstore.QueryTx[Annotation](tx)

3760 qa.FilterNonzero(Annotation{MailboxID: mb.ID})

3761 qa.FilterEqual("Expunged", false)

3762 if _, err := qa.UpdateFields(map[string]any{"ModSeq": modseq, "Expunged": true, "IsString": false, "Value": []byte(nil)}); err != nil {

3763 return nil, false, fmt.Errorf("removing annotations for mailbox: %v", err)

3764 }

3765 // Not sending changes about annotations on this mailbox, since the entire mailbox

3766 // is being removed.

3767

3768 mb.ModSeq = modseq

3769 mb.Expunged = true

3770 mb.SpecialUse = SpecialUse{}

3771

3772 if err := tx.Update(mb); err != nil {

3773 return nil, false, fmt.Errorf("updating mailbox: %v", err)

3774 }

3775

3776 changes = append(changes, mb.ChangeRemoveMailbox())

3777 return changes, false, nil

3778}

3779

3780// CheckMailboxName checks if name is valid, returning an INBOX-normalized name.

3781// I.e. it changes various casings of INBOX and INBOX/* to Inbox and Inbox/*.

3782// Name is invalid if it contains leading/trailing/double slashes, or when it isn't

3783// unicode-normalized, or when empty or has special characters.

3784//

3785// If name is the inbox, and allowInbox is false, this is indicated with the isInbox return parameter.

3786// For that case, and for other invalid names, an error is returned.

3787func CheckMailboxName(name string, allowInbox bool) (normalizedName string, isInbox bool, rerr error) {

3788 t := strings.Split(name, "/")

3789 if strings.EqualFold(t[0], "inbox") {

3790 if len(name) == len("inbox") && !allowInbox {

3791 return "", true, fmt.Errorf("special mailbox name Inbox not allowed")

3792 }

3793 name = "Inbox" + name[len("Inbox"):]

3794 }

3795

3796 if norm.NFC.String(name) != name {

3797 return "", false, errors.New("non-unicode-normalized mailbox names not allowed")

3798 }

3799

3800 for _, e := range t {

3801 switch e {

3802 case "":

3803 return "", false, errors.New("empty mailbox name")

3804 case ".":

3805 return "", false, errors.New(`"." not allowed`)

3806 case "..":

3807 return "", false, errors.New(`".." not allowed`)

3808 }

3809 }

3810 if strings.HasPrefix(name, "/") || strings.HasSuffix(name, "/") || strings.Contains(name, "//") {

3811 return "", false, errors.New("bad slashes in mailbox name")

3812 }

3813

3814 // "%" and "*" are difficult to use with the IMAP LIST command, but we allow mostly

3815 // allow them. ../rfc/3501:1002 ../rfc/9051:983

3816 if strings.HasPrefix(name, "#") {

3817 return "", false, errors.New("mailbox name cannot start with hash due to conflict with imap namespaces")

3818 }

3819

3820 // "#" and "&" are special in IMAP mailbox names. "#" for namespaces, "&" for

3821 // IMAP-UTF-7 encoding. We do allow them. ../rfc/3501:1018 ../rfc/9051:991

3822

3823 for _, c := range name {

3824 // ../rfc/3501:999 ../rfc/6855:192 ../rfc/9051:979

3825 if c <= 0x1f || c >= 0x7f && c <= 0x9f || c == 0x2028 || c == 0x2029 {

3826 return "", false, errors.New("control characters not allowed in mailbox name")

3827 }

3828 }

3829 return name, false, nil

3830}

3831