1/*

2Package store implements storage for accounts, their mailboxes, IMAP

3subscriptions and messages, and broadcasts updates (e.g. mail delivery) to

4interested sessions (e.g. IMAP connections).

5

6Layout of storage for accounts:

7

8 <DataDir>/accounts/<name>/index.db

9 <DataDir>/accounts/<name>/msg/[a-zA-Z0-9_-]+/<id>

10

11Index.db holds tables for user information, mailboxes, and messages. Messages

12are stored in the msg/ subdirectory, each in their own file. The on-disk message

13does not contain headers generated during an incoming SMTP transaction, such as

14Received and Authentication-Results headers. Those are in the database to

15prevent having to rewrite incoming messages (e.g. Authentication-Result for DKIM

16signatures can only be determined after having read the message). Messages must

17be read through MsgReader, which transparently adds the prefix from the

18database.

19*/

20package store

21

22// todo: make up a function naming scheme that indicates whether caller should broadcast changes.

23

24import (

25 "context"

26 "crypto/md5"

27 cryptorand "crypto/rand"

28 "crypto/sha1"

29 "crypto/sha256"

30 "encoding"

31 "encoding/json"

32 "errors"

33 "fmt"

34 "hash"

35 "io"

36 "log/slog"

37 "os"

38 "path/filepath"

39 "runtime/debug"

40 "slices"

41 "sort"

42 "strconv"

43 "strings"

44 "sync"

45 "time"

46

47 "golang.org/x/crypto/bcrypt"

48 "golang.org/x/text/secure/precis"

49 "golang.org/x/text/unicode/norm"

50

51 "github.com/mjl-/bstore"

52

53 "github.com/mjl-/mox/config"

54 "github.com/mjl-/mox/dns"

55 "github.com/mjl-/mox/message"

56 "github.com/mjl-/mox/metrics"

57 "github.com/mjl-/mox/mlog"

58 "github.com/mjl-/mox/mox-"

59 "github.com/mjl-/mox/moxio"

60 "github.com/mjl-/mox/publicsuffix"

61 "github.com/mjl-/mox/scram"

62 "github.com/mjl-/mox/smtp"

63)

64

65// If true, each time an account is closed its database file is checked for

66// consistency. If an inconsistency is found, panic is called. Set by default

67// because of all the packages with tests, the mox main function sets it to

68// false again.

69var CheckConsistencyOnClose = true

70

71var (

72 ErrUnknownMailbox = errors.New("no such mailbox")

73 ErrUnknownCredentials = errors.New("credentials not found")

74 ErrAccountUnknown = errors.New("no such account")

75 ErrOverQuota = errors.New("account over quota")

76)

77

78var DefaultInitialMailboxes = config.InitialMailboxes{

79 SpecialUse: config.SpecialUseMailboxes{

80 Sent: "Sent",

81 Archive: "Archive",

82 Trash: "Trash",

83 Draft: "Drafts",

84 Junk: "Junk",

85 },

86}

87

88type SCRAM struct {

89 Salt []byte

90 Iterations int

91 SaltedPassword []byte

92}

93

94// CRAMMD5 holds HMAC ipad and opad hashes that are initialized with the first

95// block with (a derivation of) the key/password, so we don't store the password in plain

96// text.

97type CRAMMD5 struct {

98 Ipad hash.Hash

99 Opad hash.Hash

100}

101

102// BinaryMarshal is used by bstore to store the ipad/opad hash states.

103func (c CRAMMD5) MarshalBinary() ([]byte, error) {

104 if c.Ipad == nil || c.Opad == nil {

105 return nil, nil

106 }

107

108 ipad, err := c.Ipad.(encoding.BinaryMarshaler).MarshalBinary()

109 if err != nil {

110 return nil, fmt.Errorf("marshal ipad: %v", err)

111 }

112 opad, err := c.Opad.(encoding.BinaryMarshaler).MarshalBinary()

113 if err != nil {

114 return nil, fmt.Errorf("marshal opad: %v", err)

115 }

116 buf := make([]byte, 2+len(ipad)+len(opad))

117 ipadlen := uint16(len(ipad))

118 buf[0] = byte(ipadlen >> 8)

119 buf[1] = byte(ipadlen >> 0)

120 copy(buf[2:], ipad)

121 copy(buf[2+len(ipad):], opad)

122 return buf, nil

123}

124

125// BinaryUnmarshal is used by bstore to restore the ipad/opad hash states.

126func (c *CRAMMD5) UnmarshalBinary(buf []byte) error {

127 if len(buf) == 0 {

128 *c = CRAMMD5{}

129 return nil

130 }

131 if len(buf) < 2 {

132 return fmt.Errorf("short buffer")

133 }

134 ipadlen := int(uint16(buf[0])<<8 | uint16(buf[1])<<0)

135 if len(buf) < 2+ipadlen {

136 return fmt.Errorf("buffer too short for ipadlen")

137 }

138 ipad := md5.New()

139 opad := md5.New()

140 if err := ipad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2 : 2+ipadlen]); err != nil {

141 return fmt.Errorf("unmarshal ipad: %v", err)

142 }

143 if err := opad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2+ipadlen:]); err != nil {

144 return fmt.Errorf("unmarshal opad: %v", err)

145 }

146 *c = CRAMMD5{ipad, opad}

147 return nil

148}

149

150// Password holds credentials in various forms, for logging in with SMTP/IMAP.

151type Password struct {

152 Hash string // bcrypt hash for IMAP LOGIN, SASL PLAIN and HTTP basic authentication.

153 CRAMMD5 CRAMMD5 // For SASL CRAM-MD5.

154 SCRAMSHA1 SCRAM // For SASL SCRAM-SHA-1.

155 SCRAMSHA256 SCRAM // For SASL SCRAM-SHA-256.

156}

157

158// Subjectpass holds the secret key used to sign subjectpass tokens.

159type Subjectpass struct {

160 Email string // Our destination address (canonical, with catchall localpart stripped).

161 Key string

162}

163

164// NextUIDValidity is a singleton record in the database with the next UIDValidity

165// to use for the next mailbox.

166type NextUIDValidity struct {

167 ID int // Just a single record with ID 1.

168 Next uint32

169}

170

171// SyncState track ModSeqs.

172type SyncState struct {

173 ID int // Just a single record with ID 1.

174

175 // Last used, next assigned will be one higher. The first value we hand out is 2.

176 // That's because 0 (the default value for old existing messages, from before the

177 // Message.ModSeq field) is special in IMAP, so we return it as 1.

178 LastModSeq ModSeq `bstore:"nonzero"`

179

180 // Highest ModSeq of expunged record that we deleted. When a clients synchronizes

181 // and requests changes based on a modseq before this one, we don't have the

182 // history to provide information about deletions. We normally keep these expunged

183 // records around, but we may periodically truly delete them to reclaim storage

184 // space. Initially set to -1 because we don't want to match with any ModSeq in the

185 // database, which can be zero values.

186 HighestDeletedModSeq ModSeq

187}

188

189// Mailbox is collection of messages, e.g. Inbox or Sent.

190type Mailbox struct {

191 ID int64

192

193 // "Inbox" is the name for the special IMAP "INBOX". Slash separated

194 // for hierarchy.

195 Name string `bstore:"nonzero,unique"`

196

197 // If UIDs are invalidated, e.g. when renaming a mailbox to a previously existing

198 // name, UIDValidity must be changed. Used by IMAP for synchronization.

199 UIDValidity uint32

200

201 // UID likely to be assigned to next message. Used by IMAP to detect messages

202 // delivered to a mailbox.

203 UIDNext UID

204

205 SpecialUse

206

207 // Keywords as used in messages. Storing a non-system keyword for a message

208 // automatically adds it to this list. Used in the IMAP FLAGS response. Only

209 // "atoms" are allowed (IMAP syntax), keywords are case-insensitive, only stored in

210 // lower case (for JMAP), sorted.

211 Keywords []string

212

213 HaveCounts bool // Whether MailboxCounts have been initialized.

214 MailboxCounts // Statistics about messages, kept up to date whenever a change happens.

215}

216

217// MailboxCounts tracks statistics about messages for a mailbox.

218type MailboxCounts struct {

219 Total int64 // Total number of messages, excluding \Deleted. For JMAP.

220 Deleted int64 // Number of messages with \Deleted flag. Used for IMAP message count that includes messages with \Deleted.

221 Unread int64 // Messages without \Seen, excluding those with \Deleted, for JMAP.

222 Unseen int64 // Messages without \Seen, including those with \Deleted, for IMAP.

223 Size int64 // Number of bytes for all messages.

224}

225

226func (mc MailboxCounts) String() string {

227 return fmt.Sprintf("%d total, %d deleted, %d unread, %d unseen, size %d bytes", mc.Total, mc.Deleted, mc.Unread, mc.Unseen, mc.Size)

228}

229

230// Add increases mailbox counts mc with those of delta.

231func (mc *MailboxCounts) Add(delta MailboxCounts) {

232 mc.Total += delta.Total

233 mc.Deleted += delta.Deleted

234 mc.Unread += delta.Unread

235 mc.Unseen += delta.Unseen

236 mc.Size += delta.Size

237}

238

239// Add decreases mailbox counts mc with those of delta.

240func (mc *MailboxCounts) Sub(delta MailboxCounts) {

241 mc.Total -= delta.Total

242 mc.Deleted -= delta.Deleted

243 mc.Unread -= delta.Unread

244 mc.Unseen -= delta.Unseen

245 mc.Size -= delta.Size

246}

247

248// SpecialUse identifies a specific role for a mailbox, used by clients to

249// understand where messages should go.

250type SpecialUse struct {

251 Archive bool

252 Draft bool

253 Junk bool

254 Sent bool

255 Trash bool

256}

257

258// CalculateCounts calculates the full current counts for messages in the mailbox.

259func (mb *Mailbox) CalculateCounts(tx *bstore.Tx) (mc MailboxCounts, err error) {

260 q := bstore.QueryTx[Message](tx)

261 q.FilterNonzero(Message{MailboxID: mb.ID})

262 q.FilterEqual("Expunged", false)

263 err = q.ForEach(func(m Message) error {

264 mc.Add(m.MailboxCounts())

265 return nil

266 })

267 return

268}

269

270// ChangeSpecialUse returns a change for special-use flags, for broadcasting to

271// other connections.

272func (mb Mailbox) ChangeSpecialUse() ChangeMailboxSpecialUse {

273 return ChangeMailboxSpecialUse{mb.ID, mb.Name, mb.SpecialUse}

274}

275

276// ChangeKeywords returns a change with new keywords for a mailbox (e.g. after

277// setting a new keyword on a message in the mailbox), for broadcasting to other

278// connections.

279func (mb Mailbox) ChangeKeywords() ChangeMailboxKeywords {

280 return ChangeMailboxKeywords{mb.ID, mb.Name, mb.Keywords}

281}

282

283// KeywordsChanged returns whether the keywords in a mailbox have changed.

284func (mb Mailbox) KeywordsChanged(origmb Mailbox) bool {

285 if len(mb.Keywords) != len(origmb.Keywords) {

286 return true

287 }

288 // Keywords are stored sorted.

289 for i, kw := range mb.Keywords {

290 if origmb.Keywords[i] != kw {

291 return true

292 }

293 }

294 return false

295}

296

297// CountsChange returns a change with mailbox counts.

298func (mb Mailbox) ChangeCounts() ChangeMailboxCounts {

299 return ChangeMailboxCounts{mb.ID, mb.Name, mb.MailboxCounts}

300}

301

302// Subscriptions are separate from existence of mailboxes.

303type Subscription struct {

304 Name string

305}

306

307// Flags for a mail message.

308type Flags struct {

309 Seen bool

310 Answered bool

311 Flagged bool

312 Forwarded bool

313 Junk bool

314 Notjunk bool

315 Deleted bool

316 Draft bool

317 Phishing bool

318 MDNSent bool

319}

320

321// FlagsAll is all flags set, for use as mask.

322var FlagsAll = Flags{true, true, true, true, true, true, true, true, true, true}

323

324// Validation of "message From" domain.

325type Validation uint8

326

327const (

328 ValidationUnknown Validation = 0

329 ValidationStrict Validation = 1 // Like DMARC, with strict policies.

330 ValidationDMARC Validation = 2 // Actual DMARC policy.

331 ValidationRelaxed Validation = 3 // Like DMARC, with relaxed policies.

332 ValidationPass Validation = 4 // For SPF.

333 ValidationNeutral Validation = 5 // For SPF.

334 ValidationTemperror Validation = 6

335 ValidationPermerror Validation = 7

336 ValidationFail Validation = 8

337 ValidationSoftfail Validation = 9 // For SPF.

338 ValidationNone Validation = 10 // E.g. No records.

339)

340

341// Message stored in database and per-message file on disk.

342//

343// Contents are always the combined data from MsgPrefix and the on-disk file named

344// based on ID.

345//

346// Messages always have a header section, even if empty. Incoming messages without

347// header section must get an empty header section added before inserting.

348type Message struct {

349 // ID, unchanged over lifetime, determines path to on-disk msg file.

350 // Set during deliver.

351 ID int64

352

353 UID UID `bstore:"nonzero"` // UID, for IMAP. Set during deliver.

354 MailboxID int64 `bstore:"nonzero,unique MailboxID+UID,index MailboxID+Received,index MailboxID+ModSeq,ref Mailbox"`

355

356 // Modification sequence, for faster syncing with IMAP QRESYNC and JMAP.

357 // ModSeq is the last modification. CreateSeq is the Seq the message was inserted,

358 // always <= ModSeq. If Expunged is set, the message has been removed and should not

359 // be returned to the user. In this case, ModSeq is the Seq where the message is

360 // removed, and will never be changed again.

361 // We have an index on both ModSeq (for JMAP that synchronizes per account) and

362 // MailboxID+ModSeq (for IMAP that synchronizes per mailbox).

363 // The index on CreateSeq helps efficiently finding created messages for JMAP.

364 // The value of ModSeq is special for IMAP. Messages that existed before ModSeq was

365 // added have 0 as value. But modseq 0 in IMAP is special, so we return it as 1. If

366 // we get modseq 1 from a client, the IMAP server will translate it to 0. When we

367 // return modseq to clients, we turn 0 into 1.

368 ModSeq ModSeq `bstore:"index"`

369 CreateSeq ModSeq `bstore:"index"`

370 Expunged bool

371

372 // If set, this message was delivered to a Rejects mailbox. When it is moved to a

373 // different mailbox, its MailboxOrigID is set to the destination mailbox and this

374 // flag cleared.

375 IsReject bool

376

377 // If set, this is a forwarded message (through a ruleset with IsForward). This

378 // causes fields used during junk analysis to be moved to their Orig variants, and

379 // masked IP fields cleared, so they aren't used in junk classifications for

380 // incoming messages. This ensures the forwarded messages don't cause negative

381 // reputation for the forwarding mail server, which may also be sending regular

382 // messages.

383 IsForward bool

384

385 // MailboxOrigID is the mailbox the message was originally delivered to. Typically

386 // Inbox or Rejects, but can also be a mailbox configured in a Ruleset, or

387 // Postmaster, TLS/DMARC reporting addresses. MailboxOrigID is not changed when the

388 // message is moved to another mailbox, e.g. Archive/Trash/Junk. Used for

389 // per-mailbox reputation.

390 //

391 // MailboxDestinedID is normally 0, but when a message is delivered to the Rejects

392 // mailbox, it is set to the intended mailbox according to delivery rules,

393 // typically that of Inbox. When such a message is moved out of Rejects, the

394 // MailboxOrigID is corrected by setting it to MailboxDestinedID. This ensures the

395 // message is used for reputation calculation for future deliveries to that

396 // mailbox.

397 //

398 // These are not bstore references to prevent having to update all messages in a

399 // mailbox when the original mailbox is removed. Use of these fields requires

400 // checking if the mailbox still exists.

401 MailboxOrigID int64

402 MailboxDestinedID int64

403

404 Received time.Time `bstore:"default now,index"`

405

406 // Full IP address of remote SMTP server. Empty if not delivered over SMTP. The

407 // masked IPs are used to classify incoming messages. They are left empty for

408 // messages matching a ruleset for forwarded messages.

409 RemoteIP string

410 RemoteIPMasked1 string `bstore:"index RemoteIPMasked1+Received"` // For IPv4 /32, for IPv6 /64, for reputation.

411 RemoteIPMasked2 string `bstore:"index RemoteIPMasked2+Received"` // For IPv4 /26, for IPv6 /48.

412 RemoteIPMasked3 string `bstore:"index RemoteIPMasked3+Received"` // For IPv4 /21, for IPv6 /32.

413

414 // Only set if present and not an IP address. Unicode string. Empty for forwarded

415 // messages.

416 EHLODomain string `bstore:"index EHLODomain+Received"`

417 MailFrom string // With localpart and domain. Can be empty.

418 MailFromLocalpart smtp.Localpart // SMTP "MAIL FROM", can be empty.

419 // Only set if it is a domain, not an IP. Unicode string. Empty for forwarded

420 // messages, but see OrigMailFromDomain.

421 MailFromDomain string `bstore:"index MailFromDomain+Received"`

422 RcptToLocalpart smtp.Localpart // SMTP "RCPT TO", can be empty.

423 RcptToDomain string // Unicode string.

424

425 // Parsed "From" message header, used for reputation along with domain validation.

426 MsgFromLocalpart smtp.Localpart

427 MsgFromDomain string `bstore:"index MsgFromDomain+Received"` // Unicode string.

428 MsgFromOrgDomain string `bstore:"index MsgFromOrgDomain+Received"` // Unicode string.

429

430 // Simplified statements of the Validation fields below, used for incoming messages

431 // to check reputation.

432 EHLOValidated bool

433 MailFromValidated bool

434 MsgFromValidated bool

435

436 EHLOValidation Validation // Validation can also take reverse IP lookup into account, not only SPF.

437 MailFromValidation Validation // Can have SPF-specific validations like ValidationSoftfail.

438 MsgFromValidation Validation // Desirable validations: Strict, DMARC, Relaxed. Will not be just Pass.

439

440 // Domains with verified DKIM signatures. Unicode string. For forwarded messages, a

441 // DKIM domain that matched a ruleset's verified domain is left out, but included

442 // in OrigDKIMDomains.

443 DKIMDomains []string `bstore:"index DKIMDomains+Received"`

444

445 // For forwarded messages,

446 OrigEHLODomain string

447 OrigDKIMDomains []string

448

449 // Canonicalized Message-Id, always lower-case and normalized quoting, without

450 // <>'s. Empty if missing. Used for matching message threads, and to prevent

451 // duplicate reject delivery.

452 MessageID string `bstore:"index"`

453 // lower-case: ../rfc/5256:495

454

455 // For matching threads in case there is no References/In-Reply-To header. It is

456 // lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed.

457 SubjectBase string `bstore:"index"`

458 // ../rfc/5256:90

459

460 // Hash of message. For rejects delivery in case there is no Message-ID, only set

461 // when delivered as reject.

462 MessageHash []byte

463

464 // ID of message starting this thread.

465 ThreadID int64 `bstore:"index"`

466 // IDs of parent messages, from closest parent to the root message. Parent messages

467 // may be in a different mailbox, or may no longer exist. ThreadParentIDs must

468 // never contain the message id itself (a cycle), and parent messages must

469 // reference the same ancestors.

470 ThreadParentIDs []int64

471 // ThreadMissingLink is true if there is no match with a direct parent. E.g. first

472 // ID in ThreadParentIDs is not the direct ancestor (an intermediate message may

473 // have been deleted), or subject-based matching was done.

474 ThreadMissingLink bool

475 // If set, newly delivered child messages are automatically marked as read. This

476 // field is copied to new child messages. Changes are propagated to the webmail

477 // client.

478 ThreadMuted bool

479 // If set, this (sub)thread is collapsed in the webmail client, for threading mode

480 // "on" (mode "unread" ignores it). This field is copied to new child message.

481 // Changes are propagated to the webmail client.

482 ThreadCollapsed bool

483

484 // If received message was known to match a mailing list rule (with modified junk

485 // filtering).

486 IsMailingList bool

487

488 // If this message is a DSN, generated by us or received. For DSNs, we don't look

489 // at the subject when matching threads.

490 DSN bool

491

492 ReceivedTLSVersion uint16 // 0 if unknown, 1 if plaintext/no TLS, otherwise TLS cipher suite.

493 ReceivedTLSCipherSuite uint16

494 ReceivedRequireTLS bool // Whether RequireTLS was known to be used for incoming delivery.

495

496 Flags

497 // For keywords other than system flags or the basic well-known $-flags. Only in

498 // "atom" syntax (IMAP), they are case-insensitive, always stored in lower-case

499 // (for JMAP), sorted.

500 Keywords []string `bstore:"index"`

501 Size int64

502 TrainedJunk *bool // If nil, no training done yet. Otherwise, true is trained as junk, false trained as nonjunk.

503 MsgPrefix []byte // Typically holds received headers and/or header separator.

504

505 // ParsedBuf message structure. Currently saved as JSON of message.Part because bstore

506 // cannot yet store recursive types. Created when first needed, and saved in the

507 // database.

508 // todo: once replaced with non-json storage, remove date fixup in ../message/part.go.

509 ParsedBuf []byte

510}

511

512// MailboxCounts returns the delta to counts this message means for its

513// mailbox.

514func (m Message) MailboxCounts() (mc MailboxCounts) {

515 if m.Expunged {

516 return

517 }

518 if m.Deleted {

519 mc.Deleted++

520 } else {

521 mc.Total++

522 }

523 if !m.Seen {

524 mc.Unseen++

525 if !m.Deleted {

526 mc.Unread++

527 }

528 }

529 mc.Size += m.Size

530 return

531}

532

533func (m Message) ChangeAddUID() ChangeAddUID {

534 return ChangeAddUID{m.MailboxID, m.UID, m.ModSeq, m.Flags, m.Keywords}

535}

536

537func (m Message) ChangeFlags(orig Flags) ChangeFlags {

538 mask := m.Flags.Changed(orig)

539 return ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, ModSeq: m.ModSeq, Mask: mask, Flags: m.Flags, Keywords: m.Keywords}

540}

541

542func (m Message) ChangeThread() ChangeThread {

543 return ChangeThread{[]int64{m.ID}, m.ThreadMuted, m.ThreadCollapsed}

544}

545

546// ModSeq represents a modseq as stored in the database. ModSeq 0 in the

547// database is sent to the client as 1, because modseq 0 is special in IMAP.

548// ModSeq coming from the client are of type int64.

549type ModSeq int64

550

551func (ms ModSeq) Client() int64 {

552 if ms == 0 {

553 return 1

554 }

555 return int64(ms)

556}

557

558// ModSeqFromClient converts a modseq from a client to a modseq for internal

559// use, e.g. in a database query.

560// ModSeq 1 is turned into 0 (the Go zero value for ModSeq).

561func ModSeqFromClient(modseq int64) ModSeq {

562 if modseq == 1 {

563 return 0

564 }

565 return ModSeq(modseq)

566}

567

568// PrepareExpunge clears fields that are no longer needed after an expunge, so

569// almost all fields. Does not change ModSeq, but does set Expunged.

570func (m *Message) PrepareExpunge() {

571 *m = Message{

572 ID: m.ID,

573 UID: m.UID,

574 MailboxID: m.MailboxID,

575 CreateSeq: m.CreateSeq,

576 ModSeq: m.ModSeq,

577 Expunged: true,

578 ThreadID: m.ThreadID,

579 }

580}

581

582// PrepareThreading sets MessageID, SubjectBase and DSN (used in threading) based

583// on the part.

584func (m *Message) PrepareThreading(log mlog.Log, part *message.Part) {

585 m.DSN = part.IsDSN()

586

587 if part.Envelope == nil {

588 return

589 }

590 messageID, raw, err := message.MessageIDCanonical(part.Envelope.MessageID)

591 if err != nil {

592 log.Debugx("parsing message-id, ignoring", err, slog.String("messageid", part.Envelope.MessageID))

593 } else if raw {

594 log.Debug("could not parse message-id as address, continuing with raw value", slog.String("messageid", part.Envelope.MessageID))

595 }

596 m.MessageID = messageID

597 m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false)

598}

599

600// LoadPart returns a message.Part by reading from m.ParsedBuf.

601func (m Message) LoadPart(r io.ReaderAt) (message.Part, error) {

602 if m.ParsedBuf == nil {

603 return message.Part{}, fmt.Errorf("message not parsed")

604 }

605 var p message.Part

606 err := json.Unmarshal(m.ParsedBuf, &p)

607 if err != nil {

608 return p, fmt.Errorf("unmarshal message part")

609 }

610 p.SetReaderAt(r)

611 return p, nil

612}

613

614// NeedsTraining returns whether message needs a training update, based on

615// TrainedJunk (current training status) and new Junk/Notjunk flags.

616func (m Message) NeedsTraining() bool {

617 untrain := m.TrainedJunk != nil

618 untrainJunk := untrain && *m.TrainedJunk

619 train := m.Junk || m.Notjunk && !(m.Junk && m.Notjunk)

620 trainJunk := m.Junk

621 return untrain != train || untrain && train && untrainJunk != trainJunk

622}

623

624// JunkFlagsForMailbox sets Junk and Notjunk flags based on mailbox name if configured. Often

625// used when delivering/moving/copying messages to a mailbox. Mail clients are not

626// very helpful with setting junk/notjunk flags. But clients can move/copy messages

627// to other mailboxes. So we set flags when clients move a message.

628func (m *Message) JunkFlagsForMailbox(mb Mailbox, conf config.Account) {

629 if mb.Junk {

630 m.Junk = true

631 m.Notjunk = false

632 return

633 }

634

635 if !conf.AutomaticJunkFlags.Enabled {

636 return

637 }

638

639 lmailbox := strings.ToLower(mb.Name)

640

641 if conf.JunkMailbox != nil && conf.JunkMailbox.MatchString(lmailbox) {

642 m.Junk = true

643 m.Notjunk = false

644 } else if conf.NeutralMailbox != nil && conf.NeutralMailbox.MatchString(lmailbox) {

645 m.Junk = false

646 m.Notjunk = false

647 } else if conf.NotJunkMailbox != nil && conf.NotJunkMailbox.MatchString(lmailbox) {

648 m.Junk = false

649 m.Notjunk = true

650 } else if conf.JunkMailbox == nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox != nil {

651 m.Junk = true

652 m.Notjunk = false

653 } else if conf.JunkMailbox != nil && conf.NeutralMailbox == nil && conf.NotJunkMailbox != nil {

654 m.Junk = false

655 m.Notjunk = false

656 } else if conf.JunkMailbox != nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox == nil {

657 m.Junk = false

658 m.Notjunk = true

659 }

660}

661

662// Recipient represents the recipient of a message. It is tracked to allow

663// first-time incoming replies from users this account has sent messages to. When a

664// mailbox is added to the Sent mailbox the message is parsed and recipients are

665// inserted as recipient. Recipients are never removed other than for removing the

666// message. On move/copy of a message, recipients aren't modified either. For IMAP,

667// this assumes a client simply appends messages to the Sent mailbox (as opposed to

668// copying messages from some place).

669type Recipient struct {

670 ID int64

671 MessageID int64 `bstore:"nonzero,ref Message"` // Ref gives it its own index, useful for fast removal as well.

672 Localpart string `bstore:"nonzero"` // Encoded localpart.

673 Domain string `bstore:"nonzero,index Domain+Localpart"` // Unicode string.

674 OrgDomain string `bstore:"nonzero,index"` // Unicode string.

675 Sent time.Time `bstore:"nonzero"`

676}

677

678// Outgoing is a message submitted for delivery from the queue. Used to enforce

679// maximum outgoing messages.

680type Outgoing struct {

681 ID int64

682 Recipient string `bstore:"nonzero,index"` // Canonical international address with utf8 domain.

683 Submitted time.Time `bstore:"nonzero,default now"`

684}

685

686// RecipientDomainTLS stores TLS capabilities of a recipient domain as encountered

687// during most recent connection (delivery attempt).

688type RecipientDomainTLS struct {

689 Domain string // Unicode.

690 Updated time.Time `bstore:"default now"`

691 STARTTLS bool // Supports STARTTLS.

692 RequireTLS bool // Supports RequireTLS SMTP extension.

693}

694

695// DiskUsage tracks quota use.

696type DiskUsage struct {

697 ID int64 // Always one record with ID 1.

698 MessageSize int64 // Sum of all messages, for quota accounting.

699}

700

701// SessionToken and CSRFToken are types to prevent mixing them up.

702// Base64 raw url encoded.

703type SessionToken string

704type CSRFToken string

705

706// LoginSession represents a login session. We keep a limited number of sessions

707// for a user, removing the oldest session when a new one is created.

708type LoginSession struct {

709 ID int64

710 Created time.Time `bstore:"nonzero,default now"` // Of original login.

711 Expires time.Time `bstore:"nonzero"` // Extended each time it is used.

712 SessionTokenBinary [16]byte `bstore:"nonzero"` // Stored in cookie, like "webmailsession" or "webaccountsession".

713 CSRFTokenBinary [16]byte // For API requests, in "x-mox-csrf" header.

714 AccountName string `bstore:"nonzero"`

715 LoginAddress string `bstore:"nonzero"`

716

717 // Set when loading from database.

718 sessionToken SessionToken

719 csrfToken CSRFToken

720}

721

722// Quoting is a setting for how to quote in replies/forwards.

723type Quoting string

724

725const (

726 Default Quoting = "" // Bottom-quote if text is selected, top-quote otherwise.

727 Bottom Quoting = "bottom"

728 Top Quoting = "top"

729)

730

731// Settings are webmail client settings.

732type Settings struct {

733 ID uint8 // Singleton ID 1.

734

735 Signature string

736 Quoting Quoting

737

738 // Whether to show the bars underneath the address input fields indicating

739 // starttls/dnssec/dane/mtasts/requiretls support by address.

740 ShowAddressSecurity bool

741

742 // Show HTML version of message by default, instead of plain text.

743 ShowHTML bool

744}

745

746// ViewMode how a message should be viewed: its text parts, html parts, or html

747// with loading external resources.

748type ViewMode string

749

750const (

751 ModeText ViewMode = "text"

752 ModeHTML ViewMode = "html"

753 ModeHTMLExt ViewMode = "htmlext" // HTML with external resources.

754)

755

756// FromAddressSettings are webmail client settings per "From" address.

757type FromAddressSettings struct {

758 FromAddress string // Unicode.

759 ViewMode ViewMode

760}

761

762// RulesetNoListID records a user "no" response to the question of

763// creating/removing a ruleset after moving a message with list-id header from/to

764// the inbox.

765type RulesetNoListID struct {

766 ID int64

767 RcptToAddress string `bstore:"nonzero"`

768 ListID string `bstore:"nonzero"`

769 ToInbox bool // Otherwise from Inbox to other mailbox.

770}

771

772// RulesetNoMsgFrom records a user "no" response to the question of

773// creating/moveing a ruleset after moving a mesage with message "from" address

774// from/to the inbox.

775type RulesetNoMsgFrom struct {

776 ID int64

777 RcptToAddress string `bstore:"nonzero"`

778 MsgFromAddress string `bstore:"nonzero"` // Unicode.

779 ToInbox bool // Otherwise from Inbox to other mailbox.

780}

781

782// RulesetNoMailbox represents a "never from/to this mailbox" response to the

783// question of adding/removing a ruleset after moving a message.

784type RulesetNoMailbox struct {

785 ID int64

786

787 // The mailbox from/to which the move has happened.

788 // Not a references, if mailbox is deleted, an entry becomes ineffective.

789 MailboxID int64 `bstore:"nonzero"`

790 ToMailbox bool // Whether MailboxID is the destination of the move (instead of source).

791}

792

793// Types stored in DB.

794var DBTypes = []any{

795 NextUIDValidity{},

796 Message{},

797 Recipient{},

798 Mailbox{},

799 Subscription{},

800 Outgoing{},

801 Password{},

802 Subjectpass{},

803 SyncState{},

804 Upgrade{},

805 RecipientDomainTLS{},

806 DiskUsage{},

807 LoginSession{},

808 Settings{},

809 FromAddressSettings{},

810 RulesetNoListID{},

811 RulesetNoMsgFrom{},

812 RulesetNoMailbox{},

813}

814

815// Account holds the information about a user, includings mailboxes, messages, imap subscriptions.

816type Account struct {

817 Name string // Name, according to configuration.

818 Dir string // Directory where account files, including the database, bloom filter, and mail messages, are stored for this account.

819 DBPath string // Path to database with mailboxes, messages, etc.

820 DB *bstore.DB // Open database connection.

821

822 // Channel that is closed if/when account has/gets "threads" accounting (see

823 // Upgrade.Threads).

824 threadsCompleted chan struct{}

825 // If threads upgrade completed with error, this is set. Used for warning during

826 // delivery, or aborting when importing.

827 threadsErr error

828

829 // Write lock must be held for account/mailbox modifications including message delivery.

830 // Read lock for reading mailboxes/messages.

831 // When making changes to mailboxes/messages, changes must be broadcasted before

832 // releasing the lock to ensure proper UID ordering.

833 sync.RWMutex

834

835 nused int // Reference count, while >0, this account is alive and shared.

836}

837

838type Upgrade struct {

839 ID byte

840 Threads byte // 0: None, 1: Adding MessageID's completed, 2: Adding ThreadID's completed.

841}

842

843// InitialUIDValidity returns a UIDValidity used for initializing an account.

844// It can be replaced during tests with a predictable value.

845var InitialUIDValidity = func() uint32 {

846 return uint32(time.Now().Unix() >> 1) // A 2-second resolution will get us far enough beyond 2038.

847}

848

849var openAccounts = struct {

850 names map[string]*Account

851 sync.Mutex

852}{

853 names: map[string]*Account{},

854}

855

856func closeAccount(acc *Account) (rerr error) {

857 openAccounts.Lock()

858 acc.nused--

859 defer openAccounts.Unlock()

860 if acc.nused == 0 {

861 // threadsCompleted must be closed now because it increased nused.

862 rerr = acc.DB.Close()

863 acc.DB = nil

864 delete(openAccounts.names, acc.Name)

865 }

866 return

867}

868

869// OpenAccount opens an account by name.

870//

871// No additional data path prefix or ".db" suffix should be added to the name.

872// A single shared account exists per name.

873func OpenAccount(log mlog.Log, name string) (*Account, error) {

874 openAccounts.Lock()

875 defer openAccounts.Unlock()

876 if acc, ok := openAccounts.names[name]; ok {

877 acc.nused++

878 return acc, nil

879 }

880

881 if _, ok := mox.Conf.Account(name); !ok {

882 return nil, ErrAccountUnknown

883 }

884

885 acc, err := openAccount(log, name)

886 if err != nil {

887 return nil, err

888 }

889 openAccounts.names[name] = acc

890 return acc, nil

891}

892

893// openAccount opens an existing account, or creates it if it is missing.

894func openAccount(log mlog.Log, name string) (a *Account, rerr error) {

895 dir := filepath.Join(mox.DataDirPath("accounts"), name)

896 return OpenAccountDB(log, dir, name)

897}

898

899// OpenAccountDB opens an account database file and returns an initialized account

900// or error. Only exported for use by subcommands that verify the database file.

901// Almost all account opens must go through OpenAccount/OpenEmail/OpenEmailAuth.

902func OpenAccountDB(log mlog.Log, accountDir, accountName string) (a *Account, rerr error) {

903 dbpath := filepath.Join(accountDir, "index.db")

904

905 // Create account if it doesn't exist yet.

906 isNew := false

907 if _, err := os.Stat(dbpath); err != nil && os.IsNotExist(err) {

908 isNew = true

909 os.MkdirAll(accountDir, 0770)

910 }

911

912 opts := bstore.Options{Timeout: 5 * time.Second, Perm: 0660, RegisterLogger: log.Logger}

913 db, err := bstore.Open(context.TODO(), dbpath, &opts, DBTypes...)

914 if err != nil {

915 return nil, err

916 }

917

918 defer func() {

919 if rerr != nil {

920 db.Close()

921 if isNew {

922 os.Remove(dbpath)

923 }

924 }

925 }()

926

927 acc := &Account{

928 Name: accountName,

929 Dir: accountDir,

930 DBPath: dbpath,

931 DB: db,

932 nused: 1,

933 threadsCompleted: make(chan struct{}),

934 }

935

936 if isNew {

937 if err := initAccount(db); err != nil {

938 return nil, fmt.Errorf("initializing account: %v", err)

939 }

940 close(acc.threadsCompleted)

941 return acc, nil

942 }

943

944 // Ensure singletons are present. Mailbox counts and total message size, Settings.

945 var mentioned bool

946 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

947 if tx.Get(&Settings{ID: 1}) == bstore.ErrAbsent {

948 if err := tx.Insert(&Settings{ID: 1, ShowAddressSecurity: true}); err != nil {

949 return err

950 }

951 }

952

953 err := bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error {

954 if !mentioned {

955 mentioned = true

956 log.Info("first calculation of mailbox counts for account", slog.String("account", accountName))

957 }

958 mc, err := mb.CalculateCounts(tx)

959 if err != nil {

960 return err

961 }

962 mb.HaveCounts = true

963 mb.MailboxCounts = mc

964 return tx.Update(&mb)

965 })

966 if err != nil {

967 return err

968 }

969

970 du := DiskUsage{ID: 1}

971 err = tx.Get(&du)

972 if err == nil || !errors.Is(err, bstore.ErrAbsent) {

973 return err

974 }

975 // No DiskUsage record yet, calculate total size and insert.

976 err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {

977 du.MessageSize += mb.Size

978 return nil

979 })

980 if err != nil {

981 return err

982 }

983 return tx.Insert(&du)

984 })

985 if err != nil {

986 return nil, fmt.Errorf("calculating counts for mailbox or inserting settings: %v", err)

987 }

988

989 // Start adding threading if needed.

990 up := Upgrade{ID: 1}

991 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

992 err := tx.Get(&up)

993 if err == bstore.ErrAbsent {

994 if err := tx.Insert(&up); err != nil {

995 return fmt.Errorf("inserting initial upgrade record: %v", err)

996 }

997 err = nil

998 }

999 return err

1000 })

1001 if err != nil {

1002 return nil, fmt.Errorf("checking message threading: %v", err)

1003 }

1004 if up.Threads == 2 {

1005 close(acc.threadsCompleted)

1006 return acc, nil

1007 }

1008

1009 // Increase account use before holding on to account in background.

1010 // Caller holds the lock. The goroutine below decreases nused by calling

1011 // closeAccount.

1012 acc.nused++

1013

1014 // Ensure all messages have a MessageID and SubjectBase, which are needed when

1015 // matching threads.

1016 // Then assign messages to threads, in the same way we do during imports.

1017 log.Info("upgrading account for threading, in background", slog.String("account", acc.Name))

1018 go func() {

1019 defer func() {

1020 err := closeAccount(acc)

1021 log.Check(err, "closing use of account after upgrading account storage for threads", slog.String("account", a.Name))

1022

1023 // Mark that upgrade has finished, possibly error is indicated in threadsErr.

1024 close(acc.threadsCompleted)

1025 }()

1026

1027 defer func() {

1028 x := recover() // Should not happen, but don't take program down if it does.

1029 if x != nil {

1030 log.Error("upgradeThreads panic", slog.Any("err", x))

1031 debug.PrintStack()

1032 metrics.PanicInc(metrics.Upgradethreads)

1033 acc.threadsErr = fmt.Errorf("panic during upgradeThreads: %v", x)

1034 }

1035 }()

1036

1037 err := upgradeThreads(mox.Shutdown, log, acc, &up)

1038 if err != nil {

1039 a.threadsErr = err

1040 log.Errorx("upgrading account for threading, aborted", err, slog.String("account", a.Name))

1041 } else {

1042 log.Info("upgrading account for threading, completed", slog.String("account", a.Name))

1043 }

1044 }()

1045 return acc, nil

1046}

1047

1048// ThreadingWait blocks until the one-time account threading upgrade for the

1049// account has completed, and returns an error if not successful.

1050//

1051// To be used before starting an import of messages.

1052func (a *Account) ThreadingWait(log mlog.Log) error {

1053 select {

1054 case <-a.threadsCompleted:

1055 return a.threadsErr

1056 default:

1057 }

1058 log.Debug("waiting for account upgrade to complete")

1059

1060 <-a.threadsCompleted

1061 return a.threadsErr

1062}

1063

1064func initAccount(db *bstore.DB) error {

1065 return db.Write(context.TODO(), func(tx *bstore.Tx) error {

1066 uidvalidity := InitialUIDValidity()

1067

1068 if err := tx.Insert(&Upgrade{ID: 1, Threads: 2}); err != nil {

1069 return err

1070 }

1071 if err := tx.Insert(&DiskUsage{ID: 1}); err != nil {

1072 return err

1073 }

1074 if err := tx.Insert(&Settings{ID: 1}); err != nil {

1075 return err

1076 }

1077

1078 if len(mox.Conf.Static.DefaultMailboxes) > 0 {

1079 // Deprecated in favor of InitialMailboxes.

1080 defaultMailboxes := mox.Conf.Static.DefaultMailboxes

1081 mailboxes := []string{"Inbox"}

1082 for _, name := range defaultMailboxes {

1083 if strings.EqualFold(name, "Inbox") {

1084 continue

1085 }

1086 mailboxes = append(mailboxes, name)

1087 }

1088 for _, name := range mailboxes {

1089 mb := Mailbox{Name: name, UIDValidity: uidvalidity, UIDNext: 1, HaveCounts: true}

1090 if strings.HasPrefix(name, "Archive") {

1091 mb.Archive = true

1092 } else if strings.HasPrefix(name, "Drafts") {

1093 mb.Draft = true

1094 } else if strings.HasPrefix(name, "Junk") {

1095 mb.Junk = true

1096 } else if strings.HasPrefix(name, "Sent") {

1097 mb.Sent = true

1098 } else if strings.HasPrefix(name, "Trash") {

1099 mb.Trash = true

1100 }

1101 if err := tx.Insert(&mb); err != nil {

1102 return fmt.Errorf("creating mailbox: %w", err)

1103 }

1104 if err := tx.Insert(&Subscription{name}); err != nil {

1105 return fmt.Errorf("adding subscription: %w", err)

1106 }

1107 }

1108 } else {

1109 mailboxes := mox.Conf.Static.InitialMailboxes

1110 var zerouse config.SpecialUseMailboxes

1111 if mailboxes.SpecialUse == zerouse && len(mailboxes.Regular) == 0 {

1112 mailboxes = DefaultInitialMailboxes

1113 }

1114

1115 add := func(name string, use SpecialUse) error {

1116 mb := Mailbox{Name: name, UIDValidity: uidvalidity, UIDNext: 1, SpecialUse: use, HaveCounts: true}

1117 if err := tx.Insert(&mb); err != nil {

1118 return fmt.Errorf("creating mailbox: %w", err)

1119 }

1120 if err := tx.Insert(&Subscription{name}); err != nil {

1121 return fmt.Errorf("adding subscription: %w", err)

1122 }

1123 return nil

1124 }

1125 addSpecialOpt := func(nameOpt string, use SpecialUse) error {

1126 if nameOpt == "" {

1127 return nil

1128 }

1129 return add(nameOpt, use)

1130 }

1131 l := []struct {

1132 nameOpt string

1133 use SpecialUse

1134 }{

1135 {"Inbox", SpecialUse{}},

1136 {mailboxes.SpecialUse.Archive, SpecialUse{Archive: true}},

1137 {mailboxes.SpecialUse.Draft, SpecialUse{Draft: true}},

1138 {mailboxes.SpecialUse.Junk, SpecialUse{Junk: true}},

1139 {mailboxes.SpecialUse.Sent, SpecialUse{Sent: true}},

1140 {mailboxes.SpecialUse.Trash, SpecialUse{Trash: true}},

1141 }

1142 for _, e := range l {

1143 if err := addSpecialOpt(e.nameOpt, e.use); err != nil {

1144 return err

1145 }

1146 }

1147 for _, name := range mailboxes.Regular {

1148 if err := add(name, SpecialUse{}); err != nil {

1149 return err

1150 }

1151 }

1152 }

1153

1154 uidvalidity++

1155 if err := tx.Insert(&NextUIDValidity{1, uidvalidity}); err != nil {

1156 return fmt.Errorf("inserting nextuidvalidity: %w", err)

1157 }

1158 return nil

1159 })

1160}

1161

1162// CheckClosed asserts that the account has a zero reference count. For use in tests.

1163func (a *Account) CheckClosed() {

1164 openAccounts.Lock()

1165 defer openAccounts.Unlock()

1166 if a.nused != 0 {

1167 panic(fmt.Sprintf("account still in use, %d refs", a.nused))

1168 }

1169}

1170

1171// Close reduces the reference count, and closes the database connection when

1172// it was the last user.

1173func (a *Account) Close() error {

1174 if CheckConsistencyOnClose {

1175 xerr := a.CheckConsistency()

1176 err := closeAccount(a)

1177 if xerr != nil {

1178 panic(xerr)

1179 }

1180 return err

1181 }

1182 return closeAccount(a)

1183}

1184

1185// CheckConsistency checks the consistency of the database and returns a non-nil

1186// error for these cases:

1187//

1188// - Missing on-disk file for message.

1189// - Mismatch between message size and length of MsgPrefix and on-disk file.

1190// - Missing HaveCounts.

1191// - Incorrect mailbox counts.

1192// - Incorrect total message size.

1193// - Message with UID >= mailbox uid next.

1194// - Mailbox uidvalidity >= account uid validity.

1195// - ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq.

1196// - All messages have a nonzero ThreadID, and no cycles in ThreadParentID, and parent messages the same ThreadParentIDs tail.

1197func (a *Account) CheckConsistency() error {

1198 var uidErrors []string // With a limit, could be many.

1199 var modseqErrors []string // With limit.

1200 var fileErrors []string // With limit.

1201 var threadidErrors []string // With limit.

1202 var threadParentErrors []string // With limit.

1203 var threadAncestorErrors []string // With limit.

1204 var errors []string

1205

1206 err := a.DB.Read(context.Background(), func(tx *bstore.Tx) error {

1207 nuv := NextUIDValidity{ID: 1}

1208 err := tx.Get(&nuv)

1209 if err != nil {

1210 return fmt.Errorf("fetching next uid validity: %v", err)

1211 }

1212

1213 mailboxes := map[int64]Mailbox{}

1214 err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {

1215 mailboxes[mb.ID] = mb

1216

1217 if mb.UIDValidity >= nuv.Next {

1218 errmsg := fmt.Sprintf("mailbox %q (id %d) has uidvalidity %d >= account next uidvalidity %d", mb.Name, mb.ID, mb.UIDValidity, nuv.Next)

1219 errors = append(errors, errmsg)

1220 }

1221 return nil

1222 })

1223 if err != nil {

1224 return fmt.Errorf("listing mailboxes: %v", err)

1225 }

1226

1227 counts := map[int64]MailboxCounts{}

1228 err = bstore.QueryTx[Message](tx).ForEach(func(m Message) error {

1229 mc := counts[m.MailboxID]

1230 mc.Add(m.MailboxCounts())

1231 counts[m.MailboxID] = mc

1232

1233 mb := mailboxes[m.MailboxID]

1234

1235 if (m.ModSeq == 0 || m.CreateSeq == 0 || m.CreateSeq > m.ModSeq) && len(modseqErrors) < 20 {

1236 modseqerr := fmt.Sprintf("message %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and createseq <= modseq", m.ID, mb.Name, mb.ID, m.ModSeq, m.CreateSeq)

1237 modseqErrors = append(modseqErrors, modseqerr)

1238 }

1239 if m.UID >= mb.UIDNext && len(uidErrors) < 20 {

1240 uiderr := fmt.Sprintf("message %d in mailbox %q (id %d) has uid %d >= mailbox uidnext %d", m.ID, mb.Name, mb.ID, m.UID, mb.UIDNext)

1241 uidErrors = append(uidErrors, uiderr)

1242 }

1243 if m.Expunged {

1244 return nil

1245 }

1246 p := a.MessagePath(m.ID)

1247 st, err := os.Stat(p)

1248 if err != nil {

1249 existserr := fmt.Sprintf("message %d in mailbox %q (id %d) on-disk file %s: %v", m.ID, mb.Name, mb.ID, p, err)

1250 fileErrors = append(fileErrors, existserr)

1251 } else if len(fileErrors) < 20 && m.Size != int64(len(m.MsgPrefix))+st.Size() {

1252 sizeerr := fmt.Sprintf("message %d in mailbox %q (id %d) has size %d != len msgprefix %d + on-disk file size %d = %d", m.ID, mb.Name, mb.ID, m.Size, len(m.MsgPrefix), st.Size(), int64(len(m.MsgPrefix))+st.Size())

1253 fileErrors = append(fileErrors, sizeerr)

1254 }

1255

1256 if m.ThreadID <= 0 && len(threadidErrors) < 20 {

1257 err := fmt.Sprintf("message %d in mailbox %q (id %d) has threadid 0", m.ID, mb.Name, mb.ID)

1258 threadidErrors = append(threadidErrors, err)

1259 }

1260 if slices.Contains(m.ThreadParentIDs, m.ID) && len(threadParentErrors) < 20 {

1261 err := fmt.Sprintf("message %d in mailbox %q (id %d) references itself in threadparentids", m.ID, mb.Name, mb.ID)

1262 threadParentErrors = append(threadParentErrors, err)

1263 }

1264 for i, pid := range m.ThreadParentIDs {

1265 am := Message{ID: pid}

1266 if err := tx.Get(&am); err == bstore.ErrAbsent {

1267 continue

1268 } else if err != nil {

1269 return fmt.Errorf("get ancestor message: %v", err)

1270 } else if !slices.Equal(m.ThreadParentIDs[i+1:], am.ThreadParentIDs) && len(threadAncestorErrors) < 20 {

1271 err := fmt.Sprintf("message %d, thread %d has ancestor ids %v, and ancestor at index %d with id %d should have the same tail but has %v\n", m.ID, m.ThreadID, m.ThreadParentIDs, i, am.ID, am.ThreadParentIDs)

1272 threadAncestorErrors = append(threadAncestorErrors, err)

1273 } else {

1274 break

1275 }

1276 }

1277 return nil

1278 })

1279 if err != nil {

1280 return fmt.Errorf("reading messages: %v", err)

1281 }

1282

1283 var totalSize int64

1284 for _, mb := range mailboxes {

1285 totalSize += mb.Size

1286 if !mb.HaveCounts {

1287 errmsg := fmt.Sprintf("mailbox %q (id %d) does not have counts, should be %#v", mb.Name, mb.ID, counts[mb.ID])

1288 errors = append(errors, errmsg)

1289 } else if mb.MailboxCounts != counts[mb.ID] {

1290 mbcounterr := fmt.Sprintf("mailbox %q (id %d) has wrong counts %s, should be %s", mb.Name, mb.ID, mb.MailboxCounts, counts[mb.ID])

1291 errors = append(errors, mbcounterr)

1292 }

1293 }

1294

1295 du := DiskUsage{ID: 1}

1296 if err := tx.Get(&du); err != nil {

1297 return fmt.Errorf("get diskusage")

1298 }

1299 if du.MessageSize != totalSize {

1300 errmsg := fmt.Sprintf("total message size in database is %d, sum of mailbox message sizes is %d", du.MessageSize, totalSize)

1301 errors = append(errors, errmsg)

1302 }

1303

1304 return nil

1305 })

1306 if err != nil {

1307 return err

1308 }

1309 errors = append(errors, uidErrors...)

1310 errors = append(errors, modseqErrors...)

1311 errors = append(errors, fileErrors...)

1312 errors = append(errors, threadidErrors...)

1313 errors = append(errors, threadParentErrors...)

1314 errors = append(errors, threadAncestorErrors...)

1315 if len(errors) > 0 {

1316 return fmt.Errorf("%s", strings.Join(errors, "; "))

1317 }

1318 return nil

1319}

1320

1321// Conf returns the configuration for this account if it still exists. During

1322// an SMTP session, a configuration update may drop an account.

1323func (a *Account) Conf() (config.Account, bool) {

1324 return mox.Conf.Account(a.Name)

1325}

1326

1327// NextUIDValidity returns the next new/unique uidvalidity to use for this account.

1328func (a *Account) NextUIDValidity(tx *bstore.Tx) (uint32, error) {

1329 nuv := NextUIDValidity{ID: 1}

1330 if err := tx.Get(&nuv); err != nil {

1331 return 0, err

1332 }

1333 v := nuv.Next

1334 nuv.Next++

1335 if err := tx.Update(&nuv); err != nil {

1336 return 0, err

1337 }

1338 return v, nil

1339}

1340

1341// NextModSeq returns the next modification sequence, which is global per account,

1342// over all types.

1343func (a *Account) NextModSeq(tx *bstore.Tx) (ModSeq, error) {

1344 v := SyncState{ID: 1}

1345 if err := tx.Get(&v); err == bstore.ErrAbsent {

1346 // We start assigning from modseq 2. Modseq 0 is not usable, so returned as 1, so

1347 // already used.

1348 // HighestDeletedModSeq is -1 so comparison against the default ModSeq zero value

1349 // makes sense.

1350 v = SyncState{1, 2, -1}

1351 return v.LastModSeq, tx.Insert(&v)

1352 } else if err != nil {

1353 return 0, err

1354 }

1355 v.LastModSeq++

1356 return v.LastModSeq, tx.Update(&v)

1357}

1358

1359func (a *Account) HighestDeletedModSeq(tx *bstore.Tx) (ModSeq, error) {

1360 v := SyncState{ID: 1}

1361 err := tx.Get(&v)

1362 if err == bstore.ErrAbsent {

1363 return 0, nil

1364 }

1365 return v.HighestDeletedModSeq, err

1366}

1367

1368// WithWLock runs fn with account writelock held. Necessary for account/mailbox

1369// modification. For message delivery, a read lock is required.

1370func (a *Account) WithWLock(fn func()) {

1371 a.Lock()

1372 defer a.Unlock()

1373 fn()

1374}

1375

1376// WithRLock runs fn with account read lock held. Needed for message delivery.

1377func (a *Account) WithRLock(fn func()) {

1378 a.RLock()

1379 defer a.RUnlock()

1380 fn()

1381}

1382

1383// DeliverMessage delivers a mail message to the account.

1384//

1385// The message, with msg.MsgPrefix and msgFile combined, must have a header

1386// section. The caller is responsible for adding a header separator to

1387// msg.MsgPrefix if missing from an incoming message.

1388//

1389// If the destination mailbox has the Sent special-use flag, the message is parsed

1390// for its recipients (to/cc/bcc). Their domains are added to Recipients for use in

1391// dmarc reputation.

1392//

1393// If sync is true, the message file and its directory are synced. Should be true

1394// for regular mail delivery, but can be false when importing many messages.

1395//

1396// If updateDiskUsage is true, the account total message size (for quota) is

1397// updated. Callers must check if a message can be added within quota before

1398// calling DeliverMessage.

1399//

1400// If CreateSeq/ModSeq is not set, it is assigned automatically.

1401//

1402// Must be called with account rlock or wlock.

1403//

1404// Caller must broadcast new message.

1405//

1406// Caller must update mailbox counts.

1407func (a *Account) DeliverMessage(log mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, sync, notrain, nothreads, updateDiskUsage bool) error {

1408 if m.Expunged {

1409 return fmt.Errorf("cannot deliver expunged message")

1410 }

1411

1412 mb := Mailbox{ID: m.MailboxID}

1413 if err := tx.Get(&mb); err != nil {

1414 return fmt.Errorf("get mailbox: %w", err)

1415 }

1416 m.UID = mb.UIDNext

1417 mb.UIDNext++

1418 if err := tx.Update(&mb); err != nil {

1419 return fmt.Errorf("updating mailbox nextuid: %w", err)

1420 }

1421

1422 if updateDiskUsage {

1423 du := DiskUsage{ID: 1}

1424 if err := tx.Get(&du); err != nil {

1425 return fmt.Errorf("get disk usage: %v", err)

1426 }

1427 du.MessageSize += m.Size

1428 if err := tx.Update(&du); err != nil {

1429 return fmt.Errorf("update disk usage: %v", err)

1430 }

1431 }

1432

1433 conf, _ := a.Conf()

1434 m.JunkFlagsForMailbox(mb, conf)

1435

1436 mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile.

1437 var part *message.Part

1438 if m.ParsedBuf == nil {

1439 p, err := message.EnsurePart(log.Logger, false, mr, m.Size)

1440 if err != nil {

1441 log.Infox("parsing delivered message", err, slog.String("parse", ""), slog.Int64("message", m.ID))

1442 // We continue, p is still valid.

1443 }

1444 part = &p

1445 buf, err := json.Marshal(part)

1446 if err != nil {

1447 return fmt.Errorf("marshal parsed message: %w", err)

1448 }

1449 m.ParsedBuf = buf

1450 } else {

1451 var p message.Part

1452 if err := json.Unmarshal(m.ParsedBuf, &p); err != nil {

1453 log.Errorx("unmarshal parsed message, continuing", err, slog.String("parse", ""))

1454 } else {

1455 part = &p

1456 }

1457 }

1458

1459 // If we are delivering to the originally intended mailbox, no need to store the mailbox ID again.

1460 if m.MailboxDestinedID != 0 && m.MailboxDestinedID == m.MailboxOrigID {

1461 m.MailboxDestinedID = 0

1462 }

1463 if m.CreateSeq == 0 || m.ModSeq == 0 {

1464 modseq, err := a.NextModSeq(tx)

1465 if err != nil {

1466 return fmt.Errorf("assigning next modseq: %w", err)

1467 }

1468 m.CreateSeq = modseq

1469 m.ModSeq = modseq

1470 }

1471

1472 if part != nil && m.MessageID == "" && m.SubjectBase == "" {

1473 m.PrepareThreading(log, part)

1474 }

1475

1476 // Assign to thread (if upgrade has completed).

1477 noThreadID := nothreads

1478 if m.ThreadID == 0 && !nothreads && part != nil {

1479 select {

1480 case <-a.threadsCompleted:

1481 if a.threadsErr != nil {

1482 log.Info("not assigning threads for new delivery, upgrading to threads failed")

1483 noThreadID = true

1484 } else {

1485 if err := assignThread(log, tx, m, part); err != nil {

1486 return fmt.Errorf("assigning thread: %w", err)

1487 }

1488 }

1489 default:

1490 // note: since we have a write transaction to get here, we can't wait for the

1491 // thread upgrade to finish.

1492 // If we don't assign a threadid the upgrade process will do it.

1493 log.Info("not assigning threads for new delivery, upgrading to threads in progress which will assign this message")

1494 noThreadID = true

1495 }

1496 }

1497

1498 if err := tx.Insert(m); err != nil {

1499 return fmt.Errorf("inserting message: %w", err)

1500 }

1501 if !noThreadID && m.ThreadID == 0 {

1502 m.ThreadID = m.ID

1503 if err := tx.Update(m); err != nil {

1504 return fmt.Errorf("updating message for its own thread id: %w", err)

1505 }

1506 }

1507

1508 // todo: perhaps we should match the recipients based on smtp submission and a matching message-id? we now miss the addresses in bcc's if the mail client doesn't save a message that includes the bcc header in the sent mailbox.

1509 if mb.Sent && part != nil && part.Envelope != nil {

1510 e := part.Envelope

1511 sent := e.Date

1512 if sent.IsZero() {

1513 sent = m.Received

1514 }

1515 if sent.IsZero() {

1516 sent = time.Now()

1517 }

1518 addrs := append(append(e.To, e.CC...), e.BCC...)

1519 for _, addr := range addrs {

1520 if addr.User == "" {

1521 // Would trigger error because Recipient.Localpart must be nonzero. todo: we could allow empty localpart in db, and filter by not using FilterNonzero.

1522 log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", slog.Any("address", addr))

1523 continue

1524 }

1525 d, err := dns.ParseDomain(addr.Host)

1526 if err != nil {

1527 log.Debugx("parsing domain in to/cc/bcc address", err, slog.Any("address", addr))

1528 continue

1529 }

1530 lp, err := smtp.ParseLocalpart(addr.User)

1531 if err != nil {

1532 log.Debugx("parsing localpart in to/cc/bcc address", err, slog.Any("address", addr))

1533 continue

1534 }

1535 mr := Recipient{

1536 MessageID: m.ID,

1537 Localpart: lp.String(),

1538 Domain: d.Name(),

1539 OrgDomain: publicsuffix.Lookup(context.TODO(), log.Logger, d).Name(),

1540 Sent: sent,

1541 }

1542 if err := tx.Insert(&mr); err != nil {

1543 return fmt.Errorf("inserting sent message recipients: %w", err)

1544 }

1545 }

1546 }

1547

1548 msgPath := a.MessagePath(m.ID)

1549 msgDir := filepath.Dir(msgPath)

1550 os.MkdirAll(msgDir, 0770)

1551

1552 // Sync file data to disk.

1553 if sync {

1554 if err := msgFile.Sync(); err != nil {

1555 return fmt.Errorf("fsync message file: %w", err)

1556 }

1557 }

1558

1559 if err := moxio.LinkOrCopy(log, msgPath, msgFile.Name(), &moxio.AtReader{R: msgFile}, true); err != nil {

1560 return fmt.Errorf("linking/copying message to new file: %w", err)

1561 }

1562

1563 if sync {

1564 if err := moxio.SyncDir(log, msgDir); err != nil {

1565 xerr := os.Remove(msgPath)

1566 log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath))

1567 return fmt.Errorf("sync directory: %w", err)

1568 }

1569 }

1570

1571 if !notrain && m.NeedsTraining() {

1572 l := []Message{*m}

1573 if err := a.RetrainMessages(context.TODO(), log, tx, l, false); err != nil {

1574 xerr := os.Remove(msgPath)

1575 log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath))

1576 return fmt.Errorf("training junkfilter: %w", err)

1577 }

1578 *m = l[0]

1579 }

1580

1581 return nil

1582}

1583

1584// SetPassword saves a new password for this account. This password is used for

1585// IMAP, SMTP (submission) sessions and the HTTP account web page.

1586func (a *Account) SetPassword(log mlog.Log, password string) error {

1587 password, err := precis.OpaqueString.String(password)

1588 if err != nil {

1589 return fmt.Errorf(`password not allowed by "precis"`)

1590 }

1591

1592 if len(password) < 8 {

1593 // We actually check for bytes...

1594 return fmt.Errorf("password must be at least 8 characters long")

1595 }

1596

1597 hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)

1598 if err != nil {

1599 return fmt.Errorf("generating password hash: %w", err)

1600 }

1601

1602 err = a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1603 if _, err := bstore.QueryTx[Password](tx).Delete(); err != nil {

1604 return fmt.Errorf("deleting existing password: %v", err)

1605 }

1606 var pw Password

1607 pw.Hash = string(hash)

1608

1609 // CRAM-MD5 calculates an HMAC-MD5, with the password as key, over a per-attempt

1610 // unique text that includes a timestamp. HMAC performs two hashes. Both times, the

1611 // first block is based on the key/password. We hash those first blocks now, and

1612 // store the hash state in the database. When we actually authenticate, we'll

1613 // complete the HMAC by hashing only the text. We cannot store crypto/hmac's hash,

1614 // because it does not expose its internal state and isn't a BinaryMarshaler.

1615 // ../rfc/2104:121

1616 pw.CRAMMD5.Ipad = md5.New()

1617 pw.CRAMMD5.Opad = md5.New()

1618 key := []byte(password)

1619 if len(key) > 64 {

1620 t := md5.Sum(key)

1621 key = t[:]

1622 }

1623 ipad := make([]byte, md5.BlockSize)

1624 opad := make([]byte, md5.BlockSize)

1625 copy(ipad, key)

1626 copy(opad, key)

1627 for i := range ipad {

1628 ipad[i] ^= 0x36

1629 opad[i] ^= 0x5c

1630 }

1631 pw.CRAMMD5.Ipad.Write(ipad)

1632 pw.CRAMMD5.Opad.Write(opad)

1633

1634 pw.SCRAMSHA1.Salt = scram.MakeRandom()

1635 pw.SCRAMSHA1.Iterations = 2 * 4096

1636 pw.SCRAMSHA1.SaltedPassword = scram.SaltPassword(sha1.New, password, pw.SCRAMSHA1.Salt, pw.SCRAMSHA1.Iterations)

1637

1638 pw.SCRAMSHA256.Salt = scram.MakeRandom()

1639 pw.SCRAMSHA256.Iterations = 4096

1640 pw.SCRAMSHA256.SaltedPassword = scram.SaltPassword(sha256.New, password, pw.SCRAMSHA256.Salt, pw.SCRAMSHA256.Iterations)

1641

1642 if err := tx.Insert(&pw); err != nil {

1643 return fmt.Errorf("inserting new password: %v", err)

1644 }

1645

1646 return sessionRemoveAll(context.TODO(), log, tx, a.Name)

1647 })

1648 if err == nil {

1649 log.Info("new password set for account", slog.String("account", a.Name))

1650 }

1651 return err

1652}

1653

1654// Subjectpass returns the signing key for use with subjectpass for the given

1655// email address with canonical localpart.

1656func (a *Account) Subjectpass(email string) (key string, err error) {

1657 return key, a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1658 v := Subjectpass{Email: email}

1659 err := tx.Get(&v)

1660 if err == nil {

1661 key = v.Key

1662 return nil

1663 }

1664 if !errors.Is(err, bstore.ErrAbsent) {

1665 return fmt.Errorf("get subjectpass key from accounts database: %w", err)

1666 }

1667 key = ""

1668 const chars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"

1669 buf := make([]byte, 16)

1670 if _, err := cryptorand.Read(buf); err != nil {

1671 return err

1672 }

1673 for _, b := range buf {

1674 key += string(chars[int(b)%len(chars)])

1675 }

1676 v.Key = key

1677 return tx.Insert(&v)

1678 })

1679}

1680

1681// Ensure mailbox is present in database, adding records for the mailbox and its

1682// parents if they aren't present.

1683//

1684// If subscribe is true, any mailboxes that were created will also be subscribed to.

1685// Caller must hold account wlock.

1686// Caller must propagate changes if any.

1687func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool) (mb Mailbox, changes []Change, rerr error) {

1688 if norm.NFC.String(name) != name {

1689 return Mailbox{}, nil, fmt.Errorf("mailbox name not normalized")

1690 }

1691

1692 // Quick sanity check.

1693 if strings.EqualFold(name, "inbox") && name != "Inbox" {

1694 return Mailbox{}, nil, fmt.Errorf("bad casing for inbox")

1695 }

1696

1697 elems := strings.Split(name, "/")

1698 q := bstore.QueryTx[Mailbox](tx)

1699 q.FilterFn(func(mb Mailbox) bool {

1700 return mb.Name == elems[0] || strings.HasPrefix(mb.Name, elems[0]+"/")

1701 })

1702 l, err := q.List()

1703 if err != nil {

1704 return Mailbox{}, nil, fmt.Errorf("list mailboxes: %v", err)

1705 }

1706

1707 mailboxes := map[string]Mailbox{}

1708 for _, xmb := range l {

1709 mailboxes[xmb.Name] = xmb

1710 }

1711

1712 p := ""

1713 for _, elem := range elems {

1714 if p != "" {

1715 p += "/"

1716 }

1717 p += elem

1718 var ok bool

1719 mb, ok = mailboxes[p]

1720 if ok {

1721 continue

1722 }

1723 uidval, err := a.NextUIDValidity(tx)

1724 if err != nil {

1725 return Mailbox{}, nil, fmt.Errorf("next uid validity: %v", err)

1726 }

1727 mb = Mailbox{

1728 Name: p,

1729 UIDValidity: uidval,

1730 UIDNext: 1,

1731 HaveCounts: true,

1732 }

1733 err = tx.Insert(&mb)

1734 if err != nil {

1735 return Mailbox{}, nil, fmt.Errorf("creating new mailbox: %v", err)

1736 }

1737

1738 var flags []string

1739 if subscribe {

1740 if tx.Get(&Subscription{p}) != nil {

1741 err := tx.Insert(&Subscription{p})

1742 if err != nil {

1743 return Mailbox{}, nil, fmt.Errorf("subscribing to mailbox: %v", err)

1744 }

1745 }

1746 flags = []string{`\Subscribed`}

1747 }

1748 changes = append(changes, ChangeAddMailbox{mb, flags})

1749 }

1750 return mb, changes, nil

1751}

1752

1753// MailboxExists checks if mailbox exists.

1754// Caller must hold account rlock.

1755func (a *Account) MailboxExists(tx *bstore.Tx, name string) (bool, error) {

1756 q := bstore.QueryTx[Mailbox](tx)

1757 q.FilterEqual("Name", name)

1758 return q.Exists()

1759}

1760

1761// MailboxFind finds a mailbox by name, returning a nil mailbox and nil error if mailbox does not exist.

1762func (a *Account) MailboxFind(tx *bstore.Tx, name string) (*Mailbox, error) {

1763 q := bstore.QueryTx[Mailbox](tx)

1764 q.FilterEqual("Name", name)

1765 mb, err := q.Get()

1766 if err == bstore.ErrAbsent {

1767 return nil, nil

1768 }

1769 if err != nil {

1770 return nil, fmt.Errorf("looking up mailbox: %w", err)

1771 }

1772 return &mb, nil

1773}

1774

1775// SubscriptionEnsure ensures a subscription for name exists. The mailbox does not

1776// have to exist. Any parents are not automatically subscribed.

1777// Changes are returned and must be broadcasted by the caller.

1778func (a *Account) SubscriptionEnsure(tx *bstore.Tx, name string) ([]Change, error) {

1779 if err := tx.Get(&Subscription{name}); err == nil {

1780 return nil, nil

1781 }

1782

1783 if err := tx.Insert(&Subscription{name}); err != nil {

1784 return nil, fmt.Errorf("inserting subscription: %w", err)

1785 }

1786

1787 q := bstore.QueryTx[Mailbox](tx)

1788 q.FilterEqual("Name", name)

1789 _, err := q.Get()

1790 if err == nil {

1791 return []Change{ChangeAddSubscription{name, nil}}, nil

1792 } else if err != bstore.ErrAbsent {

1793 return nil, fmt.Errorf("looking up mailbox for subscription: %w", err)

1794 }

1795 return []Change{ChangeAddSubscription{name, []string{`\NonExistent`}}}, nil

1796}

1797

1798// MessageRuleset returns the first ruleset (if any) that matches the message

1799// represented by msgPrefix and msgFile, with smtp and validation fields from m.

1800func MessageRuleset(log mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset {

1801 if len(dest.Rulesets) == 0 {

1802 return nil

1803 }

1804

1805 mr := FileMsgReader(msgPrefix, msgFile) // We don't close, it would close the msgFile.

1806 p, err := message.Parse(log.Logger, false, mr)

1807 if err != nil {

1808 log.Errorx("parsing message for evaluating rulesets, continuing with headers", err, slog.String("parse", ""))

1809 // note: part is still set.

1810 }

1811 // todo optimize: only parse header if needed for rulesets. and probably reuse an earlier parsing.

1812 header, err := p.Header()

1813 if err != nil {

1814 log.Errorx("parsing message headers for evaluating rulesets, delivering to default mailbox", err, slog.String("parse", ""))

1815 // todo: reject message?

1816 return nil

1817 }

1818

1819ruleset:

1820 for _, rs := range dest.Rulesets {

1821 if rs.SMTPMailFromRegexpCompiled != nil {

1822 if !rs.SMTPMailFromRegexpCompiled.MatchString(m.MailFrom) {

1823 continue ruleset

1824 }

1825 }

1826 if rs.MsgFromRegexpCompiled != nil {

1827 if m.MsgFromLocalpart == "" && m.MsgFromDomain == "" || !rs.MsgFromRegexpCompiled.MatchString(m.MsgFromLocalpart.String()+"@"+m.MsgFromDomain) {

1828 continue ruleset

1829 }

1830 }

1831

1832 if !rs.VerifiedDNSDomain.IsZero() {

1833 d := rs.VerifiedDNSDomain.Name()

1834 suffix := "." + d

1835 matchDomain := func(s string) bool {

1836 return s == d || strings.HasSuffix(s, suffix)

1837 }

1838 var ok bool

1839 if m.EHLOValidated && matchDomain(m.EHLODomain) {

1840 ok = true

1841 }

1842 if m.MailFromValidated && matchDomain(m.MailFromDomain) {

1843 ok = true

1844 }

1845 for _, d := range m.DKIMDomains {

1846 if matchDomain(d) {

1847 ok = true

1848 break

1849 }

1850 }

1851 if !ok {

1852 continue ruleset

1853 }

1854 }

1855

1856 header:

1857 for _, t := range rs.HeadersRegexpCompiled {

1858 for k, vl := range header {

1859 k = strings.ToLower(k)

1860 if !t[0].MatchString(k) {

1861 continue

1862 }

1863 for _, v := range vl {

1864 v = strings.ToLower(strings.TrimSpace(v))

1865 if t[1].MatchString(v) {

1866 continue header

1867 }

1868 }

1869 }

1870 continue ruleset

1871 }

1872 return &rs

1873 }

1874 return nil

1875}

1876

1877// MessagePath returns the file system path of a message.

1878func (a *Account) MessagePath(messageID int64) string {

1879 return strings.Join(append([]string{a.Dir, "msg"}, messagePathElems(messageID)...), string(filepath.Separator))

1880}

1881

1882// MessageReader opens a message for reading, transparently combining the

1883// message prefix with the original incoming message.

1884func (a *Account) MessageReader(m Message) *MsgReader {

1885 return &MsgReader{prefix: m.MsgPrefix, path: a.MessagePath(m.ID), size: m.Size}

1886}

1887

1888// DeliverDestination delivers an email to dest, based on the configured rulesets.

1889//

1890// Returns ErrOverQuota when account would be over quota after adding message.

1891//

1892// Caller must hold account wlock (mailbox may be created).

1893// Message delivery, possible mailbox creation, and updated mailbox counts are

1894// broadcasted.

1895func (a *Account) DeliverDestination(log mlog.Log, dest config.Destination, m *Message, msgFile *os.File) error {

1896 var mailbox string

1897 rs := MessageRuleset(log, dest, m, m.MsgPrefix, msgFile)

1898 if rs != nil {

1899 mailbox = rs.Mailbox

1900 } else if dest.Mailbox == "" {

1901 mailbox = "Inbox"

1902 } else {

1903 mailbox = dest.Mailbox

1904 }

1905 return a.DeliverMailbox(log, mailbox, m, msgFile)

1906}

1907

1908// DeliverMailbox delivers an email to the specified mailbox.

1909//

1910// Returns ErrOverQuota when account would be over quota after adding message.

1911//

1912// Caller must hold account wlock (mailbox may be created).

1913// Message delivery, possible mailbox creation, and updated mailbox counts are

1914// broadcasted.

1915func (a *Account) DeliverMailbox(log mlog.Log, mailbox string, m *Message, msgFile *os.File) error {

1916 var changes []Change

1917 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1918 if ok, _, err := a.CanAddMessageSize(tx, m.Size); err != nil {

1919 return err

1920 } else if !ok {

1921 return ErrOverQuota

1922 }

1923

1924 mb, chl, err := a.MailboxEnsure(tx, mailbox, true)

1925 if err != nil {

1926 return fmt.Errorf("ensuring mailbox: %w", err)

1927 }

1928 m.MailboxID = mb.ID

1929 m.MailboxOrigID = mb.ID

1930

1931 // Update count early, DeliverMessage will update mb too and we don't want to fetch

1932 // it again before updating.

1933 mb.MailboxCounts.Add(m.MailboxCounts())

1934 if err := tx.Update(&mb); err != nil {

1935 return fmt.Errorf("updating mailbox for delivery: %w", err)

1936 }

1937

1938 if err := a.DeliverMessage(log, tx, m, msgFile, true, false, false, true); err != nil {

1939 return err

1940 }

1941

1942 changes = append(changes, chl...)

1943 changes = append(changes, m.ChangeAddUID(), mb.ChangeCounts())

1944 return nil

1945 })

1946 // todo: if rename succeeded but transaction failed, we should remove the file.

1947 if err != nil {

1948 return err

1949 }

1950

1951 BroadcastChanges(a, changes)

1952 return nil

1953}

1954

1955// TidyRejectsMailbox removes old reject emails, and returns whether there is space for a new delivery.

1956//

1957// Caller most hold account wlock.

1958// Changes are broadcasted.

1959func (a *Account) TidyRejectsMailbox(log mlog.Log, rejectsMailbox string) (hasSpace bool, rerr error) {

1960 var changes []Change

1961

1962 var remove []Message

1963 defer func() {

1964 for _, m := range remove {

1965 p := a.MessagePath(m.ID)

1966 err := os.Remove(p)

1967 log.Check(err, "removing rejects message file", slog.String("path", p))

1968 }

1969 }()

1970

1971 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1972 mb, err := a.MailboxFind(tx, rejectsMailbox)

1973 if err != nil {

1974 return fmt.Errorf("finding mailbox: %w", err)

1975 }

1976 if mb == nil {

1977 // No messages have been delivered yet.

1978 hasSpace = true

1979 return nil

1980 }

1981

1982 // Gather old messages to remove.

1983 old := time.Now().Add(-14 * 24 * time.Hour)

1984 qdel := bstore.QueryTx[Message](tx)

1985 qdel.FilterNonzero(Message{MailboxID: mb.ID})

1986 qdel.FilterEqual("Expunged", false)

1987 qdel.FilterLess("Received", old)

1988 remove, err = qdel.List()

1989 if err != nil {

1990 return fmt.Errorf("listing old messages: %w", err)

1991 }

1992

1993 changes, err = a.rejectsRemoveMessages(context.TODO(), log, tx, mb, remove)

1994 if err != nil {

1995 return fmt.Errorf("removing messages: %w", err)

1996 }

1997

1998 // We allow up to n messages.

1999 qcount := bstore.QueryTx[Message](tx)

2000 qcount.FilterNonzero(Message{MailboxID: mb.ID})

2001 qcount.FilterEqual("Expunged", false)

2002 qcount.Limit(1000)

2003 n, err := qcount.Count()

2004 if err != nil {

2005 return fmt.Errorf("counting rejects: %w", err)

2006 }

2007 hasSpace = n < 1000

2008

2009 return nil

2010 })

2011 if err != nil {

2012 remove = nil // Don't remove files on failure.

2013 return false, err

2014 }

2015

2016 BroadcastChanges(a, changes)

2017

2018 return hasSpace, nil

2019}

2020

2021func (a *Account) rejectsRemoveMessages(ctx context.Context, log mlog.Log, tx *bstore.Tx, mb *Mailbox, l []Message) ([]Change, error) {

2022 if len(l) == 0 {

2023 return nil, nil

2024 }

2025 ids := make([]int64, len(l))

2026 anyids := make([]any, len(l))

2027 for i, m := range l {

2028 ids[i] = m.ID

2029 anyids[i] = m.ID

2030 }

2031

2032 // Remove any message recipients. Should not happen, but a user can move messages

2033 // from a Sent mailbox to the rejects mailbox...

2034 qdmr := bstore.QueryTx[Recipient](tx)

2035 qdmr.FilterEqual("MessageID", anyids...)

2036 if _, err := qdmr.Delete(); err != nil {

2037 return nil, fmt.Errorf("deleting from message recipient: %w", err)

2038 }

2039

2040 // Assign new modseq.

2041 modseq, err := a.NextModSeq(tx)

2042 if err != nil {

2043 return nil, fmt.Errorf("assign next modseq: %w", err)

2044 }

2045

2046 // Expunge the messages.

2047 qx := bstore.QueryTx[Message](tx)

2048 qx.FilterIDs(ids)

2049 var expunged []Message

2050 qx.Gather(&expunged)

2051 if _, err := qx.UpdateNonzero(Message{ModSeq: modseq, Expunged: true}); err != nil {

2052 return nil, fmt.Errorf("expunging messages: %w", err)

2053 }

2054

2055 var totalSize int64

2056 for _, m := range expunged {

2057 m.Expunged = false // Was set by update, but would cause wrong count.

2058 mb.MailboxCounts.Sub(m.MailboxCounts())

2059 totalSize += m.Size

2060 }

2061 if err := tx.Update(mb); err != nil {

2062 return nil, fmt.Errorf("updating mailbox counts: %w", err)

2063 }

2064 if err := a.AddMessageSize(log, tx, -totalSize); err != nil {

2065 return nil, fmt.Errorf("updating disk usage: %w", err)

2066 }

2067

2068 // Mark as neutral and train so junk filter gets untrained with these (junk) messages.

2069 for i := range expunged {

2070 expunged[i].Junk = false

2071 expunged[i].Notjunk = false

2072 }

2073 if err := a.RetrainMessages(ctx, log, tx, expunged, true); err != nil {

2074 return nil, fmt.Errorf("retraining expunged messages: %w", err)

2075 }

2076

2077 changes := make([]Change, len(l), len(l)+1)

2078 for i, m := range l {

2079 changes[i] = ChangeRemoveUIDs{mb.ID, []UID{m.UID}, modseq}

2080 }

2081 changes = append(changes, mb.ChangeCounts())

2082 return changes, nil

2083}

2084

2085// RejectsRemove removes a message from the rejects mailbox if present.

2086// Caller most hold account wlock.

2087// Changes are broadcasted.

2088func (a *Account) RejectsRemove(log mlog.Log, rejectsMailbox, messageID string) error {

2089 var changes []Change

2090

2091 var remove []Message

2092 defer func() {

2093 for _, m := range remove {

2094 p := a.MessagePath(m.ID)

2095 err := os.Remove(p)

2096 log.Check(err, "removing rejects message file", slog.String("path", p))

2097 }

2098 }()

2099

2100 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

2101 mb, err := a.MailboxFind(tx, rejectsMailbox)

2102 if err != nil {

2103 return fmt.Errorf("finding mailbox: %w", err)

2104 }

2105 if mb == nil {

2106 return nil

2107 }

2108

2109 q := bstore.QueryTx[Message](tx)

2110 q.FilterNonzero(Message{MailboxID: mb.ID, MessageID: messageID})

2111 q.FilterEqual("Expunged", false)

2112 remove, err = q.List()

2113 if err != nil {

2114 return fmt.Errorf("listing messages to remove: %w", err)

2115 }

2116

2117 changes, err = a.rejectsRemoveMessages(context.TODO(), log, tx, mb, remove)

2118 if err != nil {

2119 return fmt.Errorf("removing messages: %w", err)

2120 }

2121

2122 return nil

2123 })

2124 if err != nil {

2125 remove = nil // Don't remove files on failure.

2126 return err

2127 }

2128

2129 BroadcastChanges(a, changes)

2130

2131 return nil

2132}

2133

2134// AddMessageSize adjusts the DiskUsage.MessageSize by size.

2135func (a *Account) AddMessageSize(log mlog.Log, tx *bstore.Tx, size int64) error {

2136 du := DiskUsage{ID: 1}

2137 if err := tx.Get(&du); err != nil {

2138 return fmt.Errorf("get diskusage: %v", err)

2139 }

2140 du.MessageSize += size

2141 if du.MessageSize < 0 {

2142 log.Error("negative total message size", slog.Int64("delta", size), slog.Int64("newtotalsize", du.MessageSize))

2143 }

2144 if err := tx.Update(&du); err != nil {

2145 return fmt.Errorf("update total message size: %v", err)

2146 }

2147 return nil

2148}

2149

2150// QuotaMessageSize returns the effective maximum total message size for an

2151// account. Returns 0 if there is no maximum.

2152func (a *Account) QuotaMessageSize() int64 {

2153 conf, _ := a.Conf()

2154 size := conf.QuotaMessageSize

2155 if size == 0 {

2156 size = mox.Conf.Static.QuotaMessageSize

2157 }

2158 if size < 0 {

2159 size = 0

2160 }

2161 return size

2162}

2163

2164// CanAddMessageSize checks if a message of size bytes can be added, depending on

2165// total message size and configured quota for account.

2166func (a *Account) CanAddMessageSize(tx *bstore.Tx, size int64) (ok bool, maxSize int64, err error) {

2167 maxSize = a.QuotaMessageSize()

2168 if maxSize <= 0 {

2169 return true, 0, nil

2170 }

2171

2172 du := DiskUsage{ID: 1}

2173 if err := tx.Get(&du); err != nil {

2174 return false, maxSize, fmt.Errorf("get diskusage: %v", err)

2175 }

2176 return du.MessageSize+size <= maxSize, maxSize, nil

2177}

2178

2179// We keep a cache of recent successful authentications, so we don't have to bcrypt successful calls each time.

2180var authCache = struct {

2181 sync.Mutex

2182 success map[authKey]string

2183}{

2184 success: map[authKey]string{},

2185}

2186

2187type authKey struct {

2188 email, hash string

2189}

2190

2191// StartAuthCache starts a goroutine that regularly clears the auth cache.

2192func StartAuthCache() {

2193 go manageAuthCache()

2194}

2195

2196func manageAuthCache() {

2197 for {

2198 authCache.Lock()

2199 authCache.success = map[authKey]string{}

2200 authCache.Unlock()

2201 time.Sleep(15 * time.Minute)

2202 }

2203}

2204

2205// OpenEmailAuth opens an account given an email address and password.

2206//

2207// The email address may contain a catchall separator.

2208func OpenEmailAuth(log mlog.Log, email string, password string) (acc *Account, rerr error) {

2209 password, err := precis.OpaqueString.String(password)

2210 if err != nil {

2211 return nil, ErrUnknownCredentials

2212 }

2213

2214 acc, _, rerr = OpenEmail(log, email)

2215 if rerr != nil {

2216 return

2217 }

2218

2219 defer func() {

2220 if rerr != nil && acc != nil {

2221 err := acc.Close()

2222 log.Check(err, "closing account after open auth failure")

2223 acc = nil

2224 }

2225 }()

2226

2227 pw, err := bstore.QueryDB[Password](context.TODO(), acc.DB).Get()

2228 if err != nil {

2229 if err == bstore.ErrAbsent {

2230 return acc, ErrUnknownCredentials

2231 }

2232 return acc, fmt.Errorf("looking up password: %v", err)

2233 }

2234 authCache.Lock()

2235 ok := len(password) >= 8 && authCache.success[authKey{email, pw.Hash}] == password

2236 authCache.Unlock()

2237 if ok {

2238 return

2239 }

2240 if err := bcrypt.CompareHashAndPassword([]byte(pw.Hash), []byte(password)); err != nil {

2241 rerr = ErrUnknownCredentials

2242 } else {

2243 authCache.Lock()

2244 authCache.success[authKey{email, pw.Hash}] = password

2245 authCache.Unlock()

2246 }

2247 return

2248}

2249

2250// OpenEmail opens an account given an email address.

2251//

2252// The email address may contain a catchall separator.

2253func OpenEmail(log mlog.Log, email string) (*Account, config.Destination, error) {

2254 addr, err := smtp.ParseAddress(email)

2255 if err != nil {

2256 return nil, config.Destination{}, fmt.Errorf("%w: %v", ErrUnknownCredentials, err)

2257 }

2258 accountName, _, _, dest, err := mox.LookupAddress(addr.Localpart, addr.Domain, false, false)

2259 if err != nil && (errors.Is(err, mox.ErrAddressNotFound) || errors.Is(err, mox.ErrDomainNotFound)) {

2260 return nil, config.Destination{}, ErrUnknownCredentials

2261 } else if err != nil {

2262 return nil, config.Destination{}, fmt.Errorf("looking up address: %v", err)

2263 }

2264 acc, err := OpenAccount(log, accountName)

2265 if err != nil {

2266 return nil, config.Destination{}, err

2267 }

2268 return acc, dest, nil

2269}

2270

2271// 64 characters, must be power of 2 for MessagePath

2272const msgDirChars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ-_"

2273

2274// MessagePath returns the filename of the on-disk filename, relative to the

2275// containing directory such as <account>/msg or queue.

2276// Returns names like "AB/1".

2277func MessagePath(messageID int64) string {

2278 return strings.Join(messagePathElems(messageID), string(filepath.Separator))

2279}

2280

2281// messagePathElems returns the elems, for a single join without intermediate

2282// string allocations.

2283func messagePathElems(messageID int64) []string {

2284 v := messageID >> 13 // 8k files per directory.

2285 dir := ""

2286 for {

2287 dir += string(msgDirChars[int(v)&(len(msgDirChars)-1)])

2288 v >>= 6

2289 if v == 0 {

2290 break

2291 }

2292 }

2293 return []string{dir, strconv.FormatInt(messageID, 10)}

2294}

2295

2296// Set returns a copy of f, with each flag that is true in mask set to the

2297// value from flags.

2298func (f Flags) Set(mask, flags Flags) Flags {

2299 set := func(d *bool, m, v bool) {

2300 if m {

2301 *d = v

2302 }

2303 }

2304 r := f

2305 set(&r.Seen, mask.Seen, flags.Seen)

2306 set(&r.Answered, mask.Answered, flags.Answered)

2307 set(&r.Flagged, mask.Flagged, flags.Flagged)

2308 set(&r.Forwarded, mask.Forwarded, flags.Forwarded)

2309 set(&r.Junk, mask.Junk, flags.Junk)

2310 set(&r.Notjunk, mask.Notjunk, flags.Notjunk)

2311 set(&r.Deleted, mask.Deleted, flags.Deleted)

2312 set(&r.Draft, mask.Draft, flags.Draft)

2313 set(&r.Phishing, mask.Phishing, flags.Phishing)

2314 set(&r.MDNSent, mask.MDNSent, flags.MDNSent)

2315 return r

2316}

2317

2318// Changed returns a mask of flags that have been between f and other.

2319func (f Flags) Changed(other Flags) (mask Flags) {

2320 mask.Seen = f.Seen != other.Seen

2321 mask.Answered = f.Answered != other.Answered

2322 mask.Flagged = f.Flagged != other.Flagged

2323 mask.Forwarded = f.Forwarded != other.Forwarded

2324 mask.Junk = f.Junk != other.Junk

2325 mask.Notjunk = f.Notjunk != other.Notjunk

2326 mask.Deleted = f.Deleted != other.Deleted

2327 mask.Draft = f.Draft != other.Draft

2328 mask.Phishing = f.Phishing != other.Phishing

2329 mask.MDNSent = f.MDNSent != other.MDNSent

2330 return

2331}

2332

2333// Strings returns the flags that are set in their string form.

2334func (f Flags) Strings() []string {

2335 fields := []struct {

2336 word string

2337 have bool

2338 }{

2339 {`$forwarded`, f.Forwarded},

2340 {`$junk`, f.Junk},

2341 {`$mdnsent`, f.MDNSent},

2342 {`$notjunk`, f.Notjunk},

2343 {`$phishing`, f.Phishing},

2344 {`\answered`, f.Answered},

2345 {`\deleted`, f.Deleted},

2346 {`\draft`, f.Draft},

2347 {`\flagged`, f.Flagged},

2348 {`\seen`, f.Seen},

2349 }

2350 var l []string

2351 for _, fh := range fields {

2352 if fh.have {

2353 l = append(l, fh.word)

2354 }

2355 }

2356 return l

2357}

2358

2359var systemWellKnownFlags = map[string]bool{

2360 `\answered`: true,

2361 `\flagged`: true,

2362 `\deleted`: true,

2363 `\seen`: true,

2364 `\draft`: true,

2365 `$junk`: true,

2366 `$notjunk`: true,

2367 `$forwarded`: true,

2368 `$phishing`: true,

2369 `$mdnsent`: true,

2370}

2371

2372// ParseFlagsKeywords parses a list of textual flags into system/known flags, and

2373// other keywords. Keywords are lower-cased and sorted and check for valid syntax.

2374func ParseFlagsKeywords(l []string) (flags Flags, keywords []string, rerr error) {

2375 fields := map[string]*bool{

2376 `\answered`: &flags.Answered,

2377 `\flagged`: &flags.Flagged,

2378 `\deleted`: &flags.Deleted,

2379 `\seen`: &flags.Seen,

2380 `\draft`: &flags.Draft,

2381 `$junk`: &flags.Junk,

2382 `$notjunk`: &flags.Notjunk,

2383 `$forwarded`: &flags.Forwarded,

2384 `$phishing`: &flags.Phishing,

2385 `$mdnsent`: &flags.MDNSent,

2386 }

2387 seen := map[string]bool{}

2388 for _, f := range l {

2389 f = strings.ToLower(f)

2390 if field, ok := fields[f]; ok {

2391 *field = true

2392 } else if seen[f] {

2393 if mox.Pedantic {

2394 return Flags{}, nil, fmt.Errorf("duplicate keyword %s", f)

2395 }

2396 } else {

2397 if err := CheckKeyword(f); err != nil {

2398 return Flags{}, nil, fmt.Errorf("invalid keyword %s", f)

2399 }

2400 keywords = append(keywords, f)

2401 seen[f] = true

2402 }

2403 }

2404 sort.Strings(keywords)

2405 return flags, keywords, nil

2406}

2407

2408// RemoveKeywords removes keywords from l, returning whether any modifications were

2409// made, and a slice, a new slice in case of modifications. Keywords must have been

2410// validated earlier, e.g. through ParseFlagKeywords or CheckKeyword. Should only

2411// be used with valid keywords, not with system flags like \Seen.

2412func RemoveKeywords(l, remove []string) ([]string, bool) {

2413 var copied bool

2414 var changed bool

2415 for _, k := range remove {

2416 if i := slices.Index(l, k); i >= 0 {

2417 if !copied {

2418 l = append([]string{}, l...)

2419 copied = true

2420 }

2421 copy(l[i:], l[i+1:])

2422 l = l[:len(l)-1]

2423 changed = true

2424 }

2425 }

2426 return l, changed

2427}

2428

2429// MergeKeywords adds keywords from add into l, returning whether it added any

2430// keyword, and the slice with keywords, a new slice if modifications were made.

2431// Keywords are only added if they aren't already present. Should only be used with

2432// keywords, not with system flags like \Seen.

2433func MergeKeywords(l, add []string) ([]string, bool) {

2434 var copied bool

2435 var changed bool

2436 for _, k := range add {

2437 if !slices.Contains(l, k) {

2438 if !copied {

2439 l = append([]string{}, l...)

2440 copied = true

2441 }

2442 l = append(l, k)

2443 changed = true

2444 }

2445 }

2446 if changed {

2447 sort.Strings(l)

2448 }

2449 return l, changed

2450}

2451

2452// CheckKeyword returns an error if kw is not a valid keyword. Kw should

2453// already be in lower-case.

2454func CheckKeyword(kw string) error {

2455 if kw == "" {

2456 return fmt.Errorf("keyword cannot be empty")

2457 }

2458 if systemWellKnownFlags[kw] {

2459 return fmt.Errorf("cannot use well-known flag as keyword")

2460 }

2461 for _, c := range kw {

2462 // ../rfc/9051:6334

2463 if c <= ' ' || c > 0x7e || c >= 'A' && c <= 'Z' || strings.ContainsRune(`(){%*"\]`, c) {

2464 return errors.New(`not a valid keyword, must be lower-case ascii without spaces and without any of these characters: (){%*"\]`)

2465 }

2466 }

2467 return nil

2468}

2469

2470// SendLimitReached checks whether sending a message to recipients would reach

2471// the limit of outgoing messages for the account. If so, the message should

2472// not be sent. If the returned numbers are >= 0, the limit was reached and the

2473// values are the configured limits.

2474//

2475// To limit damage to the internet and our reputation in case of account

2476// compromise, we limit the max number of messages sent in a 24 hour window, both

2477// total number of messages and number of first-time recipients.

2478func (a *Account) SendLimitReached(tx *bstore.Tx, recipients []smtp.Path) (msglimit, rcptlimit int, rerr error) {

2479 conf, _ := a.Conf()

2480 msgmax := conf.MaxOutgoingMessagesPerDay

2481 if msgmax == 0 {

2482 // For human senders, 1000 recipients in a day is quite a lot.

2483 msgmax = 1000

2484 }

2485 rcptmax := conf.MaxFirstTimeRecipientsPerDay

2486 if rcptmax == 0 {

2487 // Human senders may address a new human-sized list of people once in a while. In

2488 // case of a compromise, a spammer will probably try to send to many new addresses.

2489 rcptmax = 200

2490 }

2491

2492 rcpts := map[string]time.Time{}

2493 n := 0

2494 err := bstore.QueryTx[Outgoing](tx).FilterGreater("Submitted", time.Now().Add(-24*time.Hour)).ForEach(func(o Outgoing) error {

2495 n++

2496 if rcpts[o.Recipient].IsZero() || o.Submitted.Before(rcpts[o.Recipient]) {

2497 rcpts[o.Recipient] = o.Submitted

2498 }

2499 return nil

2500 })

2501 if err != nil {

2502 return -1, -1, fmt.Errorf("querying message recipients in past 24h: %w", err)

2503 }

2504 if n+len(recipients) > msgmax {

2505 return msgmax, -1, nil

2506 }

2507

2508 // Only check if max first-time recipients is reached if there are enough messages

2509 // to trigger the limit.

2510 if n+len(recipients) < rcptmax {

2511 return -1, -1, nil

2512 }

2513

2514 isFirstTime := func(rcpt string, before time.Time) (bool, error) {

2515 exists, err := bstore.QueryTx[Outgoing](tx).FilterNonzero(Outgoing{Recipient: rcpt}).FilterLess("Submitted", before).Exists()

2516 return !exists, err

2517 }

2518

2519 firsttime := 0

2520 now := time.Now()

2521 for _, r := range recipients {

2522 if first, err := isFirstTime(r.XString(true), now); err != nil {

2523 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

2524 } else if first {

2525 firsttime++

2526 }

2527 }

2528 for r, t := range rcpts {

2529 if first, err := isFirstTime(r, t); err != nil {

2530 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

2531 } else if first {

2532 firsttime++

2533 }

2534 }

2535 if firsttime > rcptmax {

2536 return -1, rcptmax, nil

2537 }

2538 return -1, -1, nil

2539}

2540

2541// MailboxCreate creates a new mailbox, including any missing parent mailboxes,

2542// the total list of created mailboxes is returned in created. On success, if

2543// exists is false and rerr nil, the changes must be broadcasted by the caller.

2544//

2545// Name must be in normalized form.

2546func (a *Account) MailboxCreate(tx *bstore.Tx, name string) (changes []Change, created []string, exists bool, rerr error) {

2547 elems := strings.Split(name, "/")

2548 var p string

2549 for i, elem := range elems {

2550 if i > 0 {

2551 p += "/"

2552 }

2553 p += elem

2554 exists, err := a.MailboxExists(tx, p)

2555 if err != nil {

2556 return nil, nil, false, fmt.Errorf("checking if mailbox exists")

2557 }

2558 if exists {

2559 if i == len(elems)-1 {

2560 return nil, nil, true, fmt.Errorf("mailbox already exists")

2561 }

2562 continue

2563 }

2564 _, nchanges, err := a.MailboxEnsure(tx, p, true)

2565 if err != nil {

2566 return nil, nil, false, fmt.Errorf("ensuring mailbox exists")

2567 }

2568 changes = append(changes, nchanges...)

2569 created = append(created, p)

2570 }

2571 return changes, created, false, nil

2572}

2573

2574// MailboxRename renames mailbox mbsrc to dst, and any missing parents for the

2575// destination, and any children of mbsrc and the destination.

2576//

2577// Names must be normalized and cannot be Inbox.

2578func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc Mailbox, dst string) (changes []Change, isInbox, notExists, alreadyExists bool, rerr error) {

2579 if mbsrc.Name == "Inbox" || dst == "Inbox" {

2580 return nil, true, false, false, fmt.Errorf("inbox cannot be renamed")

2581 }

2582

2583 // We gather existing mailboxes that we need for deciding what to create/delete/update.

2584 q := bstore.QueryTx[Mailbox](tx)

2585 srcPrefix := mbsrc.Name + "/"

2586 dstRoot := strings.SplitN(dst, "/", 2)[0]

2587 dstRootPrefix := dstRoot + "/"

2588 q.FilterFn(func(mb Mailbox) bool {

2589 return mb.Name == mbsrc.Name || strings.HasPrefix(mb.Name, srcPrefix) || mb.Name == dstRoot || strings.HasPrefix(mb.Name, dstRootPrefix)

2590 })

2591 q.SortAsc("Name") // We'll rename the parents before children.

2592 l, err := q.List()

2593 if err != nil {

2594 return nil, false, false, false, fmt.Errorf("listing relevant mailboxes: %v", err)

2595 }

2596

2597 mailboxes := map[string]Mailbox{}

2598 for _, mb := range l {

2599 mailboxes[mb.Name] = mb

2600 }

2601

2602 if _, ok := mailboxes[mbsrc.Name]; !ok {

2603 return nil, false, true, false, fmt.Errorf("mailbox does not exist")

2604 }

2605

2606 uidval, err := a.NextUIDValidity(tx)

2607 if err != nil {

2608 return nil, false, false, false, fmt.Errorf("next uid validity: %v", err)

2609 }

2610

2611 // Ensure parent mailboxes for the destination paths exist.

2612 var parent string

2613 dstElems := strings.Split(dst, "/")

2614 for i, elem := range dstElems[:len(dstElems)-1] {

2615 if i > 0 {

2616 parent += "/"

2617 }

2618 parent += elem

2619

2620 mb, ok := mailboxes[parent]

2621 if ok {

2622 continue

2623 }

2624 omb := mb

2625 mb = Mailbox{

2626 ID: omb.ID,

2627 Name: parent,

2628 UIDValidity: uidval,

2629 UIDNext: 1,

2630 HaveCounts: true,

2631 }

2632 if err := tx.Insert(&mb); err != nil {

2633 return nil, false, false, false, fmt.Errorf("creating parent mailbox %q: %v", mb.Name, err)

2634 }

2635 if err := tx.Get(&Subscription{Name: parent}); err != nil {

2636 if err := tx.Insert(&Subscription{Name: parent}); err != nil {

2637 return nil, false, false, false, fmt.Errorf("creating subscription for %q: %v", parent, err)

2638 }

2639 }

2640 changes = append(changes, ChangeAddMailbox{Mailbox: mb, Flags: []string{`\Subscribed`}})

2641 }

2642

2643 // Process src mailboxes, renaming them to dst.

2644 for _, srcmb := range l {

2645 if srcmb.Name != mbsrc.Name && !strings.HasPrefix(srcmb.Name, srcPrefix) {

2646 continue

2647 }

2648 srcName := srcmb.Name

2649 dstName := dst + srcmb.Name[len(mbsrc.Name):]

2650 if _, ok := mailboxes[dstName]; ok {

2651 return nil, false, false, true, fmt.Errorf("destination mailbox %q already exists", dstName)

2652 }

2653

2654 srcmb.Name = dstName

2655 srcmb.UIDValidity = uidval

2656 if err := tx.Update(&srcmb); err != nil {

2657 return nil, false, false, false, fmt.Errorf("renaming mailbox: %v", err)

2658 }

2659

2660 var dstFlags []string

2661 if tx.Get(&Subscription{Name: dstName}) == nil {

2662 dstFlags = []string{`\Subscribed`}

2663 }

2664 changes = append(changes, ChangeRenameMailbox{MailboxID: srcmb.ID, OldName: srcName, NewName: dstName, Flags: dstFlags})

2665 }

2666

2667 // If we renamed e.g. a/b to a/b/c/d, and a/b/c to a/b/c/d/c, we'll have to recreate a/b and a/b/c.

2668 srcElems := strings.Split(mbsrc.Name, "/")

2669 xsrc := mbsrc.Name

2670 for i := 0; i < len(dstElems) && strings.HasPrefix(dst, xsrc+"/"); i++ {

2671 mb := Mailbox{

2672 UIDValidity: uidval,

2673 UIDNext: 1,

2674 Name: xsrc,

2675 HaveCounts: true,

2676 }

2677 if err := tx.Insert(&mb); err != nil {

2678 return nil, false, false, false, fmt.Errorf("creating mailbox at old path %q: %v", mb.Name, err)

2679 }

2680 xsrc += "/" + dstElems[len(srcElems)+i]

2681 }

2682 return changes, false, false, false, nil

2683}

2684

2685// MailboxDelete deletes a mailbox by ID. If it has children, the return value

2686// indicates that and an error is returned.

2687//

2688// Caller should broadcast the changes and remove files for the removed message IDs.

2689func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx, mailbox Mailbox) (changes []Change, removeMessageIDs []int64, hasChildren bool, rerr error) {

2690 // Look for existence of child mailboxes. There is a lot of text in the IMAP RFCs about

2691 // NoInferior and NoSelect. We just require only leaf mailboxes are deleted.

2692 qmb := bstore.QueryTx[Mailbox](tx)

2693 mbprefix := mailbox.Name + "/"

2694 qmb.FilterFn(func(mb Mailbox) bool {

2695 return strings.HasPrefix(mb.Name, mbprefix)

2696 })

2697 if childExists, err := qmb.Exists(); err != nil {

2698 return nil, nil, false, fmt.Errorf("checking if mailbox has child: %v", err)

2699 } else if childExists {

2700 return nil, nil, true, fmt.Errorf("mailbox has a child, only leaf mailboxes can be deleted")

2701 }

2702

2703 // todo jmap: instead of completely deleting a mailbox and its messages, we need to mark them all as expunged.

2704

2705 qm := bstore.QueryTx[Message](tx)

2706 qm.FilterNonzero(Message{MailboxID: mailbox.ID})

2707 remove, err := qm.List()

2708 if err != nil {

2709 return nil, nil, false, fmt.Errorf("listing messages to remove: %v", err)

2710 }

2711

2712 if len(remove) > 0 {

2713 removeIDs := make([]any, len(remove))

2714 for i, m := range remove {

2715 removeIDs[i] = m.ID

2716 }

2717 qmr := bstore.QueryTx[Recipient](tx)

2718 qmr.FilterEqual("MessageID", removeIDs...)

2719 if _, err = qmr.Delete(); err != nil {

2720 return nil, nil, false, fmt.Errorf("removing message recipients for messages: %v", err)

2721 }

2722

2723 qm = bstore.QueryTx[Message](tx)

2724 qm.FilterNonzero(Message{MailboxID: mailbox.ID})

2725 if _, err := qm.Delete(); err != nil {

2726 return nil, nil, false, fmt.Errorf("removing messages: %v", err)

2727 }

2728

2729 var totalSize int64

2730 for _, m := range remove {

2731 if !m.Expunged {

2732 removeMessageIDs = append(removeMessageIDs, m.ID)

2733 totalSize += m.Size

2734 }

2735 }

2736 if err := a.AddMessageSize(log, tx, -totalSize); err != nil {

2737 return nil, nil, false, fmt.Errorf("updating disk usage: %v", err)

2738 }

2739

2740 // Mark messages as not needing training. Then retrain them, so they are untrained if they were.

2741 n := 0

2742 o := 0

2743 for _, m := range remove {

2744 if !m.Expunged {

2745 remove[o] = m

2746 remove[o].Junk = false

2747 remove[o].Notjunk = false

2748 n++

2749 }

2750 }

2751 remove = remove[:n]

2752 if err := a.RetrainMessages(ctx, log, tx, remove, true); err != nil {

2753 return nil, nil, false, fmt.Errorf("untraining deleted messages: %v", err)

2754 }

2755 }

2756

2757 if err := tx.Delete(&Mailbox{ID: mailbox.ID}); err != nil {

2758 return nil, nil, false, fmt.Errorf("removing mailbox: %v", err)

2759 }

2760 return []Change{ChangeRemoveMailbox{MailboxID: mailbox.ID, Name: mailbox.Name}}, removeMessageIDs, false, nil

2761}

2762

2763// CheckMailboxName checks if name is valid, returning an INBOX-normalized name.

2764// I.e. it changes various casings of INBOX and INBOX/* to Inbox and Inbox/*.

2765// Name is invalid if it contains leading/trailing/double slashes, or when it isn't

2766// unicode-normalized, or when empty or has special characters.

2767//

2768// If name is the inbox, and allowInbox is false, this is indicated with the isInbox return parameter.

2769// For that case, and for other invalid names, an error is returned.

2770func CheckMailboxName(name string, allowInbox bool) (normalizedName string, isInbox bool, rerr error) {

2771 first := strings.SplitN(name, "/", 2)[0]

2772 if strings.EqualFold(first, "inbox") {

2773 if len(name) == len("inbox") && !allowInbox {

2774 return "", true, fmt.Errorf("special mailbox name Inbox not allowed")

2775 }

2776 name = "Inbox" + name[len("Inbox"):]

2777 }

2778

2779 if norm.NFC.String(name) != name {

2780 return "", false, errors.New("non-unicode-normalized mailbox names not allowed")

2781 }

2782

2783 if name == "" {

2784 return "", false, errors.New("empty mailbox name")

2785 }

2786 if strings.HasPrefix(name, "/") || strings.HasSuffix(name, "/") || strings.Contains(name, "//") {

2787 return "", false, errors.New("bad slashes in mailbox name")

2788 }

2789

2790 // "%" and "*" are difficult to use with the IMAP LIST command, but we allow mostly

2791 // allow them. ../rfc/3501:1002 ../rfc/9051:983

2792 if strings.HasPrefix(name, "#") {

2793 return "", false, errors.New("mailbox name cannot start with hash due to conflict with imap namespaces")

2794 }

2795

2796 // "#" and "&" are special in IMAP mailbox names. "#" for namespaces, "&" for

2797 // IMAP-UTF-7 encoding. We do allow them. ../rfc/3501:1018 ../rfc/9051:991

2798

2799 for _, c := range name {

2800 // ../rfc/3501:999 ../rfc/6855:192 ../rfc/9051:979

2801 if c <= 0x1f || c >= 0x7f && c <= 0x9f || c == 0x2028 || c == 0x2029 {

2802 return "", false, errors.New("control characters not allowed in mailbox name")

2803 }

2804 }

2805 return name, false, nil

2806}

2807