1/*

2Package store implements storage for accounts, their mailboxes, IMAP

3subscriptions and messages, and broadcasts updates (e.g. mail delivery) to

4interested sessions (e.g. IMAP connections).

5

6Layout of storage for accounts:

7

8 <DataDir>/accounts/<name>/index.db

9 <DataDir>/accounts/<name>/msg/[a-zA-Z0-9_-]+/<id>

10

11Index.db holds tables for user information, mailboxes, and messages. Messages

12are stored in the msg/ subdirectory, each in their own file. The on-disk message

13does not contain headers generated during an incoming SMTP transaction, such as

14Received and Authentication-Results headers. Those are in the database to

15prevent having to rewrite incoming messages (e.g. Authentication-Result for DKIM

16signatures can only be determined after having read the message). Messages must

17be read through MsgReader, which transparently adds the prefix from the

18database.

19*/

20package store

21

22// todo: make up a function naming scheme that indicates whether caller should broadcast changes.

23

24import (

25 "context"

26 "crypto/md5"

27 cryptorand "crypto/rand"

28 "crypto/sha1"

29 "crypto/sha256"

30 "encoding"

31 "encoding/json"

32 "errors"

33 "fmt"

34 "hash"

35 "io"

36 "log/slog"

37 "os"

38 "path/filepath"

39 "runtime/debug"

40 "slices"

41 "sort"

42 "strconv"

43 "strings"

44 "sync"

45 "time"

46

47 "golang.org/x/crypto/bcrypt"

48 "golang.org/x/text/secure/precis"

49 "golang.org/x/text/unicode/norm"

50

51 "github.com/mjl-/bstore"

52

53 "github.com/mjl-/mox/config"

54 "github.com/mjl-/mox/dns"

55 "github.com/mjl-/mox/message"

56 "github.com/mjl-/mox/metrics"

57 "github.com/mjl-/mox/mlog"

58 "github.com/mjl-/mox/mox-"

59 "github.com/mjl-/mox/moxio"

60 "github.com/mjl-/mox/publicsuffix"

61 "github.com/mjl-/mox/scram"

62 "github.com/mjl-/mox/smtp"

63)

64

65// If true, each time an account is closed its database file is checked for

66// consistency. If an inconsistency is found, panic is called. Set by default

67// because of all the packages with tests, the mox main function sets it to

68// false again.

69var CheckConsistencyOnClose = true

70

71var (

72 ErrUnknownMailbox = errors.New("no such mailbox")

73 ErrUnknownCredentials = errors.New("credentials not found")

74 ErrAccountUnknown = errors.New("no such account")

75 ErrOverQuota = errors.New("account over quota")

76)

77

78var DefaultInitialMailboxes = config.InitialMailboxes{

79 SpecialUse: config.SpecialUseMailboxes{

80 Sent: "Sent",

81 Archive: "Archive",

82 Trash: "Trash",

83 Draft: "Drafts",

84 Junk: "Junk",

85 },

86}

87

88type SCRAM struct {

89 Salt []byte

90 Iterations int

91 SaltedPassword []byte

92}

93

94// CRAMMD5 holds HMAC ipad and opad hashes that are initialized with the first

95// block with (a derivation of) the key/password, so we don't store the password in plain

96// text.

97type CRAMMD5 struct {

98 Ipad hash.Hash

99 Opad hash.Hash

100}

101

102// BinaryMarshal is used by bstore to store the ipad/opad hash states.

103func (c CRAMMD5) MarshalBinary() ([]byte, error) {

104 if c.Ipad == nil || c.Opad == nil {

105 return nil, nil

106 }

107

108 ipad, err := c.Ipad.(encoding.BinaryMarshaler).MarshalBinary()

109 if err != nil {

110 return nil, fmt.Errorf("marshal ipad: %v", err)

111 }

112 opad, err := c.Opad.(encoding.BinaryMarshaler).MarshalBinary()

113 if err != nil {

114 return nil, fmt.Errorf("marshal opad: %v", err)

115 }

116 buf := make([]byte, 2+len(ipad)+len(opad))

117 ipadlen := uint16(len(ipad))

118 buf[0] = byte(ipadlen >> 8)

119 buf[1] = byte(ipadlen >> 0)

120 copy(buf[2:], ipad)

121 copy(buf[2+len(ipad):], opad)

122 return buf, nil

123}

124

125// BinaryUnmarshal is used by bstore to restore the ipad/opad hash states.

126func (c *CRAMMD5) UnmarshalBinary(buf []byte) error {

127 if len(buf) == 0 {

128 *c = CRAMMD5{}

129 return nil

130 }

131 if len(buf) < 2 {

132 return fmt.Errorf("short buffer")

133 }

134 ipadlen := int(uint16(buf[0])<<8 | uint16(buf[1])<<0)

135 if len(buf) < 2+ipadlen {

136 return fmt.Errorf("buffer too short for ipadlen")

137 }

138 ipad := md5.New()

139 opad := md5.New()

140 if err := ipad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2 : 2+ipadlen]); err != nil {

141 return fmt.Errorf("unmarshal ipad: %v", err)

142 }

143 if err := opad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2+ipadlen:]); err != nil {

144 return fmt.Errorf("unmarshal opad: %v", err)

145 }

146 *c = CRAMMD5{ipad, opad}

147 return nil

148}

149

150// Password holds credentials in various forms, for logging in with SMTP/IMAP.

151type Password struct {

152 Hash string // bcrypt hash for IMAP LOGIN, SASL PLAIN and HTTP basic authentication.

153 CRAMMD5 CRAMMD5 // For SASL CRAM-MD5.

154 SCRAMSHA1 SCRAM // For SASL SCRAM-SHA-1.

155 SCRAMSHA256 SCRAM // For SASL SCRAM-SHA-256.

156}

157

158// Subjectpass holds the secret key used to sign subjectpass tokens.

159type Subjectpass struct {

160 Email string // Our destination address (canonical, with catchall localpart stripped).

161 Key string

162}

163

164// NextUIDValidity is a singleton record in the database with the next UIDValidity

165// to use for the next mailbox.

166type NextUIDValidity struct {

167 ID int // Just a single record with ID 1.

168 Next uint32

169}

170

171// SyncState track ModSeqs.

172type SyncState struct {

173 ID int // Just a single record with ID 1.

174

175 // Last used, next assigned will be one higher. The first value we hand out is 2.

176 // That's because 0 (the default value for old existing messages, from before the

177 // Message.ModSeq field) is special in IMAP, so we return it as 1.

178 LastModSeq ModSeq `bstore:"nonzero"`

179

180 // Highest ModSeq of expunged record that we deleted. When a clients synchronizes

181 // and requests changes based on a modseq before this one, we don't have the

182 // history to provide information about deletions. We normally keep these expunged

183 // records around, but we may periodically truly delete them to reclaim storage

184 // space. Initially set to -1 because we don't want to match with any ModSeq in the

185 // database, which can be zero values.

186 HighestDeletedModSeq ModSeq

187}

188

189// Mailbox is collection of messages, e.g. Inbox or Sent.

190type Mailbox struct {

191 ID int64

192

193 // "Inbox" is the name for the special IMAP "INBOX". Slash separated

194 // for hierarchy.

195 Name string `bstore:"nonzero,unique"`

196

197 // If UIDs are invalidated, e.g. when renaming a mailbox to a previously existing

198 // name, UIDValidity must be changed. Used by IMAP for synchronization.

199 UIDValidity uint32

200

201 // UID likely to be assigned to next message. Used by IMAP to detect messages

202 // delivered to a mailbox.

203 UIDNext UID

204

205 SpecialUse

206

207 // Keywords as used in messages. Storing a non-system keyword for a message

208 // automatically adds it to this list. Used in the IMAP FLAGS response. Only

209 // "atoms" are allowed (IMAP syntax), keywords are case-insensitive, only stored in

210 // lower case (for JMAP), sorted.

211 Keywords []string

212

213 HaveCounts bool // Whether MailboxCounts have been initialized.

214 MailboxCounts // Statistics about messages, kept up to date whenever a change happens.

215}

216

217// MailboxCounts tracks statistics about messages for a mailbox.

218type MailboxCounts struct {

219 Total int64 // Total number of messages, excluding \Deleted. For JMAP.

220 Deleted int64 // Number of messages with \Deleted flag. Used for IMAP message count that includes messages with \Deleted.

221 Unread int64 // Messages without \Seen, excluding those with \Deleted, for JMAP.

222 Unseen int64 // Messages without \Seen, including those with \Deleted, for IMAP.

223 Size int64 // Number of bytes for all messages.

224}

225

226func (mc MailboxCounts) String() string {

227 return fmt.Sprintf("%d total, %d deleted, %d unread, %d unseen, size %d bytes", mc.Total, mc.Deleted, mc.Unread, mc.Unseen, mc.Size)

228}

229

230// Add increases mailbox counts mc with those of delta.

231func (mc *MailboxCounts) Add(delta MailboxCounts) {

232 mc.Total += delta.Total

233 mc.Deleted += delta.Deleted

234 mc.Unread += delta.Unread

235 mc.Unseen += delta.Unseen

236 mc.Size += delta.Size

237}

238

239// Add decreases mailbox counts mc with those of delta.

240func (mc *MailboxCounts) Sub(delta MailboxCounts) {

241 mc.Total -= delta.Total

242 mc.Deleted -= delta.Deleted

243 mc.Unread -= delta.Unread

244 mc.Unseen -= delta.Unseen

245 mc.Size -= delta.Size

246}

247

248// SpecialUse identifies a specific role for a mailbox, used by clients to

249// understand where messages should go.

250type SpecialUse struct {

251 Archive bool

252 Draft bool

253 Junk bool

254 Sent bool

255 Trash bool

256}

257

258// CalculateCounts calculates the full current counts for messages in the mailbox.

259func (mb *Mailbox) CalculateCounts(tx *bstore.Tx) (mc MailboxCounts, err error) {

260 q := bstore.QueryTx[Message](tx)

261 q.FilterNonzero(Message{MailboxID: mb.ID})

262 q.FilterEqual("Expunged", false)

263 err = q.ForEach(func(m Message) error {

264 mc.Add(m.MailboxCounts())

265 return nil

266 })

267 return

268}

269

270// ChangeSpecialUse returns a change for special-use flags, for broadcasting to

271// other connections.

272func (mb Mailbox) ChangeSpecialUse() ChangeMailboxSpecialUse {

273 return ChangeMailboxSpecialUse{mb.ID, mb.Name, mb.SpecialUse}

274}

275

276// ChangeKeywords returns a change with new keywords for a mailbox (e.g. after

277// setting a new keyword on a message in the mailbox), for broadcasting to other

278// connections.

279func (mb Mailbox) ChangeKeywords() ChangeMailboxKeywords {

280 return ChangeMailboxKeywords{mb.ID, mb.Name, mb.Keywords}

281}

282

283// KeywordsChanged returns whether the keywords in a mailbox have changed.

284func (mb Mailbox) KeywordsChanged(origmb Mailbox) bool {

285 if len(mb.Keywords) != len(origmb.Keywords) {

286 return true

287 }

288 // Keywords are stored sorted.

289 for i, kw := range mb.Keywords {

290 if origmb.Keywords[i] != kw {

291 return true

292 }

293 }

294 return false

295}

296

297// CountsChange returns a change with mailbox counts.

298func (mb Mailbox) ChangeCounts() ChangeMailboxCounts {

299 return ChangeMailboxCounts{mb.ID, mb.Name, mb.MailboxCounts}

300}

301

302// Subscriptions are separate from existence of mailboxes.

303type Subscription struct {

304 Name string

305}

306

307// Flags for a mail message.

308type Flags struct {

309 Seen bool

310 Answered bool

311 Flagged bool

312 Forwarded bool

313 Junk bool

314 Notjunk bool

315 Deleted bool

316 Draft bool

317 Phishing bool

318 MDNSent bool

319}

320

321// FlagsAll is all flags set, for use as mask.

322var FlagsAll = Flags{true, true, true, true, true, true, true, true, true, true}

323

324// Validation of "message From" domain.

325type Validation uint8

326

327const (

328 ValidationUnknown Validation = 0

329 ValidationStrict Validation = 1 // Like DMARC, with strict policies.

330 ValidationDMARC Validation = 2 // Actual DMARC policy.

331 ValidationRelaxed Validation = 3 // Like DMARC, with relaxed policies.

332 ValidationPass Validation = 4 // For SPF.

333 ValidationNeutral Validation = 5 // For SPF.

334 ValidationTemperror Validation = 6

335 ValidationPermerror Validation = 7

336 ValidationFail Validation = 8

337 ValidationSoftfail Validation = 9 // For SPF.

338 ValidationNone Validation = 10 // E.g. No records.

339)

340

341// Message stored in database and per-message file on disk.

342//

343// Contents are always the combined data from MsgPrefix and the on-disk file named

344// based on ID.

345//

346// Messages always have a header section, even if empty. Incoming messages without

347// header section must get an empty header section added before inserting.

348type Message struct {

349 // ID, unchanged over lifetime, determines path to on-disk msg file.

350 // Set during deliver.

351 ID int64

352

353 UID UID `bstore:"nonzero"` // UID, for IMAP. Set during deliver.

354 MailboxID int64 `bstore:"nonzero,unique MailboxID+UID,index MailboxID+Received,index MailboxID+ModSeq,ref Mailbox"`

355

356 // Modification sequence, for faster syncing with IMAP QRESYNC and JMAP.

357 // ModSeq is the last modification. CreateSeq is the Seq the message was inserted,

358 // always <= ModSeq. If Expunged is set, the message has been removed and should not

359 // be returned to the user. In this case, ModSeq is the Seq where the message is

360 // removed, and will never be changed again.

361 // We have an index on both ModSeq (for JMAP that synchronizes per account) and

362 // MailboxID+ModSeq (for IMAP that synchronizes per mailbox).

363 // The index on CreateSeq helps efficiently finding created messages for JMAP.

364 // The value of ModSeq is special for IMAP. Messages that existed before ModSeq was

365 // added have 0 as value. But modseq 0 in IMAP is special, so we return it as 1. If

366 // we get modseq 1 from a client, the IMAP server will translate it to 0. When we

367 // return modseq to clients, we turn 0 into 1.

368 ModSeq ModSeq `bstore:"index"`

369 CreateSeq ModSeq `bstore:"index"`

370 Expunged bool

371

372 // If set, this message was delivered to a Rejects mailbox. When it is moved to a

373 // different mailbox, its MailboxOrigID is set to the destination mailbox and this

374 // flag cleared.

375 IsReject bool

376

377 // If set, this is a forwarded message (through a ruleset with IsForward). This

378 // causes fields used during junk analysis to be moved to their Orig variants, and

379 // masked IP fields cleared, so they aren't used in junk classifications for

380 // incoming messages. This ensures the forwarded messages don't cause negative

381 // reputation for the forwarding mail server, which may also be sending regular

382 // messages.

383 IsForward bool

384

385 // MailboxOrigID is the mailbox the message was originally delivered to. Typically

386 // Inbox or Rejects, but can also be a mailbox configured in a Ruleset, or

387 // Postmaster, TLS/DMARC reporting addresses. MailboxOrigID is not changed when the

388 // message is moved to another mailbox, e.g. Archive/Trash/Junk. Used for

389 // per-mailbox reputation.

390 //

391 // MailboxDestinedID is normally 0, but when a message is delivered to the Rejects

392 // mailbox, it is set to the intended mailbox according to delivery rules,

393 // typically that of Inbox. When such a message is moved out of Rejects, the

394 // MailboxOrigID is corrected by setting it to MailboxDestinedID. This ensures the

395 // message is used for reputation calculation for future deliveries to that

396 // mailbox.

397 //

398 // These are not bstore references to prevent having to update all messages in a

399 // mailbox when the original mailbox is removed. Use of these fields requires

400 // checking if the mailbox still exists.

401 MailboxOrigID int64

402 MailboxDestinedID int64

403

404 Received time.Time `bstore:"default now,index"`

405

406 // Full IP address of remote SMTP server. Empty if not delivered over SMTP. The

407 // masked IPs are used to classify incoming messages. They are left empty for

408 // messages matching a ruleset for forwarded messages.

409 RemoteIP string

410 RemoteIPMasked1 string `bstore:"index RemoteIPMasked1+Received"` // For IPv4 /32, for IPv6 /64, for reputation.

411 RemoteIPMasked2 string `bstore:"index RemoteIPMasked2+Received"` // For IPv4 /26, for IPv6 /48.

412 RemoteIPMasked3 string `bstore:"index RemoteIPMasked3+Received"` // For IPv4 /21, for IPv6 /32.

413

414 // Only set if present and not an IP address. Unicode string. Empty for forwarded

415 // messages.

416 EHLODomain string `bstore:"index EHLODomain+Received"`

417 MailFrom string // With localpart and domain. Can be empty.

418 MailFromLocalpart smtp.Localpart // SMTP "MAIL FROM", can be empty.

419 // Only set if it is a domain, not an IP. Unicode string. Empty for forwarded

420 // messages, but see OrigMailFromDomain.

421 MailFromDomain string `bstore:"index MailFromDomain+Received"`

422 RcptToLocalpart smtp.Localpart // SMTP "RCPT TO", can be empty.

423 RcptToDomain string // Unicode string.

424

425 // Parsed "From" message header, used for reputation along with domain validation.

426 MsgFromLocalpart smtp.Localpart

427 MsgFromDomain string `bstore:"index MsgFromDomain+Received"` // Unicode string.

428 MsgFromOrgDomain string `bstore:"index MsgFromOrgDomain+Received"` // Unicode string.

429

430 // Simplified statements of the Validation fields below, used for incoming messages

431 // to check reputation.

432 EHLOValidated bool

433 MailFromValidated bool

434 MsgFromValidated bool

435

436 EHLOValidation Validation // Validation can also take reverse IP lookup into account, not only SPF.

437 MailFromValidation Validation // Can have SPF-specific validations like ValidationSoftfail.

438 MsgFromValidation Validation // Desirable validations: Strict, DMARC, Relaxed. Will not be just Pass.

439

440 // Domains with verified DKIM signatures. Unicode string. For forwarded messages, a

441 // DKIM domain that matched a ruleset's verified domain is left out, but included

442 // in OrigDKIMDomains.

443 DKIMDomains []string `bstore:"index DKIMDomains+Received"`

444

445 // For forwarded messages,

446 OrigEHLODomain string

447 OrigDKIMDomains []string

448

449 // Canonicalized Message-Id, always lower-case and normalized quoting, without

450 // <>'s. Empty if missing. Used for matching message threads, and to prevent

451 // duplicate reject delivery.

452 MessageID string `bstore:"index"`

453 // lower-case: ../rfc/5256:495

454

455 // For matching threads in case there is no References/In-Reply-To header. It is

456 // lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed.

457 SubjectBase string `bstore:"index"`

458 // ../rfc/5256:90

459

460 // Hash of message. For rejects delivery in case there is no Message-ID, only set

461 // when delivered as reject.

462 MessageHash []byte

463

464 // ID of message starting this thread.

465 ThreadID int64 `bstore:"index"`

466 // IDs of parent messages, from closest parent to the root message. Parent messages

467 // may be in a different mailbox, or may no longer exist. ThreadParentIDs must

468 // never contain the message id itself (a cycle), and parent messages must

469 // reference the same ancestors.

470 ThreadParentIDs []int64

471 // ThreadMissingLink is true if there is no match with a direct parent. E.g. first

472 // ID in ThreadParentIDs is not the direct ancestor (an intermediate message may

473 // have been deleted), or subject-based matching was done.

474 ThreadMissingLink bool

475 // If set, newly delivered child messages are automatically marked as read. This

476 // field is copied to new child messages. Changes are propagated to the webmail

477 // client.

478 ThreadMuted bool

479 // If set, this (sub)thread is collapsed in the webmail client, for threading mode

480 // "on" (mode "unread" ignores it). This field is copied to new child message.

481 // Changes are propagated to the webmail client.

482 ThreadCollapsed bool

483

484 // If received message was known to match a mailing list rule (with modified junk

485 // filtering).

486 IsMailingList bool

487

488 // If this message is a DSN, generated by us or received. For DSNs, we don't look

489 // at the subject when matching threads.

490 DSN bool

491

492 ReceivedTLSVersion uint16 // 0 if unknown, 1 if plaintext/no TLS, otherwise TLS cipher suite.

493 ReceivedTLSCipherSuite uint16

494 ReceivedRequireTLS bool // Whether RequireTLS was known to be used for incoming delivery.

495

496 Flags

497 // For keywords other than system flags or the basic well-known $-flags. Only in

498 // "atom" syntax (IMAP), they are case-insensitive, always stored in lower-case

499 // (for JMAP), sorted.

500 Keywords []string `bstore:"index"`

501 Size int64

502 TrainedJunk *bool // If nil, no training done yet. Otherwise, true is trained as junk, false trained as nonjunk.

503 MsgPrefix []byte // Typically holds received headers and/or header separator.

504

505 // ParsedBuf message structure. Currently saved as JSON of message.Part because bstore

506 // cannot yet store recursive types. Created when first needed, and saved in the

507 // database.

508 // todo: once replaced with non-json storage, remove date fixup in ../message/part.go.

509 ParsedBuf []byte

510}

511

512// MailboxCounts returns the delta to counts this message means for its

513// mailbox.

514func (m Message) MailboxCounts() (mc MailboxCounts) {

515 if m.Expunged {

516 return

517 }

518 if m.Deleted {

519 mc.Deleted++

520 } else {

521 mc.Total++

522 }

523 if !m.Seen {

524 mc.Unseen++

525 if !m.Deleted {

526 mc.Unread++

527 }

528 }

529 mc.Size += m.Size

530 return

531}

532

533func (m Message) ChangeAddUID() ChangeAddUID {

534 return ChangeAddUID{m.MailboxID, m.UID, m.ModSeq, m.Flags, m.Keywords}

535}

536

537func (m Message) ChangeFlags(orig Flags) ChangeFlags {

538 mask := m.Flags.Changed(orig)

539 return ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, ModSeq: m.ModSeq, Mask: mask, Flags: m.Flags, Keywords: m.Keywords}

540}

541

542func (m Message) ChangeThread() ChangeThread {

543 return ChangeThread{[]int64{m.ID}, m.ThreadMuted, m.ThreadCollapsed}

544}

545

546// ModSeq represents a modseq as stored in the database. ModSeq 0 in the

547// database is sent to the client as 1, because modseq 0 is special in IMAP.

548// ModSeq coming from the client are of type int64.

549type ModSeq int64

550

551func (ms ModSeq) Client() int64 {

552 if ms == 0 {

553 return 1

554 }

555 return int64(ms)

556}

557

558// ModSeqFromClient converts a modseq from a client to a modseq for internal

559// use, e.g. in a database query.

560// ModSeq 1 is turned into 0 (the Go zero value for ModSeq).

561func ModSeqFromClient(modseq int64) ModSeq {

562 if modseq == 1 {

563 return 0

564 }

565 return ModSeq(modseq)

566}

567

568// PrepareExpunge clears fields that are no longer needed after an expunge, so

569// almost all fields. Does not change ModSeq, but does set Expunged.

570func (m *Message) PrepareExpunge() {

571 *m = Message{

572 ID: m.ID,

573 UID: m.UID,

574 MailboxID: m.MailboxID,

575 CreateSeq: m.CreateSeq,

576 ModSeq: m.ModSeq,

577 Expunged: true,

578 ThreadID: m.ThreadID,

579 }

580}

581

582// PrepareThreading sets MessageID, SubjectBase and DSN (used in threading) based

583// on the part.

584func (m *Message) PrepareThreading(log mlog.Log, part *message.Part) {

585 m.DSN = part.IsDSN()

586

587 if part.Envelope == nil {

588 return

589 }

590 messageID, raw, err := message.MessageIDCanonical(part.Envelope.MessageID)

591 if err != nil {

592 log.Debugx("parsing message-id, ignoring", err, slog.String("messageid", part.Envelope.MessageID))

593 } else if raw {

594 log.Debug("could not parse message-id as address, continuing with raw value", slog.String("messageid", part.Envelope.MessageID))

595 }

596 m.MessageID = messageID

597 m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false)

598}

599

600// LoadPart returns a message.Part by reading from m.ParsedBuf.

601func (m Message) LoadPart(r io.ReaderAt) (message.Part, error) {

602 if m.ParsedBuf == nil {

603 return message.Part{}, fmt.Errorf("message not parsed")

604 }

605 var p message.Part

606 err := json.Unmarshal(m.ParsedBuf, &p)

607 if err != nil {

608 return p, fmt.Errorf("unmarshal message part")

609 }

610 p.SetReaderAt(r)

611 return p, nil

612}

613

614// NeedsTraining returns whether message needs a training update, based on

615// TrainedJunk (current training status) and new Junk/Notjunk flags.

616func (m Message) NeedsTraining() bool {

617 untrain := m.TrainedJunk != nil

618 untrainJunk := untrain && *m.TrainedJunk

619 train := m.Junk || m.Notjunk && !(m.Junk && m.Notjunk)

620 trainJunk := m.Junk

621 return untrain != train || untrain && train && untrainJunk != trainJunk

622}

623

624// JunkFlagsForMailbox sets Junk and Notjunk flags based on mailbox name if configured. Often

625// used when delivering/moving/copying messages to a mailbox. Mail clients are not

626// very helpful with setting junk/notjunk flags. But clients can move/copy messages

627// to other mailboxes. So we set flags when clients move a message.

628func (m *Message) JunkFlagsForMailbox(mb Mailbox, conf config.Account) {

629 if mb.Junk {

630 m.Junk = true

631 m.Notjunk = false

632 return

633 }

634

635 if !conf.AutomaticJunkFlags.Enabled {

636 return

637 }

638

639 lmailbox := strings.ToLower(mb.Name)

640

641 if conf.JunkMailbox != nil && conf.JunkMailbox.MatchString(lmailbox) {

642 m.Junk = true

643 m.Notjunk = false

644 } else if conf.NeutralMailbox != nil && conf.NeutralMailbox.MatchString(lmailbox) {

645 m.Junk = false

646 m.Notjunk = false

647 } else if conf.NotJunkMailbox != nil && conf.NotJunkMailbox.MatchString(lmailbox) {

648 m.Junk = false

649 m.Notjunk = true

650 } else if conf.JunkMailbox == nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox != nil {

651 m.Junk = true

652 m.Notjunk = false

653 } else if conf.JunkMailbox != nil && conf.NeutralMailbox == nil && conf.NotJunkMailbox != nil {

654 m.Junk = false

655 m.Notjunk = false

656 } else if conf.JunkMailbox != nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox == nil {

657 m.Junk = false

658 m.Notjunk = true

659 }

660}

661

662// Recipient represents the recipient of a message. It is tracked to allow

663// first-time incoming replies from users this account has sent messages to. When a

664// mailbox is added to the Sent mailbox the message is parsed and recipients are

665// inserted as recipient. Recipients are never removed other than for removing the

666// message. On move/copy of a message, recipients aren't modified either. For IMAP,

667// this assumes a client simply appends messages to the Sent mailbox (as opposed to

668// copying messages from some place).

669type Recipient struct {

670 ID int64

671 MessageID int64 `bstore:"nonzero,ref Message"` // Ref gives it its own index, useful for fast removal as well.

672 Localpart string `bstore:"nonzero"` // Encoded localpart.

673 Domain string `bstore:"nonzero,index Domain+Localpart"` // Unicode string.

674 OrgDomain string `bstore:"nonzero,index"` // Unicode string.

675 Sent time.Time `bstore:"nonzero"`

676}

677

678// Outgoing is a message submitted for delivery from the queue. Used to enforce

679// maximum outgoing messages.

680type Outgoing struct {

681 ID int64

682 Recipient string `bstore:"nonzero,index"` // Canonical international address with utf8 domain.

683 Submitted time.Time `bstore:"nonzero,default now"`

684}

685

686// RecipientDomainTLS stores TLS capabilities of a recipient domain as encountered

687// during most recent connection (delivery attempt).

688type RecipientDomainTLS struct {

689 Domain string // Unicode.

690 Updated time.Time `bstore:"default now"`

691 STARTTLS bool // Supports STARTTLS.

692 RequireTLS bool // Supports RequireTLS SMTP extension.

693}

694

695// DiskUsage tracks quota use.

696type DiskUsage struct {

697 ID int64 // Always one record with ID 1.

698 MessageSize int64 // Sum of all messages, for quota accounting.

699}

700

701// SessionToken and CSRFToken are types to prevent mixing them up.

702// Base64 raw url encoded.

703type SessionToken string

704type CSRFToken string

705

706// LoginSession represents a login session. We keep a limited number of sessions

707// for a user, removing the oldest session when a new one is created.

708type LoginSession struct {

709 ID int64

710 Created time.Time `bstore:"nonzero,default now"` // Of original login.

711 Expires time.Time `bstore:"nonzero"` // Extended each time it is used.

712 SessionTokenBinary [16]byte `bstore:"nonzero"` // Stored in cookie, like "webmailsession" or "webaccountsession".

713 CSRFTokenBinary [16]byte // For API requests, in "x-mox-csrf" header.

714 AccountName string `bstore:"nonzero"`

715 LoginAddress string `bstore:"nonzero"`

716

717 // Set when loading from database.

718 sessionToken SessionToken

719 csrfToken CSRFToken

720}

721

722// Quoting is a setting for how to quote in replies/forwards.

723type Quoting string

724

725const (

726 Default Quoting = "" // Bottom-quote if text is selected, top-quote otherwise.

727 Bottom Quoting = "bottom"

728 Top Quoting = "top"

729)

730

731// Settings are webmail client settings.

732type Settings struct {

733 ID uint8 // Singleton ID 1.

734

735 Signature string

736 Quoting Quoting

737

738 // Whether to show the bars underneath the address input fields indicating

739 // starttls/dnssec/dane/mtasts/requiretls support by address.

740 ShowAddressSecurity bool

741

742 // Show HTML version of message by default, instead of plain text.

743 ShowHTML bool

744

745 // If true, don't show shortcuts in webmail after mouse interaction.

746 NoShowShortcuts bool

747

748 // Additional headers to display in message view. E.g. Delivered-To, User-Agent, X-Mox-Reason.

749 ShowHeaders []string

750}

751

752// ViewMode how a message should be viewed: its text parts, html parts, or html

753// with loading external resources.

754type ViewMode string

755

756const (

757 ModeText ViewMode = "text"

758 ModeHTML ViewMode = "html"

759 ModeHTMLExt ViewMode = "htmlext" // HTML with external resources.

760)

761

762// FromAddressSettings are webmail client settings per "From" address.

763type FromAddressSettings struct {

764 FromAddress string // Unicode.

765 ViewMode ViewMode

766}

767

768// RulesetNoListID records a user "no" response to the question of

769// creating/removing a ruleset after moving a message with list-id header from/to

770// the inbox.

771type RulesetNoListID struct {

772 ID int64

773 RcptToAddress string `bstore:"nonzero"`

774 ListID string `bstore:"nonzero"`

775 ToInbox bool // Otherwise from Inbox to other mailbox.

776}

777

778// RulesetNoMsgFrom records a user "no" response to the question of

779// creating/moveing a ruleset after moving a mesage with message "from" address

780// from/to the inbox.

781type RulesetNoMsgFrom struct {

782 ID int64

783 RcptToAddress string `bstore:"nonzero"`

784 MsgFromAddress string `bstore:"nonzero"` // Unicode.

785 ToInbox bool // Otherwise from Inbox to other mailbox.

786}

787

788// RulesetNoMailbox represents a "never from/to this mailbox" response to the

789// question of adding/removing a ruleset after moving a message.

790type RulesetNoMailbox struct {

791 ID int64

792

793 // The mailbox from/to which the move has happened.

794 // Not a references, if mailbox is deleted, an entry becomes ineffective.

795 MailboxID int64 `bstore:"nonzero"`

796 ToMailbox bool // Whether MailboxID is the destination of the move (instead of source).

797}

798

799// Types stored in DB.

800var DBTypes = []any{

801 NextUIDValidity{},

802 Message{},

803 Recipient{},

804 Mailbox{},

805 Subscription{},

806 Outgoing{},

807 Password{},

808 Subjectpass{},

809 SyncState{},

810 Upgrade{},

811 RecipientDomainTLS{},

812 DiskUsage{},

813 LoginSession{},

814 Settings{},

815 FromAddressSettings{},

816 RulesetNoListID{},

817 RulesetNoMsgFrom{},

818 RulesetNoMailbox{},

819}

820

821// Account holds the information about a user, includings mailboxes, messages, imap subscriptions.

822type Account struct {

823 Name string // Name, according to configuration.

824 Dir string // Directory where account files, including the database, bloom filter, and mail messages, are stored for this account.

825 DBPath string // Path to database with mailboxes, messages, etc.

826 DB *bstore.DB // Open database connection.

827

828 // Channel that is closed if/when account has/gets "threads" accounting (see

829 // Upgrade.Threads).

830 threadsCompleted chan struct{}

831 // If threads upgrade completed with error, this is set. Used for warning during

832 // delivery, or aborting when importing.

833 threadsErr error

834

835 // Write lock must be held for account/mailbox modifications including message delivery.

836 // Read lock for reading mailboxes/messages.

837 // When making changes to mailboxes/messages, changes must be broadcasted before

838 // releasing the lock to ensure proper UID ordering.

839 sync.RWMutex

840

841 nused int // Reference count, while >0, this account is alive and shared.

842}

843

844type Upgrade struct {

845 ID byte

846 Threads byte // 0: None, 1: Adding MessageID's completed, 2: Adding ThreadID's completed.

847}

848

849// InitialUIDValidity returns a UIDValidity used for initializing an account.

850// It can be replaced during tests with a predictable value.

851var InitialUIDValidity = func() uint32 {

852 return uint32(time.Now().Unix() >> 1) // A 2-second resolution will get us far enough beyond 2038.

853}

854

855var openAccounts = struct {

856 names map[string]*Account

857 sync.Mutex

858}{

859 names: map[string]*Account{},

860}

861

862func closeAccount(acc *Account) (rerr error) {

863 openAccounts.Lock()

864 acc.nused--

865 defer openAccounts.Unlock()

866 if acc.nused == 0 {

867 // threadsCompleted must be closed now because it increased nused.

868 rerr = acc.DB.Close()

869 acc.DB = nil

870 delete(openAccounts.names, acc.Name)

871 }

872 return

873}

874

875// OpenAccount opens an account by name.

876//

877// No additional data path prefix or ".db" suffix should be added to the name.

878// A single shared account exists per name.

879func OpenAccount(log mlog.Log, name string) (*Account, error) {

880 openAccounts.Lock()

881 defer openAccounts.Unlock()

882 if acc, ok := openAccounts.names[name]; ok {

883 acc.nused++

884 return acc, nil

885 }

886

887 if _, ok := mox.Conf.Account(name); !ok {

888 return nil, ErrAccountUnknown

889 }

890

891 acc, err := openAccount(log, name)

892 if err != nil {

893 return nil, err

894 }

895 openAccounts.names[name] = acc

896 return acc, nil

897}

898

899// openAccount opens an existing account, or creates it if it is missing.

900func openAccount(log mlog.Log, name string) (a *Account, rerr error) {

901 dir := filepath.Join(mox.DataDirPath("accounts"), name)

902 return OpenAccountDB(log, dir, name)

903}

904

905// OpenAccountDB opens an account database file and returns an initialized account

906// or error. Only exported for use by subcommands that verify the database file.

907// Almost all account opens must go through OpenAccount/OpenEmail/OpenEmailAuth.

908func OpenAccountDB(log mlog.Log, accountDir, accountName string) (a *Account, rerr error) {

909 dbpath := filepath.Join(accountDir, "index.db")

910

911 // Create account if it doesn't exist yet.

912 isNew := false

913 if _, err := os.Stat(dbpath); err != nil && os.IsNotExist(err) {

914 isNew = true

915 os.MkdirAll(accountDir, 0770)

916 }

917

918 opts := bstore.Options{Timeout: 5 * time.Second, Perm: 0660, RegisterLogger: log.Logger}

919 db, err := bstore.Open(context.TODO(), dbpath, &opts, DBTypes...)

920 if err != nil {

921 return nil, err

922 }

923

924 defer func() {

925 if rerr != nil {

926 db.Close()

927 if isNew {

928 os.Remove(dbpath)

929 }

930 }

931 }()

932

933 acc := &Account{

934 Name: accountName,

935 Dir: accountDir,

936 DBPath: dbpath,

937 DB: db,

938 nused: 1,

939 threadsCompleted: make(chan struct{}),

940 }

941

942 if isNew {

943 if err := initAccount(db); err != nil {

944 return nil, fmt.Errorf("initializing account: %v", err)

945 }

946 close(acc.threadsCompleted)

947 return acc, nil

948 }

949

950 // Ensure singletons are present. Mailbox counts and total message size, Settings.

951 var mentioned bool

952 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

953 if tx.Get(&Settings{ID: 1}) == bstore.ErrAbsent {

954 if err := tx.Insert(&Settings{ID: 1, ShowAddressSecurity: true}); err != nil {

955 return err

956 }

957 }

958

959 err := bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error {

960 if !mentioned {

961 mentioned = true

962 log.Info("first calculation of mailbox counts for account", slog.String("account", accountName))

963 }

964 mc, err := mb.CalculateCounts(tx)

965 if err != nil {

966 return err

967 }

968 mb.HaveCounts = true

969 mb.MailboxCounts = mc

970 return tx.Update(&mb)

971 })

972 if err != nil {

973 return err

974 }

975

976 du := DiskUsage{ID: 1}

977 err = tx.Get(&du)

978 if err == nil || !errors.Is(err, bstore.ErrAbsent) {

979 return err

980 }

981 // No DiskUsage record yet, calculate total size and insert.

982 err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {

983 du.MessageSize += mb.Size

984 return nil

985 })

986 if err != nil {

987 return err

988 }

989 return tx.Insert(&du)

990 })

991 if err != nil {

992 return nil, fmt.Errorf("calculating counts for mailbox or inserting settings: %v", err)

993 }

994

995 // Start adding threading if needed.

996 up := Upgrade{ID: 1}

997 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

998 err := tx.Get(&up)

999 if err == bstore.ErrAbsent {

1000 if err := tx.Insert(&up); err != nil {

1001 return fmt.Errorf("inserting initial upgrade record: %v", err)

1002 }

1003 err = nil

1004 }

1005 return err

1006 })

1007 if err != nil {

1008 return nil, fmt.Errorf("checking message threading: %v", err)

1009 }

1010 if up.Threads == 2 {

1011 close(acc.threadsCompleted)

1012 return acc, nil

1013 }

1014

1015 // Increase account use before holding on to account in background.

1016 // Caller holds the lock. The goroutine below decreases nused by calling

1017 // closeAccount.

1018 acc.nused++

1019

1020 // Ensure all messages have a MessageID and SubjectBase, which are needed when

1021 // matching threads.

1022 // Then assign messages to threads, in the same way we do during imports.

1023 log.Info("upgrading account for threading, in background", slog.String("account", acc.Name))

1024 go func() {

1025 defer func() {

1026 err := closeAccount(acc)

1027 log.Check(err, "closing use of account after upgrading account storage for threads", slog.String("account", a.Name))

1028

1029 // Mark that upgrade has finished, possibly error is indicated in threadsErr.

1030 close(acc.threadsCompleted)

1031 }()

1032

1033 defer func() {

1034 x := recover() // Should not happen, but don't take program down if it does.

1035 if x != nil {

1036 log.Error("upgradeThreads panic", slog.Any("err", x))

1037 debug.PrintStack()

1038 metrics.PanicInc(metrics.Upgradethreads)

1039 acc.threadsErr = fmt.Errorf("panic during upgradeThreads: %v", x)

1040 }

1041 }()

1042

1043 err := upgradeThreads(mox.Shutdown, log, acc, &up)

1044 if err != nil {

1045 a.threadsErr = err

1046 log.Errorx("upgrading account for threading, aborted", err, slog.String("account", a.Name))

1047 } else {

1048 log.Info("upgrading account for threading, completed", slog.String("account", a.Name))

1049 }

1050 }()

1051 return acc, nil

1052}

1053

1054// ThreadingWait blocks until the one-time account threading upgrade for the

1055// account has completed, and returns an error if not successful.

1056//

1057// To be used before starting an import of messages.

1058func (a *Account) ThreadingWait(log mlog.Log) error {

1059 select {

1060 case <-a.threadsCompleted:

1061 return a.threadsErr

1062 default:

1063 }

1064 log.Debug("waiting for account upgrade to complete")

1065

1066 <-a.threadsCompleted

1067 return a.threadsErr

1068}

1069

1070func initAccount(db *bstore.DB) error {

1071 return db.Write(context.TODO(), func(tx *bstore.Tx) error {

1072 uidvalidity := InitialUIDValidity()

1073

1074 if err := tx.Insert(&Upgrade{ID: 1, Threads: 2}); err != nil {

1075 return err

1076 }

1077 if err := tx.Insert(&DiskUsage{ID: 1}); err != nil {

1078 return err

1079 }

1080 if err := tx.Insert(&Settings{ID: 1}); err != nil {

1081 return err

1082 }

1083

1084 if len(mox.Conf.Static.DefaultMailboxes) > 0 {

1085 // Deprecated in favor of InitialMailboxes.

1086 defaultMailboxes := mox.Conf.Static.DefaultMailboxes

1087 mailboxes := []string{"Inbox"}

1088 for _, name := range defaultMailboxes {

1089 if strings.EqualFold(name, "Inbox") {

1090 continue

1091 }

1092 mailboxes = append(mailboxes, name)

1093 }

1094 for _, name := range mailboxes {

1095 mb := Mailbox{Name: name, UIDValidity: uidvalidity, UIDNext: 1, HaveCounts: true}

1096 if strings.HasPrefix(name, "Archive") {

1097 mb.Archive = true

1098 } else if strings.HasPrefix(name, "Drafts") {

1099 mb.Draft = true

1100 } else if strings.HasPrefix(name, "Junk") {

1101 mb.Junk = true

1102 } else if strings.HasPrefix(name, "Sent") {

1103 mb.Sent = true

1104 } else if strings.HasPrefix(name, "Trash") {

1105 mb.Trash = true

1106 }

1107 if err := tx.Insert(&mb); err != nil {

1108 return fmt.Errorf("creating mailbox: %w", err)

1109 }

1110 if err := tx.Insert(&Subscription{name}); err != nil {

1111 return fmt.Errorf("adding subscription: %w", err)

1112 }

1113 }

1114 } else {

1115 mailboxes := mox.Conf.Static.InitialMailboxes

1116 var zerouse config.SpecialUseMailboxes

1117 if mailboxes.SpecialUse == zerouse && len(mailboxes.Regular) == 0 {

1118 mailboxes = DefaultInitialMailboxes

1119 }

1120

1121 add := func(name string, use SpecialUse) error {

1122 mb := Mailbox{Name: name, UIDValidity: uidvalidity, UIDNext: 1, SpecialUse: use, HaveCounts: true}

1123 if err := tx.Insert(&mb); err != nil {

1124 return fmt.Errorf("creating mailbox: %w", err)

1125 }

1126 if err := tx.Insert(&Subscription{name}); err != nil {

1127 return fmt.Errorf("adding subscription: %w", err)

1128 }

1129 return nil

1130 }

1131 addSpecialOpt := func(nameOpt string, use SpecialUse) error {

1132 if nameOpt == "" {

1133 return nil

1134 }

1135 return add(nameOpt, use)

1136 }

1137 l := []struct {

1138 nameOpt string

1139 use SpecialUse

1140 }{

1141 {"Inbox", SpecialUse{}},

1142 {mailboxes.SpecialUse.Archive, SpecialUse{Archive: true}},

1143 {mailboxes.SpecialUse.Draft, SpecialUse{Draft: true}},

1144 {mailboxes.SpecialUse.Junk, SpecialUse{Junk: true}},

1145 {mailboxes.SpecialUse.Sent, SpecialUse{Sent: true}},

1146 {mailboxes.SpecialUse.Trash, SpecialUse{Trash: true}},

1147 }

1148 for _, e := range l {

1149 if err := addSpecialOpt(e.nameOpt, e.use); err != nil {

1150 return err

1151 }

1152 }

1153 for _, name := range mailboxes.Regular {

1154 if err := add(name, SpecialUse{}); err != nil {

1155 return err

1156 }

1157 }

1158 }

1159

1160 uidvalidity++

1161 if err := tx.Insert(&NextUIDValidity{1, uidvalidity}); err != nil {

1162 return fmt.Errorf("inserting nextuidvalidity: %w", err)

1163 }

1164 return nil

1165 })

1166}

1167

1168// CheckClosed asserts that the account has a zero reference count. For use in tests.

1169func (a *Account) CheckClosed() {

1170 openAccounts.Lock()

1171 defer openAccounts.Unlock()

1172 if a.nused != 0 {

1173 panic(fmt.Sprintf("account still in use, %d refs", a.nused))

1174 }

1175}

1176

1177// Close reduces the reference count, and closes the database connection when

1178// it was the last user.

1179func (a *Account) Close() error {

1180 if CheckConsistencyOnClose {

1181 xerr := a.CheckConsistency()

1182 err := closeAccount(a)

1183 if xerr != nil {

1184 panic(xerr)

1185 }

1186 return err

1187 }

1188 return closeAccount(a)

1189}

1190

1191// CheckConsistency checks the consistency of the database and returns a non-nil

1192// error for these cases:

1193//

1194// - Missing on-disk file for message.

1195// - Mismatch between message size and length of MsgPrefix and on-disk file.

1196// - Missing HaveCounts.

1197// - Incorrect mailbox counts.

1198// - Incorrect total message size.

1199// - Message with UID >= mailbox uid next.

1200// - Mailbox uidvalidity >= account uid validity.

1201// - ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq.

1202// - All messages have a nonzero ThreadID, and no cycles in ThreadParentID, and parent messages the same ThreadParentIDs tail.

1203func (a *Account) CheckConsistency() error {

1204 var uidErrors []string // With a limit, could be many.

1205 var modseqErrors []string // With limit.

1206 var fileErrors []string // With limit.

1207 var threadidErrors []string // With limit.

1208 var threadParentErrors []string // With limit.

1209 var threadAncestorErrors []string // With limit.

1210 var errors []string

1211

1212 err := a.DB.Read(context.Background(), func(tx *bstore.Tx) error {

1213 nuv := NextUIDValidity{ID: 1}

1214 err := tx.Get(&nuv)

1215 if err != nil {

1216 return fmt.Errorf("fetching next uid validity: %v", err)

1217 }

1218

1219 mailboxes := map[int64]Mailbox{}

1220 err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {

1221 mailboxes[mb.ID] = mb

1222

1223 if mb.UIDValidity >= nuv.Next {

1224 errmsg := fmt.Sprintf("mailbox %q (id %d) has uidvalidity %d >= account next uidvalidity %d", mb.Name, mb.ID, mb.UIDValidity, nuv.Next)

1225 errors = append(errors, errmsg)

1226 }

1227 return nil

1228 })

1229 if err != nil {

1230 return fmt.Errorf("listing mailboxes: %v", err)

1231 }

1232

1233 counts := map[int64]MailboxCounts{}

1234 err = bstore.QueryTx[Message](tx).ForEach(func(m Message) error {

1235 mc := counts[m.MailboxID]

1236 mc.Add(m.MailboxCounts())

1237 counts[m.MailboxID] = mc

1238

1239 mb := mailboxes[m.MailboxID]

1240

1241 if (m.ModSeq == 0 || m.CreateSeq == 0 || m.CreateSeq > m.ModSeq) && len(modseqErrors) < 20 {

1242 modseqerr := fmt.Sprintf("message %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and createseq <= modseq", m.ID, mb.Name, mb.ID, m.ModSeq, m.CreateSeq)

1243 modseqErrors = append(modseqErrors, modseqerr)

1244 }

1245 if m.UID >= mb.UIDNext && len(uidErrors) < 20 {

1246 uiderr := fmt.Sprintf("message %d in mailbox %q (id %d) has uid %d >= mailbox uidnext %d", m.ID, mb.Name, mb.ID, m.UID, mb.UIDNext)

1247 uidErrors = append(uidErrors, uiderr)

1248 }

1249 if m.Expunged {

1250 return nil

1251 }

1252 p := a.MessagePath(m.ID)

1253 st, err := os.Stat(p)

1254 if err != nil {

1255 existserr := fmt.Sprintf("message %d in mailbox %q (id %d) on-disk file %s: %v", m.ID, mb.Name, mb.ID, p, err)

1256 fileErrors = append(fileErrors, existserr)

1257 } else if len(fileErrors) < 20 && m.Size != int64(len(m.MsgPrefix))+st.Size() {

1258 sizeerr := fmt.Sprintf("message %d in mailbox %q (id %d) has size %d != len msgprefix %d + on-disk file size %d = %d", m.ID, mb.Name, mb.ID, m.Size, len(m.MsgPrefix), st.Size(), int64(len(m.MsgPrefix))+st.Size())

1259 fileErrors = append(fileErrors, sizeerr)

1260 }

1261

1262 if m.ThreadID <= 0 && len(threadidErrors) < 20 {

1263 err := fmt.Sprintf("message %d in mailbox %q (id %d) has threadid 0", m.ID, mb.Name, mb.ID)

1264 threadidErrors = append(threadidErrors, err)

1265 }

1266 if slices.Contains(m.ThreadParentIDs, m.ID) && len(threadParentErrors) < 20 {

1267 err := fmt.Sprintf("message %d in mailbox %q (id %d) references itself in threadparentids", m.ID, mb.Name, mb.ID)

1268 threadParentErrors = append(threadParentErrors, err)

1269 }

1270 for i, pid := range m.ThreadParentIDs {

1271 am := Message{ID: pid}

1272 if err := tx.Get(&am); err == bstore.ErrAbsent {

1273 continue

1274 } else if err != nil {

1275 return fmt.Errorf("get ancestor message: %v", err)

1276 } else if !slices.Equal(m.ThreadParentIDs[i+1:], am.ThreadParentIDs) && len(threadAncestorErrors) < 20 {

1277 err := fmt.Sprintf("message %d, thread %d has ancestor ids %v, and ancestor at index %d with id %d should have the same tail but has %v\n", m.ID, m.ThreadID, m.ThreadParentIDs, i, am.ID, am.ThreadParentIDs)

1278 threadAncestorErrors = append(threadAncestorErrors, err)

1279 } else {

1280 break

1281 }

1282 }

1283 return nil

1284 })

1285 if err != nil {

1286 return fmt.Errorf("reading messages: %v", err)

1287 }

1288

1289 var totalSize int64

1290 for _, mb := range mailboxes {

1291 totalSize += mb.Size

1292 if !mb.HaveCounts {

1293 errmsg := fmt.Sprintf("mailbox %q (id %d) does not have counts, should be %#v", mb.Name, mb.ID, counts[mb.ID])

1294 errors = append(errors, errmsg)

1295 } else if mb.MailboxCounts != counts[mb.ID] {

1296 mbcounterr := fmt.Sprintf("mailbox %q (id %d) has wrong counts %s, should be %s", mb.Name, mb.ID, mb.MailboxCounts, counts[mb.ID])

1297 errors = append(errors, mbcounterr)

1298 }

1299 }

1300

1301 du := DiskUsage{ID: 1}

1302 if err := tx.Get(&du); err != nil {

1303 return fmt.Errorf("get diskusage")

1304 }

1305 if du.MessageSize != totalSize {

1306 errmsg := fmt.Sprintf("total message size in database is %d, sum of mailbox message sizes is %d", du.MessageSize, totalSize)

1307 errors = append(errors, errmsg)

1308 }

1309

1310 return nil

1311 })

1312 if err != nil {

1313 return err

1314 }

1315 errors = append(errors, uidErrors...)

1316 errors = append(errors, modseqErrors...)

1317 errors = append(errors, fileErrors...)

1318 errors = append(errors, threadidErrors...)

1319 errors = append(errors, threadParentErrors...)

1320 errors = append(errors, threadAncestorErrors...)

1321 if len(errors) > 0 {

1322 return fmt.Errorf("%s", strings.Join(errors, "; "))

1323 }

1324 return nil

1325}

1326

1327// Conf returns the configuration for this account if it still exists. During

1328// an SMTP session, a configuration update may drop an account.

1329func (a *Account) Conf() (config.Account, bool) {

1330 return mox.Conf.Account(a.Name)

1331}

1332

1333// NextUIDValidity returns the next new/unique uidvalidity to use for this account.

1334func (a *Account) NextUIDValidity(tx *bstore.Tx) (uint32, error) {

1335 nuv := NextUIDValidity{ID: 1}

1336 if err := tx.Get(&nuv); err != nil {

1337 return 0, err

1338 }

1339 v := nuv.Next

1340 nuv.Next++

1341 if err := tx.Update(&nuv); err != nil {

1342 return 0, err

1343 }

1344 return v, nil

1345}

1346

1347// NextModSeq returns the next modification sequence, which is global per account,

1348// over all types.

1349func (a *Account) NextModSeq(tx *bstore.Tx) (ModSeq, error) {

1350 v := SyncState{ID: 1}

1351 if err := tx.Get(&v); err == bstore.ErrAbsent {

1352 // We start assigning from modseq 2. Modseq 0 is not usable, so returned as 1, so

1353 // already used.

1354 // HighestDeletedModSeq is -1 so comparison against the default ModSeq zero value

1355 // makes sense.

1356 v = SyncState{1, 2, -1}

1357 return v.LastModSeq, tx.Insert(&v)

1358 } else if err != nil {

1359 return 0, err

1360 }

1361 v.LastModSeq++

1362 return v.LastModSeq, tx.Update(&v)

1363}

1364

1365func (a *Account) HighestDeletedModSeq(tx *bstore.Tx) (ModSeq, error) {

1366 v := SyncState{ID: 1}

1367 err := tx.Get(&v)

1368 if err == bstore.ErrAbsent {

1369 return 0, nil

1370 }

1371 return v.HighestDeletedModSeq, err

1372}

1373

1374// WithWLock runs fn with account writelock held. Necessary for account/mailbox

1375// modification. For message delivery, a read lock is required.

1376func (a *Account) WithWLock(fn func()) {

1377 a.Lock()

1378 defer a.Unlock()

1379 fn()

1380}

1381

1382// WithRLock runs fn with account read lock held. Needed for message delivery.

1383func (a *Account) WithRLock(fn func()) {

1384 a.RLock()

1385 defer a.RUnlock()

1386 fn()

1387}

1388

1389// DeliverMessage delivers a mail message to the account.

1390//

1391// The message, with msg.MsgPrefix and msgFile combined, must have a header

1392// section. The caller is responsible for adding a header separator to

1393// msg.MsgPrefix if missing from an incoming message.

1394//

1395// If the destination mailbox has the Sent special-use flag, the message is parsed

1396// for its recipients (to/cc/bcc). Their domains are added to Recipients for use in

1397// dmarc reputation.

1398//

1399// If sync is true, the message file and its directory are synced. Should be true

1400// for regular mail delivery, but can be false when importing many messages.

1401//

1402// If updateDiskUsage is true, the account total message size (for quota) is

1403// updated. Callers must check if a message can be added within quota before

1404// calling DeliverMessage.

1405//

1406// If CreateSeq/ModSeq is not set, it is assigned automatically.

1407//

1408// Must be called with account rlock or wlock.

1409//

1410// Caller must broadcast new message.

1411//

1412// Caller must update mailbox counts.

1413func (a *Account) DeliverMessage(log mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, sync, notrain, nothreads, updateDiskUsage bool) error {

1414 if m.Expunged {

1415 return fmt.Errorf("cannot deliver expunged message")

1416 }

1417

1418 mb := Mailbox{ID: m.MailboxID}

1419 if err := tx.Get(&mb); err != nil {

1420 return fmt.Errorf("get mailbox: %w", err)

1421 }

1422 m.UID = mb.UIDNext

1423 mb.UIDNext++

1424 if err := tx.Update(&mb); err != nil {

1425 return fmt.Errorf("updating mailbox nextuid: %w", err)

1426 }

1427

1428 if updateDiskUsage {

1429 du := DiskUsage{ID: 1}

1430 if err := tx.Get(&du); err != nil {

1431 return fmt.Errorf("get disk usage: %v", err)

1432 }

1433 du.MessageSize += m.Size

1434 if err := tx.Update(&du); err != nil {

1435 return fmt.Errorf("update disk usage: %v", err)

1436 }

1437 }

1438

1439 conf, _ := a.Conf()

1440 m.JunkFlagsForMailbox(mb, conf)

1441

1442 mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile.

1443 var part *message.Part

1444 if m.ParsedBuf == nil {

1445 p, err := message.EnsurePart(log.Logger, false, mr, m.Size)

1446 if err != nil {

1447 log.Infox("parsing delivered message", err, slog.String("parse", ""), slog.Int64("message", m.ID))

1448 // We continue, p is still valid.

1449 }

1450 part = &p

1451 buf, err := json.Marshal(part)

1452 if err != nil {

1453 return fmt.Errorf("marshal parsed message: %w", err)

1454 }

1455 m.ParsedBuf = buf

1456 } else {

1457 var p message.Part

1458 if err := json.Unmarshal(m.ParsedBuf, &p); err != nil {

1459 log.Errorx("unmarshal parsed message, continuing", err, slog.String("parse", ""))

1460 } else {

1461 part = &p

1462 }

1463 }

1464

1465 // If we are delivering to the originally intended mailbox, no need to store the mailbox ID again.

1466 if m.MailboxDestinedID != 0 && m.MailboxDestinedID == m.MailboxOrigID {

1467 m.MailboxDestinedID = 0

1468 }

1469 if m.CreateSeq == 0 || m.ModSeq == 0 {

1470 modseq, err := a.NextModSeq(tx)

1471 if err != nil {

1472 return fmt.Errorf("assigning next modseq: %w", err)

1473 }

1474 m.CreateSeq = modseq

1475 m.ModSeq = modseq

1476 }

1477

1478 if part != nil && m.MessageID == "" && m.SubjectBase == "" {

1479 m.PrepareThreading(log, part)

1480 }

1481

1482 // Assign to thread (if upgrade has completed).

1483 noThreadID := nothreads

1484 if m.ThreadID == 0 && !nothreads && part != nil {

1485 select {

1486 case <-a.threadsCompleted:

1487 if a.threadsErr != nil {

1488 log.Info("not assigning threads for new delivery, upgrading to threads failed")

1489 noThreadID = true

1490 } else {

1491 if err := assignThread(log, tx, m, part); err != nil {

1492 return fmt.Errorf("assigning thread: %w", err)

1493 }

1494 }

1495 default:

1496 // note: since we have a write transaction to get here, we can't wait for the

1497 // thread upgrade to finish.

1498 // If we don't assign a threadid the upgrade process will do it.

1499 log.Info("not assigning threads for new delivery, upgrading to threads in progress which will assign this message")

1500 noThreadID = true

1501 }

1502 }

1503

1504 if err := tx.Insert(m); err != nil {

1505 return fmt.Errorf("inserting message: %w", err)

1506 }

1507 if !noThreadID && m.ThreadID == 0 {

1508 m.ThreadID = m.ID

1509 if err := tx.Update(m); err != nil {

1510 return fmt.Errorf("updating message for its own thread id: %w", err)

1511 }

1512 }

1513

1514 // todo: perhaps we should match the recipients based on smtp submission and a matching message-id? we now miss the addresses in bcc's if the mail client doesn't save a message that includes the bcc header in the sent mailbox.

1515 if mb.Sent && part != nil && part.Envelope != nil {

1516 e := part.Envelope

1517 sent := e.Date

1518 if sent.IsZero() {

1519 sent = m.Received

1520 }

1521 if sent.IsZero() {

1522 sent = time.Now()

1523 }

1524 addrs := append(append(e.To, e.CC...), e.BCC...)

1525 for _, addr := range addrs {

1526 if addr.User == "" {

1527 // Would trigger error because Recipient.Localpart must be nonzero. todo: we could allow empty localpart in db, and filter by not using FilterNonzero.

1528 log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", slog.Any("address", addr))

1529 continue

1530 }

1531 d, err := dns.ParseDomain(addr.Host)

1532 if err != nil {

1533 log.Debugx("parsing domain in to/cc/bcc address", err, slog.Any("address", addr))

1534 continue

1535 }

1536 lp, err := smtp.ParseLocalpart(addr.User)

1537 if err != nil {

1538 log.Debugx("parsing localpart in to/cc/bcc address", err, slog.Any("address", addr))

1539 continue

1540 }

1541 mr := Recipient{

1542 MessageID: m.ID,

1543 Localpart: lp.String(),

1544 Domain: d.Name(),

1545 OrgDomain: publicsuffix.Lookup(context.TODO(), log.Logger, d).Name(),

1546 Sent: sent,

1547 }

1548 if err := tx.Insert(&mr); err != nil {

1549 return fmt.Errorf("inserting sent message recipients: %w", err)

1550 }

1551 }

1552 }

1553

1554 msgPath := a.MessagePath(m.ID)

1555 msgDir := filepath.Dir(msgPath)

1556 os.MkdirAll(msgDir, 0770)

1557

1558 // Sync file data to disk.

1559 if sync {

1560 if err := msgFile.Sync(); err != nil {

1561 return fmt.Errorf("fsync message file: %w", err)

1562 }

1563 }

1564

1565 if err := moxio.LinkOrCopy(log, msgPath, msgFile.Name(), &moxio.AtReader{R: msgFile}, true); err != nil {

1566 return fmt.Errorf("linking/copying message to new file: %w", err)

1567 }

1568

1569 if sync {

1570 if err := moxio.SyncDir(log, msgDir); err != nil {

1571 xerr := os.Remove(msgPath)

1572 log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath))

1573 return fmt.Errorf("sync directory: %w", err)

1574 }

1575 }

1576

1577 if !notrain && m.NeedsTraining() {

1578 l := []Message{*m}

1579 if err := a.RetrainMessages(context.TODO(), log, tx, l, false); err != nil {

1580 xerr := os.Remove(msgPath)

1581 log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath))

1582 return fmt.Errorf("training junkfilter: %w", err)

1583 }

1584 *m = l[0]

1585 }

1586

1587 return nil

1588}

1589

1590// SetPassword saves a new password for this account. This password is used for

1591// IMAP, SMTP (submission) sessions and the HTTP account web page.

1592func (a *Account) SetPassword(log mlog.Log, password string) error {

1593 password, err := precis.OpaqueString.String(password)

1594 if err != nil {

1595 return fmt.Errorf(`password not allowed by "precis"`)

1596 }

1597

1598 if len(password) < 8 {

1599 // We actually check for bytes...

1600 return fmt.Errorf("password must be at least 8 characters long")

1601 }

1602

1603 hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)

1604 if err != nil {

1605 return fmt.Errorf("generating password hash: %w", err)

1606 }

1607

1608 err = a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1609 if _, err := bstore.QueryTx[Password](tx).Delete(); err != nil {

1610 return fmt.Errorf("deleting existing password: %v", err)

1611 }

1612 var pw Password

1613 pw.Hash = string(hash)

1614

1615 // CRAM-MD5 calculates an HMAC-MD5, with the password as key, over a per-attempt

1616 // unique text that includes a timestamp. HMAC performs two hashes. Both times, the

1617 // first block is based on the key/password. We hash those first blocks now, and

1618 // store the hash state in the database. When we actually authenticate, we'll

1619 // complete the HMAC by hashing only the text. We cannot store crypto/hmac's hash,

1620 // because it does not expose its internal state and isn't a BinaryMarshaler.

1621 // ../rfc/2104:121

1622 pw.CRAMMD5.Ipad = md5.New()

1623 pw.CRAMMD5.Opad = md5.New()

1624 key := []byte(password)

1625 if len(key) > 64 {

1626 t := md5.Sum(key)

1627 key = t[:]

1628 }

1629 ipad := make([]byte, md5.BlockSize)

1630 opad := make([]byte, md5.BlockSize)

1631 copy(ipad, key)

1632 copy(opad, key)

1633 for i := range ipad {

1634 ipad[i] ^= 0x36

1635 opad[i] ^= 0x5c

1636 }

1637 pw.CRAMMD5.Ipad.Write(ipad)

1638 pw.CRAMMD5.Opad.Write(opad)

1639

1640 pw.SCRAMSHA1.Salt = scram.MakeRandom()

1641 pw.SCRAMSHA1.Iterations = 2 * 4096

1642 pw.SCRAMSHA1.SaltedPassword = scram.SaltPassword(sha1.New, password, pw.SCRAMSHA1.Salt, pw.SCRAMSHA1.Iterations)

1643

1644 pw.SCRAMSHA256.Salt = scram.MakeRandom()

1645 pw.SCRAMSHA256.Iterations = 4096

1646 pw.SCRAMSHA256.SaltedPassword = scram.SaltPassword(sha256.New, password, pw.SCRAMSHA256.Salt, pw.SCRAMSHA256.Iterations)

1647

1648 if err := tx.Insert(&pw); err != nil {

1649 return fmt.Errorf("inserting new password: %v", err)

1650 }

1651

1652 return sessionRemoveAll(context.TODO(), log, tx, a.Name)

1653 })

1654 if err == nil {

1655 log.Info("new password set for account", slog.String("account", a.Name))

1656 }

1657 return err

1658}

1659

1660// Subjectpass returns the signing key for use with subjectpass for the given

1661// email address with canonical localpart.

1662func (a *Account) Subjectpass(email string) (key string, err error) {

1663 return key, a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1664 v := Subjectpass{Email: email}

1665 err := tx.Get(&v)

1666 if err == nil {

1667 key = v.Key

1668 return nil

1669 }

1670 if !errors.Is(err, bstore.ErrAbsent) {

1671 return fmt.Errorf("get subjectpass key from accounts database: %w", err)

1672 }

1673 key = ""

1674 const chars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"

1675 buf := make([]byte, 16)

1676 if _, err := cryptorand.Read(buf); err != nil {

1677 return err

1678 }

1679 for _, b := range buf {

1680 key += string(chars[int(b)%len(chars)])

1681 }

1682 v.Key = key

1683 return tx.Insert(&v)

1684 })

1685}

1686

1687// Ensure mailbox is present in database, adding records for the mailbox and its

1688// parents if they aren't present.

1689//

1690// If subscribe is true, any mailboxes that were created will also be subscribed to.

1691// Caller must hold account wlock.

1692// Caller must propagate changes if any.

1693func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool) (mb Mailbox, changes []Change, rerr error) {

1694 if norm.NFC.String(name) != name {

1695 return Mailbox{}, nil, fmt.Errorf("mailbox name not normalized")

1696 }

1697

1698 // Quick sanity check.

1699 if strings.EqualFold(name, "inbox") && name != "Inbox" {

1700 return Mailbox{}, nil, fmt.Errorf("bad casing for inbox")

1701 }

1702

1703 elems := strings.Split(name, "/")

1704 q := bstore.QueryTx[Mailbox](tx)

1705 q.FilterFn(func(mb Mailbox) bool {

1706 return mb.Name == elems[0] || strings.HasPrefix(mb.Name, elems[0]+"/")

1707 })

1708 l, err := q.List()

1709 if err != nil {

1710 return Mailbox{}, nil, fmt.Errorf("list mailboxes: %v", err)

1711 }

1712

1713 mailboxes := map[string]Mailbox{}

1714 for _, xmb := range l {

1715 mailboxes[xmb.Name] = xmb

1716 }

1717

1718 p := ""

1719 for _, elem := range elems {

1720 if p != "" {

1721 p += "/"

1722 }

1723 p += elem

1724 var ok bool

1725 mb, ok = mailboxes[p]

1726 if ok {

1727 continue

1728 }

1729 uidval, err := a.NextUIDValidity(tx)

1730 if err != nil {

1731 return Mailbox{}, nil, fmt.Errorf("next uid validity: %v", err)

1732 }

1733 mb = Mailbox{

1734 Name: p,

1735 UIDValidity: uidval,

1736 UIDNext: 1,

1737 HaveCounts: true,

1738 }

1739 err = tx.Insert(&mb)

1740 if err != nil {

1741 return Mailbox{}, nil, fmt.Errorf("creating new mailbox: %v", err)

1742 }

1743

1744 var flags []string

1745 if subscribe {

1746 if tx.Get(&Subscription{p}) != nil {

1747 err := tx.Insert(&Subscription{p})

1748 if err != nil {

1749 return Mailbox{}, nil, fmt.Errorf("subscribing to mailbox: %v", err)

1750 }

1751 }

1752 flags = []string{`\Subscribed`}

1753 }

1754 changes = append(changes, ChangeAddMailbox{mb, flags})

1755 }

1756 return mb, changes, nil

1757}

1758

1759// MailboxExists checks if mailbox exists.

1760// Caller must hold account rlock.

1761func (a *Account) MailboxExists(tx *bstore.Tx, name string) (bool, error) {

1762 q := bstore.QueryTx[Mailbox](tx)

1763 q.FilterEqual("Name", name)

1764 return q.Exists()

1765}

1766

1767// MailboxFind finds a mailbox by name, returning a nil mailbox and nil error if mailbox does not exist.

1768func (a *Account) MailboxFind(tx *bstore.Tx, name string) (*Mailbox, error) {

1769 q := bstore.QueryTx[Mailbox](tx)

1770 q.FilterEqual("Name", name)

1771 mb, err := q.Get()

1772 if err == bstore.ErrAbsent {

1773 return nil, nil

1774 }

1775 if err != nil {

1776 return nil, fmt.Errorf("looking up mailbox: %w", err)

1777 }

1778 return &mb, nil

1779}

1780

1781// SubscriptionEnsure ensures a subscription for name exists. The mailbox does not

1782// have to exist. Any parents are not automatically subscribed.

1783// Changes are returned and must be broadcasted by the caller.

1784func (a *Account) SubscriptionEnsure(tx *bstore.Tx, name string) ([]Change, error) {

1785 if err := tx.Get(&Subscription{name}); err == nil {

1786 return nil, nil

1787 }

1788

1789 if err := tx.Insert(&Subscription{name}); err != nil {

1790 return nil, fmt.Errorf("inserting subscription: %w", err)

1791 }

1792

1793 q := bstore.QueryTx[Mailbox](tx)

1794 q.FilterEqual("Name", name)

1795 _, err := q.Get()

1796 if err == nil {

1797 return []Change{ChangeAddSubscription{name, nil}}, nil

1798 } else if err != bstore.ErrAbsent {

1799 return nil, fmt.Errorf("looking up mailbox for subscription: %w", err)

1800 }

1801 return []Change{ChangeAddSubscription{name, []string{`\NonExistent`}}}, nil

1802}

1803

1804// MessageRuleset returns the first ruleset (if any) that matches the message

1805// represented by msgPrefix and msgFile, with smtp and validation fields from m.

1806func MessageRuleset(log mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset {

1807 if len(dest.Rulesets) == 0 {

1808 return nil

1809 }

1810

1811 mr := FileMsgReader(msgPrefix, msgFile) // We don't close, it would close the msgFile.

1812 p, err := message.Parse(log.Logger, false, mr)

1813 if err != nil {

1814 log.Errorx("parsing message for evaluating rulesets, continuing with headers", err, slog.String("parse", ""))

1815 // note: part is still set.

1816 }

1817 // todo optimize: only parse header if needed for rulesets. and probably reuse an earlier parsing.

1818 header, err := p.Header()

1819 if err != nil {

1820 log.Errorx("parsing message headers for evaluating rulesets, delivering to default mailbox", err, slog.String("parse", ""))

1821 // todo: reject message?

1822 return nil

1823 }

1824

1825ruleset:

1826 for _, rs := range dest.Rulesets {

1827 if rs.SMTPMailFromRegexpCompiled != nil {

1828 if !rs.SMTPMailFromRegexpCompiled.MatchString(m.MailFrom) {

1829 continue ruleset

1830 }

1831 }

1832 if rs.MsgFromRegexpCompiled != nil {

1833 if m.MsgFromLocalpart == "" && m.MsgFromDomain == "" || !rs.MsgFromRegexpCompiled.MatchString(m.MsgFromLocalpart.String()+"@"+m.MsgFromDomain) {

1834 continue ruleset

1835 }

1836 }

1837

1838 if !rs.VerifiedDNSDomain.IsZero() {

1839 d := rs.VerifiedDNSDomain.Name()

1840 suffix := "." + d

1841 matchDomain := func(s string) bool {

1842 return s == d || strings.HasSuffix(s, suffix)

1843 }

1844 var ok bool

1845 if m.EHLOValidated && matchDomain(m.EHLODomain) {

1846 ok = true

1847 }

1848 if m.MailFromValidated && matchDomain(m.MailFromDomain) {

1849 ok = true

1850 }

1851 for _, d := range m.DKIMDomains {

1852 if matchDomain(d) {

1853 ok = true

1854 break

1855 }

1856 }

1857 if !ok {

1858 continue ruleset

1859 }

1860 }

1861

1862 header:

1863 for _, t := range rs.HeadersRegexpCompiled {

1864 for k, vl := range header {

1865 k = strings.ToLower(k)

1866 if !t[0].MatchString(k) {

1867 continue

1868 }

1869 for _, v := range vl {

1870 v = strings.ToLower(strings.TrimSpace(v))

1871 if t[1].MatchString(v) {

1872 continue header

1873 }

1874 }

1875 }

1876 continue ruleset

1877 }

1878 return &rs

1879 }

1880 return nil

1881}

1882

1883// MessagePath returns the file system path of a message.

1884func (a *Account) MessagePath(messageID int64) string {

1885 return strings.Join(append([]string{a.Dir, "msg"}, messagePathElems(messageID)...), string(filepath.Separator))

1886}

1887

1888// MessageReader opens a message for reading, transparently combining the

1889// message prefix with the original incoming message.

1890func (a *Account) MessageReader(m Message) *MsgReader {

1891 return &MsgReader{prefix: m.MsgPrefix, path: a.MessagePath(m.ID), size: m.Size}

1892}

1893

1894// DeliverDestination delivers an email to dest, based on the configured rulesets.

1895//

1896// Returns ErrOverQuota when account would be over quota after adding message.

1897//

1898// Caller must hold account wlock (mailbox may be created).

1899// Message delivery, possible mailbox creation, and updated mailbox counts are

1900// broadcasted.

1901func (a *Account) DeliverDestination(log mlog.Log, dest config.Destination, m *Message, msgFile *os.File) error {

1902 var mailbox string

1903 rs := MessageRuleset(log, dest, m, m.MsgPrefix, msgFile)

1904 if rs != nil {

1905 mailbox = rs.Mailbox

1906 } else if dest.Mailbox == "" {

1907 mailbox = "Inbox"

1908 } else {

1909 mailbox = dest.Mailbox

1910 }

1911 return a.DeliverMailbox(log, mailbox, m, msgFile)

1912}

1913

1914// DeliverMailbox delivers an email to the specified mailbox.

1915//

1916// Returns ErrOverQuota when account would be over quota after adding message.

1917//

1918// Caller must hold account wlock (mailbox may be created).

1919// Message delivery, possible mailbox creation, and updated mailbox counts are

1920// broadcasted.

1921func (a *Account) DeliverMailbox(log mlog.Log, mailbox string, m *Message, msgFile *os.File) error {

1922 var changes []Change

1923 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1924 if ok, _, err := a.CanAddMessageSize(tx, m.Size); err != nil {

1925 return err

1926 } else if !ok {

1927 return ErrOverQuota

1928 }

1929

1930 mb, chl, err := a.MailboxEnsure(tx, mailbox, true)

1931 if err != nil {

1932 return fmt.Errorf("ensuring mailbox: %w", err)

1933 }

1934 m.MailboxID = mb.ID

1935 m.MailboxOrigID = mb.ID

1936

1937 // Update count early, DeliverMessage will update mb too and we don't want to fetch

1938 // it again before updating.

1939 mb.MailboxCounts.Add(m.MailboxCounts())

1940 if err := tx.Update(&mb); err != nil {

1941 return fmt.Errorf("updating mailbox for delivery: %w", err)

1942 }

1943

1944 if err := a.DeliverMessage(log, tx, m, msgFile, true, false, false, true); err != nil {

1945 return err

1946 }

1947

1948 changes = append(changes, chl...)

1949 changes = append(changes, m.ChangeAddUID(), mb.ChangeCounts())

1950 return nil

1951 })

1952 // todo: if rename succeeded but transaction failed, we should remove the file.

1953 if err != nil {

1954 return err

1955 }

1956

1957 BroadcastChanges(a, changes)

1958 return nil

1959}

1960

1961// TidyRejectsMailbox removes old reject emails, and returns whether there is space for a new delivery.

1962//

1963// Caller most hold account wlock.

1964// Changes are broadcasted.

1965func (a *Account) TidyRejectsMailbox(log mlog.Log, rejectsMailbox string) (hasSpace bool, rerr error) {

1966 var changes []Change

1967

1968 var remove []Message

1969 defer func() {

1970 for _, m := range remove {

1971 p := a.MessagePath(m.ID)

1972 err := os.Remove(p)

1973 log.Check(err, "removing rejects message file", slog.String("path", p))

1974 }

1975 }()

1976

1977 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1978 mb, err := a.MailboxFind(tx, rejectsMailbox)

1979 if err != nil {

1980 return fmt.Errorf("finding mailbox: %w", err)

1981 }

1982 if mb == nil {

1983 // No messages have been delivered yet.

1984 hasSpace = true

1985 return nil

1986 }

1987

1988 // Gather old messages to remove.

1989 old := time.Now().Add(-14 * 24 * time.Hour)

1990 qdel := bstore.QueryTx[Message](tx)

1991 qdel.FilterNonzero(Message{MailboxID: mb.ID})

1992 qdel.FilterEqual("Expunged", false)

1993 qdel.FilterLess("Received", old)

1994 remove, err = qdel.List()

1995 if err != nil {

1996 return fmt.Errorf("listing old messages: %w", err)

1997 }

1998

1999 changes, err = a.rejectsRemoveMessages(context.TODO(), log, tx, mb, remove)

2000 if err != nil {

2001 return fmt.Errorf("removing messages: %w", err)

2002 }

2003

2004 // We allow up to n messages.

2005 qcount := bstore.QueryTx[Message](tx)

2006 qcount.FilterNonzero(Message{MailboxID: mb.ID})

2007 qcount.FilterEqual("Expunged", false)

2008 qcount.Limit(1000)

2009 n, err := qcount.Count()

2010 if err != nil {

2011 return fmt.Errorf("counting rejects: %w", err)

2012 }

2013 hasSpace = n < 1000

2014

2015 return nil

2016 })

2017 if err != nil {

2018 remove = nil // Don't remove files on failure.

2019 return false, err

2020 }

2021

2022 BroadcastChanges(a, changes)

2023

2024 return hasSpace, nil

2025}

2026

2027func (a *Account) rejectsRemoveMessages(ctx context.Context, log mlog.Log, tx *bstore.Tx, mb *Mailbox, l []Message) ([]Change, error) {

2028 if len(l) == 0 {

2029 return nil, nil

2030 }

2031 ids := make([]int64, len(l))

2032 anyids := make([]any, len(l))

2033 for i, m := range l {

2034 ids[i] = m.ID

2035 anyids[i] = m.ID

2036 }

2037

2038 // Remove any message recipients. Should not happen, but a user can move messages

2039 // from a Sent mailbox to the rejects mailbox...

2040 qdmr := bstore.QueryTx[Recipient](tx)

2041 qdmr.FilterEqual("MessageID", anyids...)

2042 if _, err := qdmr.Delete(); err != nil {

2043 return nil, fmt.Errorf("deleting from message recipient: %w", err)

2044 }

2045

2046 // Assign new modseq.

2047 modseq, err := a.NextModSeq(tx)

2048 if err != nil {

2049 return nil, fmt.Errorf("assign next modseq: %w", err)

2050 }

2051

2052 // Expunge the messages.

2053 qx := bstore.QueryTx[Message](tx)

2054 qx.FilterIDs(ids)

2055 var expunged []Message

2056 qx.Gather(&expunged)

2057 if _, err := qx.UpdateNonzero(Message{ModSeq: modseq, Expunged: true}); err != nil {

2058 return nil, fmt.Errorf("expunging messages: %w", err)

2059 }

2060

2061 var totalSize int64

2062 for _, m := range expunged {

2063 m.Expunged = false // Was set by update, but would cause wrong count.

2064 mb.MailboxCounts.Sub(m.MailboxCounts())

2065 totalSize += m.Size

2066 }

2067 if err := tx.Update(mb); err != nil {

2068 return nil, fmt.Errorf("updating mailbox counts: %w", err)

2069 }

2070 if err := a.AddMessageSize(log, tx, -totalSize); err != nil {

2071 return nil, fmt.Errorf("updating disk usage: %w", err)

2072 }

2073

2074 // Mark as neutral and train so junk filter gets untrained with these (junk) messages.

2075 for i := range expunged {

2076 expunged[i].Junk = false

2077 expunged[i].Notjunk = false

2078 }

2079 if err := a.RetrainMessages(ctx, log, tx, expunged, true); err != nil {

2080 return nil, fmt.Errorf("retraining expunged messages: %w", err)

2081 }

2082

2083 changes := make([]Change, len(l), len(l)+1)

2084 for i, m := range l {

2085 changes[i] = ChangeRemoveUIDs{mb.ID, []UID{m.UID}, modseq}

2086 }

2087 changes = append(changes, mb.ChangeCounts())

2088 return changes, nil

2089}

2090

2091// RejectsRemove removes a message from the rejects mailbox if present.

2092// Caller most hold account wlock.

2093// Changes are broadcasted.

2094func (a *Account) RejectsRemove(log mlog.Log, rejectsMailbox, messageID string) error {

2095 var changes []Change

2096

2097 var remove []Message

2098 defer func() {

2099 for _, m := range remove {

2100 p := a.MessagePath(m.ID)

2101 err := os.Remove(p)

2102 log.Check(err, "removing rejects message file", slog.String("path", p))

2103 }

2104 }()

2105

2106 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

2107 mb, err := a.MailboxFind(tx, rejectsMailbox)

2108 if err != nil {

2109 return fmt.Errorf("finding mailbox: %w", err)

2110 }

2111 if mb == nil {

2112 return nil

2113 }

2114

2115 q := bstore.QueryTx[Message](tx)

2116 q.FilterNonzero(Message{MailboxID: mb.ID, MessageID: messageID})

2117 q.FilterEqual("Expunged", false)

2118 remove, err = q.List()

2119 if err != nil {

2120 return fmt.Errorf("listing messages to remove: %w", err)

2121 }

2122

2123 changes, err = a.rejectsRemoveMessages(context.TODO(), log, tx, mb, remove)

2124 if err != nil {

2125 return fmt.Errorf("removing messages: %w", err)

2126 }

2127

2128 return nil

2129 })

2130 if err != nil {

2131 remove = nil // Don't remove files on failure.

2132 return err

2133 }

2134

2135 BroadcastChanges(a, changes)

2136

2137 return nil

2138}

2139

2140// AddMessageSize adjusts the DiskUsage.MessageSize by size.

2141func (a *Account) AddMessageSize(log mlog.Log, tx *bstore.Tx, size int64) error {

2142 du := DiskUsage{ID: 1}

2143 if err := tx.Get(&du); err != nil {

2144 return fmt.Errorf("get diskusage: %v", err)

2145 }

2146 du.MessageSize += size

2147 if du.MessageSize < 0 {

2148 log.Error("negative total message size", slog.Int64("delta", size), slog.Int64("newtotalsize", du.MessageSize))

2149 }

2150 if err := tx.Update(&du); err != nil {

2151 return fmt.Errorf("update total message size: %v", err)

2152 }

2153 return nil

2154}

2155

2156// QuotaMessageSize returns the effective maximum total message size for an

2157// account. Returns 0 if there is no maximum.

2158func (a *Account) QuotaMessageSize() int64 {

2159 conf, _ := a.Conf()

2160 size := conf.QuotaMessageSize

2161 if size == 0 {

2162 size = mox.Conf.Static.QuotaMessageSize

2163 }

2164 if size < 0 {

2165 size = 0

2166 }

2167 return size

2168}

2169

2170// CanAddMessageSize checks if a message of size bytes can be added, depending on

2171// total message size and configured quota for account.

2172func (a *Account) CanAddMessageSize(tx *bstore.Tx, size int64) (ok bool, maxSize int64, err error) {

2173 maxSize = a.QuotaMessageSize()

2174 if maxSize <= 0 {

2175 return true, 0, nil

2176 }

2177

2178 du := DiskUsage{ID: 1}

2179 if err := tx.Get(&du); err != nil {

2180 return false, maxSize, fmt.Errorf("get diskusage: %v", err)

2181 }

2182 return du.MessageSize+size <= maxSize, maxSize, nil

2183}

2184

2185// We keep a cache of recent successful authentications, so we don't have to bcrypt successful calls each time.

2186var authCache = struct {

2187 sync.Mutex

2188 success map[authKey]string

2189}{

2190 success: map[authKey]string{},

2191}

2192

2193type authKey struct {

2194 email, hash string

2195}

2196

2197// StartAuthCache starts a goroutine that regularly clears the auth cache.

2198func StartAuthCache() {

2199 go manageAuthCache()

2200}

2201

2202func manageAuthCache() {

2203 for {

2204 authCache.Lock()

2205 authCache.success = map[authKey]string{}

2206 authCache.Unlock()

2207 time.Sleep(15 * time.Minute)

2208 }

2209}

2210

2211// OpenEmailAuth opens an account given an email address and password.

2212//

2213// The email address may contain a catchall separator.

2214func OpenEmailAuth(log mlog.Log, email string, password string) (acc *Account, rerr error) {

2215 password, err := precis.OpaqueString.String(password)

2216 if err != nil {

2217 return nil, ErrUnknownCredentials

2218 }

2219

2220 acc, _, rerr = OpenEmail(log, email)

2221 if rerr != nil {

2222 return

2223 }

2224

2225 defer func() {

2226 if rerr != nil && acc != nil {

2227 err := acc.Close()

2228 log.Check(err, "closing account after open auth failure")

2229 acc = nil

2230 }

2231 }()

2232

2233 pw, err := bstore.QueryDB[Password](context.TODO(), acc.DB).Get()

2234 if err != nil {

2235 if err == bstore.ErrAbsent {

2236 return acc, ErrUnknownCredentials

2237 }

2238 return acc, fmt.Errorf("looking up password: %v", err)

2239 }

2240 authCache.Lock()

2241 ok := len(password) >= 8 && authCache.success[authKey{email, pw.Hash}] == password

2242 authCache.Unlock()

2243 if ok {

2244 return

2245 }

2246 if err := bcrypt.CompareHashAndPassword([]byte(pw.Hash), []byte(password)); err != nil {

2247 rerr = ErrUnknownCredentials

2248 } else {

2249 authCache.Lock()

2250 authCache.success[authKey{email, pw.Hash}] = password

2251 authCache.Unlock()

2252 }

2253 return

2254}

2255

2256// OpenEmail opens an account given an email address.

2257//

2258// The email address may contain a catchall separator.

2259func OpenEmail(log mlog.Log, email string) (*Account, config.Destination, error) {

2260 addr, err := smtp.ParseAddress(email)

2261 if err != nil {

2262 return nil, config.Destination{}, fmt.Errorf("%w: %v", ErrUnknownCredentials, err)

2263 }

2264 accountName, _, _, dest, err := mox.LookupAddress(addr.Localpart, addr.Domain, false, false)

2265 if err != nil && (errors.Is(err, mox.ErrAddressNotFound) || errors.Is(err, mox.ErrDomainNotFound)) {

2266 return nil, config.Destination{}, ErrUnknownCredentials

2267 } else if err != nil {

2268 return nil, config.Destination{}, fmt.Errorf("looking up address: %v", err)

2269 }

2270 acc, err := OpenAccount(log, accountName)

2271 if err != nil {

2272 return nil, config.Destination{}, err

2273 }

2274 return acc, dest, nil

2275}

2276

2277// 64 characters, must be power of 2 for MessagePath

2278const msgDirChars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ-_"

2279

2280// MessagePath returns the filename of the on-disk filename, relative to the

2281// containing directory such as <account>/msg or queue.

2282// Returns names like "AB/1".

2283func MessagePath(messageID int64) string {

2284 return strings.Join(messagePathElems(messageID), string(filepath.Separator))

2285}

2286

2287// messagePathElems returns the elems, for a single join without intermediate

2288// string allocations.

2289func messagePathElems(messageID int64) []string {

2290 v := messageID >> 13 // 8k files per directory.

2291 dir := ""

2292 for {

2293 dir += string(msgDirChars[int(v)&(len(msgDirChars)-1)])

2294 v >>= 6

2295 if v == 0 {

2296 break

2297 }

2298 }

2299 return []string{dir, strconv.FormatInt(messageID, 10)}

2300}

2301

2302// Set returns a copy of f, with each flag that is true in mask set to the

2303// value from flags.

2304func (f Flags) Set(mask, flags Flags) Flags {

2305 set := func(d *bool, m, v bool) {

2306 if m {

2307 *d = v

2308 }

2309 }

2310 r := f

2311 set(&r.Seen, mask.Seen, flags.Seen)

2312 set(&r.Answered, mask.Answered, flags.Answered)

2313 set(&r.Flagged, mask.Flagged, flags.Flagged)

2314 set(&r.Forwarded, mask.Forwarded, flags.Forwarded)

2315 set(&r.Junk, mask.Junk, flags.Junk)

2316 set(&r.Notjunk, mask.Notjunk, flags.Notjunk)

2317 set(&r.Deleted, mask.Deleted, flags.Deleted)

2318 set(&r.Draft, mask.Draft, flags.Draft)

2319 set(&r.Phishing, mask.Phishing, flags.Phishing)

2320 set(&r.MDNSent, mask.MDNSent, flags.MDNSent)

2321 return r

2322}

2323

2324// Changed returns a mask of flags that have been between f and other.

2325func (f Flags) Changed(other Flags) (mask Flags) {

2326 mask.Seen = f.Seen != other.Seen

2327 mask.Answered = f.Answered != other.Answered

2328 mask.Flagged = f.Flagged != other.Flagged

2329 mask.Forwarded = f.Forwarded != other.Forwarded

2330 mask.Junk = f.Junk != other.Junk

2331 mask.Notjunk = f.Notjunk != other.Notjunk

2332 mask.Deleted = f.Deleted != other.Deleted

2333 mask.Draft = f.Draft != other.Draft

2334 mask.Phishing = f.Phishing != other.Phishing

2335 mask.MDNSent = f.MDNSent != other.MDNSent

2336 return

2337}

2338

2339// Strings returns the flags that are set in their string form.

2340func (f Flags) Strings() []string {

2341 fields := []struct {

2342 word string

2343 have bool

2344 }{

2345 {`$forwarded`, f.Forwarded},

2346 {`$junk`, f.Junk},

2347 {`$mdnsent`, f.MDNSent},

2348 {`$notjunk`, f.Notjunk},

2349 {`$phishing`, f.Phishing},

2350 {`\answered`, f.Answered},

2351 {`\deleted`, f.Deleted},

2352 {`\draft`, f.Draft},

2353 {`\flagged`, f.Flagged},

2354 {`\seen`, f.Seen},

2355 }

2356 var l []string

2357 for _, fh := range fields {

2358 if fh.have {

2359 l = append(l, fh.word)

2360 }

2361 }

2362 return l

2363}

2364

2365var systemWellKnownFlags = map[string]bool{

2366 `\answered`: true,

2367 `\flagged`: true,

2368 `\deleted`: true,

2369 `\seen`: true,

2370 `\draft`: true,

2371 `$junk`: true,

2372 `$notjunk`: true,

2373 `$forwarded`: true,

2374 `$phishing`: true,

2375 `$mdnsent`: true,

2376}

2377

2378// ParseFlagsKeywords parses a list of textual flags into system/known flags, and

2379// other keywords. Keywords are lower-cased and sorted and check for valid syntax.

2380func ParseFlagsKeywords(l []string) (flags Flags, keywords []string, rerr error) {

2381 fields := map[string]*bool{

2382 `\answered`: &flags.Answered,

2383 `\flagged`: &flags.Flagged,

2384 `\deleted`: &flags.Deleted,

2385 `\seen`: &flags.Seen,

2386 `\draft`: &flags.Draft,

2387 `$junk`: &flags.Junk,

2388 `$notjunk`: &flags.Notjunk,

2389 `$forwarded`: &flags.Forwarded,

2390 `$phishing`: &flags.Phishing,

2391 `$mdnsent`: &flags.MDNSent,

2392 }

2393 seen := map[string]bool{}

2394 for _, f := range l {

2395 f = strings.ToLower(f)

2396 if field, ok := fields[f]; ok {

2397 *field = true

2398 } else if seen[f] {

2399 if mox.Pedantic {

2400 return Flags{}, nil, fmt.Errorf("duplicate keyword %s", f)

2401 }

2402 } else {

2403 if err := CheckKeyword(f); err != nil {

2404 return Flags{}, nil, fmt.Errorf("invalid keyword %s", f)

2405 }

2406 keywords = append(keywords, f)

2407 seen[f] = true

2408 }

2409 }

2410 sort.Strings(keywords)

2411 return flags, keywords, nil

2412}

2413

2414// RemoveKeywords removes keywords from l, returning whether any modifications were

2415// made, and a slice, a new slice in case of modifications. Keywords must have been

2416// validated earlier, e.g. through ParseFlagKeywords or CheckKeyword. Should only

2417// be used with valid keywords, not with system flags like \Seen.

2418func RemoveKeywords(l, remove []string) ([]string, bool) {

2419 var copied bool

2420 var changed bool

2421 for _, k := range remove {

2422 if i := slices.Index(l, k); i >= 0 {

2423 if !copied {

2424 l = append([]string{}, l...)

2425 copied = true

2426 }

2427 copy(l[i:], l[i+1:])

2428 l = l[:len(l)-1]

2429 changed = true

2430 }

2431 }

2432 return l, changed

2433}

2434

2435// MergeKeywords adds keywords from add into l, returning whether it added any

2436// keyword, and the slice with keywords, a new slice if modifications were made.

2437// Keywords are only added if they aren't already present. Should only be used with

2438// keywords, not with system flags like \Seen.

2439func MergeKeywords(l, add []string) ([]string, bool) {

2440 var copied bool

2441 var changed bool

2442 for _, k := range add {

2443 if !slices.Contains(l, k) {

2444 if !copied {

2445 l = append([]string{}, l...)

2446 copied = true

2447 }

2448 l = append(l, k)

2449 changed = true

2450 }

2451 }

2452 if changed {

2453 sort.Strings(l)

2454 }

2455 return l, changed

2456}

2457

2458// CheckKeyword returns an error if kw is not a valid keyword. Kw should

2459// already be in lower-case.

2460func CheckKeyword(kw string) error {

2461 if kw == "" {

2462 return fmt.Errorf("keyword cannot be empty")

2463 }

2464 if systemWellKnownFlags[kw] {

2465 return fmt.Errorf("cannot use well-known flag as keyword")

2466 }

2467 for _, c := range kw {

2468 // ../rfc/9051:6334

2469 if c <= ' ' || c > 0x7e || c >= 'A' && c <= 'Z' || strings.ContainsRune(`(){%*"\]`, c) {

2470 return errors.New(`not a valid keyword, must be lower-case ascii without spaces and without any of these characters: (){%*"\]`)

2471 }

2472 }

2473 return nil

2474}

2475

2476// SendLimitReached checks whether sending a message to recipients would reach

2477// the limit of outgoing messages for the account. If so, the message should

2478// not be sent. If the returned numbers are >= 0, the limit was reached and the

2479// values are the configured limits.

2480//

2481// To limit damage to the internet and our reputation in case of account

2482// compromise, we limit the max number of messages sent in a 24 hour window, both

2483// total number of messages and number of first-time recipients.

2484func (a *Account) SendLimitReached(tx *bstore.Tx, recipients []smtp.Path) (msglimit, rcptlimit int, rerr error) {

2485 conf, _ := a.Conf()

2486 msgmax := conf.MaxOutgoingMessagesPerDay

2487 if msgmax == 0 {

2488 // For human senders, 1000 recipients in a day is quite a lot.

2489 msgmax = 1000

2490 }

2491 rcptmax := conf.MaxFirstTimeRecipientsPerDay

2492 if rcptmax == 0 {

2493 // Human senders may address a new human-sized list of people once in a while. In

2494 // case of a compromise, a spammer will probably try to send to many new addresses.

2495 rcptmax = 200

2496 }

2497

2498 rcpts := map[string]time.Time{}

2499 n := 0

2500 err := bstore.QueryTx[Outgoing](tx).FilterGreater("Submitted", time.Now().Add(-24*time.Hour)).ForEach(func(o Outgoing) error {

2501 n++

2502 if rcpts[o.Recipient].IsZero() || o.Submitted.Before(rcpts[o.Recipient]) {

2503 rcpts[o.Recipient] = o.Submitted

2504 }

2505 return nil

2506 })

2507 if err != nil {

2508 return -1, -1, fmt.Errorf("querying message recipients in past 24h: %w", err)

2509 }

2510 if n+len(recipients) > msgmax {

2511 return msgmax, -1, nil

2512 }

2513

2514 // Only check if max first-time recipients is reached if there are enough messages

2515 // to trigger the limit.

2516 if n+len(recipients) < rcptmax {

2517 return -1, -1, nil

2518 }

2519

2520 isFirstTime := func(rcpt string, before time.Time) (bool, error) {

2521 exists, err := bstore.QueryTx[Outgoing](tx).FilterNonzero(Outgoing{Recipient: rcpt}).FilterLess("Submitted", before).Exists()

2522 return !exists, err

2523 }

2524

2525 firsttime := 0

2526 now := time.Now()

2527 for _, r := range recipients {

2528 if first, err := isFirstTime(r.XString(true), now); err != nil {

2529 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

2530 } else if first {

2531 firsttime++

2532 }

2533 }

2534 for r, t := range rcpts {

2535 if first, err := isFirstTime(r, t); err != nil {

2536 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

2537 } else if first {

2538 firsttime++

2539 }

2540 }

2541 if firsttime > rcptmax {

2542 return -1, rcptmax, nil

2543 }

2544 return -1, -1, nil

2545}

2546

2547// MailboxCreate creates a new mailbox, including any missing parent mailboxes,

2548// the total list of created mailboxes is returned in created. On success, if

2549// exists is false and rerr nil, the changes must be broadcasted by the caller.

2550//

2551// Name must be in normalized form.

2552func (a *Account) MailboxCreate(tx *bstore.Tx, name string) (changes []Change, created []string, exists bool, rerr error) {

2553 elems := strings.Split(name, "/")

2554 var p string

2555 for i, elem := range elems {

2556 if i > 0 {

2557 p += "/"

2558 }

2559 p += elem

2560 exists, err := a.MailboxExists(tx, p)

2561 if err != nil {

2562 return nil, nil, false, fmt.Errorf("checking if mailbox exists")

2563 }

2564 if exists {

2565 if i == len(elems)-1 {

2566 return nil, nil, true, fmt.Errorf("mailbox already exists")

2567 }

2568 continue

2569 }

2570 _, nchanges, err := a.MailboxEnsure(tx, p, true)

2571 if err != nil {

2572 return nil, nil, false, fmt.Errorf("ensuring mailbox exists")

2573 }

2574 changes = append(changes, nchanges...)

2575 created = append(created, p)

2576 }

2577 return changes, created, false, nil

2578}

2579

2580// MailboxRename renames mailbox mbsrc to dst, and any missing parents for the

2581// destination, and any children of mbsrc and the destination.

2582//

2583// Names must be normalized and cannot be Inbox.

2584func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc Mailbox, dst string) (changes []Change, isInbox, notExists, alreadyExists bool, rerr error) {

2585 if mbsrc.Name == "Inbox" || dst == "Inbox" {

2586 return nil, true, false, false, fmt.Errorf("inbox cannot be renamed")

2587 }

2588

2589 // We gather existing mailboxes that we need for deciding what to create/delete/update.

2590 q := bstore.QueryTx[Mailbox](tx)

2591 srcPrefix := mbsrc.Name + "/"

2592 dstRoot := strings.SplitN(dst, "/", 2)[0]

2593 dstRootPrefix := dstRoot + "/"

2594 q.FilterFn(func(mb Mailbox) bool {

2595 return mb.Name == mbsrc.Name || strings.HasPrefix(mb.Name, srcPrefix) || mb.Name == dstRoot || strings.HasPrefix(mb.Name, dstRootPrefix)

2596 })

2597 q.SortAsc("Name") // We'll rename the parents before children.

2598 l, err := q.List()

2599 if err != nil {

2600 return nil, false, false, false, fmt.Errorf("listing relevant mailboxes: %v", err)

2601 }

2602

2603 mailboxes := map[string]Mailbox{}

2604 for _, mb := range l {

2605 mailboxes[mb.Name] = mb

2606 }

2607

2608 if _, ok := mailboxes[mbsrc.Name]; !ok {

2609 return nil, false, true, false, fmt.Errorf("mailbox does not exist")

2610 }

2611

2612 uidval, err := a.NextUIDValidity(tx)

2613 if err != nil {

2614 return nil, false, false, false, fmt.Errorf("next uid validity: %v", err)

2615 }

2616

2617 // Ensure parent mailboxes for the destination paths exist.

2618 var parent string

2619 dstElems := strings.Split(dst, "/")

2620 for i, elem := range dstElems[:len(dstElems)-1] {

2621 if i > 0 {

2622 parent += "/"

2623 }

2624 parent += elem

2625

2626 mb, ok := mailboxes[parent]

2627 if ok {

2628 continue

2629 }

2630 omb := mb

2631 mb = Mailbox{

2632 ID: omb.ID,

2633 Name: parent,

2634 UIDValidity: uidval,

2635 UIDNext: 1,

2636 HaveCounts: true,

2637 }

2638 if err := tx.Insert(&mb); err != nil {

2639 return nil, false, false, false, fmt.Errorf("creating parent mailbox %q: %v", mb.Name, err)

2640 }

2641 if err := tx.Get(&Subscription{Name: parent}); err != nil {

2642 if err := tx.Insert(&Subscription{Name: parent}); err != nil {

2643 return nil, false, false, false, fmt.Errorf("creating subscription for %q: %v", parent, err)

2644 }

2645 }

2646 changes = append(changes, ChangeAddMailbox{Mailbox: mb, Flags: []string{`\Subscribed`}})

2647 }

2648

2649 // Process src mailboxes, renaming them to dst.

2650 for _, srcmb := range l {

2651 if srcmb.Name != mbsrc.Name && !strings.HasPrefix(srcmb.Name, srcPrefix) {

2652 continue

2653 }

2654 srcName := srcmb.Name

2655 dstName := dst + srcmb.Name[len(mbsrc.Name):]

2656 if _, ok := mailboxes[dstName]; ok {

2657 return nil, false, false, true, fmt.Errorf("destination mailbox %q already exists", dstName)

2658 }

2659

2660 srcmb.Name = dstName

2661 srcmb.UIDValidity = uidval

2662 if err := tx.Update(&srcmb); err != nil {

2663 return nil, false, false, false, fmt.Errorf("renaming mailbox: %v", err)

2664 }

2665

2666 var dstFlags []string

2667 if tx.Get(&Subscription{Name: dstName}) == nil {

2668 dstFlags = []string{`\Subscribed`}

2669 }

2670 changes = append(changes, ChangeRenameMailbox{MailboxID: srcmb.ID, OldName: srcName, NewName: dstName, Flags: dstFlags})

2671 }

2672

2673 // If we renamed e.g. a/b to a/b/c/d, and a/b/c to a/b/c/d/c, we'll have to recreate a/b and a/b/c.

2674 srcElems := strings.Split(mbsrc.Name, "/")

2675 xsrc := mbsrc.Name

2676 for i := 0; i < len(dstElems) && strings.HasPrefix(dst, xsrc+"/"); i++ {

2677 mb := Mailbox{

2678 UIDValidity: uidval,

2679 UIDNext: 1,

2680 Name: xsrc,

2681 HaveCounts: true,

2682 }

2683 if err := tx.Insert(&mb); err != nil {

2684 return nil, false, false, false, fmt.Errorf("creating mailbox at old path %q: %v", mb.Name, err)

2685 }

2686 xsrc += "/" + dstElems[len(srcElems)+i]

2687 }

2688 return changes, false, false, false, nil

2689}

2690

2691// MailboxDelete deletes a mailbox by ID. If it has children, the return value

2692// indicates that and an error is returned.

2693//

2694// Caller should broadcast the changes and remove files for the removed message IDs.

2695func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx, mailbox Mailbox) (changes []Change, removeMessageIDs []int64, hasChildren bool, rerr error) {

2696 // Look for existence of child mailboxes. There is a lot of text in the IMAP RFCs about

2697 // NoInferior and NoSelect. We just require only leaf mailboxes are deleted.

2698 qmb := bstore.QueryTx[Mailbox](tx)

2699 mbprefix := mailbox.Name + "/"

2700 qmb.FilterFn(func(mb Mailbox) bool {

2701 return strings.HasPrefix(mb.Name, mbprefix)

2702 })

2703 if childExists, err := qmb.Exists(); err != nil {

2704 return nil, nil, false, fmt.Errorf("checking if mailbox has child: %v", err)

2705 } else if childExists {

2706 return nil, nil, true, fmt.Errorf("mailbox has a child, only leaf mailboxes can be deleted")

2707 }

2708

2709 // todo jmap: instead of completely deleting a mailbox and its messages, we need to mark them all as expunged.

2710

2711 qm := bstore.QueryTx[Message](tx)

2712 qm.FilterNonzero(Message{MailboxID: mailbox.ID})

2713 remove, err := qm.List()

2714 if err != nil {

2715 return nil, nil, false, fmt.Errorf("listing messages to remove: %v", err)

2716 }

2717

2718 if len(remove) > 0 {

2719 removeIDs := make([]any, len(remove))

2720 for i, m := range remove {

2721 removeIDs[i] = m.ID

2722 }

2723 qmr := bstore.QueryTx[Recipient](tx)

2724 qmr.FilterEqual("MessageID", removeIDs...)

2725 if _, err = qmr.Delete(); err != nil {

2726 return nil, nil, false, fmt.Errorf("removing message recipients for messages: %v", err)

2727 }

2728

2729 qm = bstore.QueryTx[Message](tx)

2730 qm.FilterNonzero(Message{MailboxID: mailbox.ID})

2731 if _, err := qm.Delete(); err != nil {

2732 return nil, nil, false, fmt.Errorf("removing messages: %v", err)

2733 }

2734

2735 var totalSize int64

2736 for _, m := range remove {

2737 if !m.Expunged {

2738 removeMessageIDs = append(removeMessageIDs, m.ID)

2739 totalSize += m.Size

2740 }

2741 }

2742 if err := a.AddMessageSize(log, tx, -totalSize); err != nil {

2743 return nil, nil, false, fmt.Errorf("updating disk usage: %v", err)

2744 }

2745

2746 // Mark messages as not needing training. Then retrain them, so they are untrained if they were.

2747 n := 0

2748 o := 0

2749 for _, m := range remove {

2750 if !m.Expunged {

2751 remove[o] = m

2752 remove[o].Junk = false

2753 remove[o].Notjunk = false

2754 n++

2755 }

2756 }

2757 remove = remove[:n]

2758 if err := a.RetrainMessages(ctx, log, tx, remove, true); err != nil {

2759 return nil, nil, false, fmt.Errorf("untraining deleted messages: %v", err)

2760 }

2761 }

2762

2763 if err := tx.Delete(&Mailbox{ID: mailbox.ID}); err != nil {

2764 return nil, nil, false, fmt.Errorf("removing mailbox: %v", err)

2765 }

2766 return []Change{ChangeRemoveMailbox{MailboxID: mailbox.ID, Name: mailbox.Name}}, removeMessageIDs, false, nil

2767}

2768

2769// CheckMailboxName checks if name is valid, returning an INBOX-normalized name.

2770// I.e. it changes various casings of INBOX and INBOX/* to Inbox and Inbox/*.

2771// Name is invalid if it contains leading/trailing/double slashes, or when it isn't

2772// unicode-normalized, or when empty or has special characters.

2773//

2774// If name is the inbox, and allowInbox is false, this is indicated with the isInbox return parameter.

2775// For that case, and for other invalid names, an error is returned.

2776func CheckMailboxName(name string, allowInbox bool) (normalizedName string, isInbox bool, rerr error) {

2777 first := strings.SplitN(name, "/", 2)[0]

2778 if strings.EqualFold(first, "inbox") {

2779 if len(name) == len("inbox") && !allowInbox {

2780 return "", true, fmt.Errorf("special mailbox name Inbox not allowed")

2781 }

2782 name = "Inbox" + name[len("Inbox"):]

2783 }

2784

2785 if norm.NFC.String(name) != name {

2786 return "", false, errors.New("non-unicode-normalized mailbox names not allowed")

2787 }

2788

2789 if name == "" {

2790 return "", false, errors.New("empty mailbox name")

2791 }

2792 if strings.HasPrefix(name, "/") || strings.HasSuffix(name, "/") || strings.Contains(name, "//") {

2793 return "", false, errors.New("bad slashes in mailbox name")

2794 }

2795

2796 // "%" and "*" are difficult to use with the IMAP LIST command, but we allow mostly

2797 // allow them. ../rfc/3501:1002 ../rfc/9051:983

2798 if strings.HasPrefix(name, "#") {

2799 return "", false, errors.New("mailbox name cannot start with hash due to conflict with imap namespaces")

2800 }

2801

2802 // "#" and "&" are special in IMAP mailbox names. "#" for namespaces, "&" for

2803 // IMAP-UTF-7 encoding. We do allow them. ../rfc/3501:1018 ../rfc/9051:991

2804

2805 for _, c := range name {

2806 // ../rfc/3501:999 ../rfc/6855:192 ../rfc/9051:979

2807 if c <= 0x1f || c >= 0x7f && c <= 0x9f || c == 0x2028 || c == 0x2029 {

2808 return "", false, errors.New("control characters not allowed in mailbox name")

2809 }

2810 }

2811 return name, false, nil

2812}

2813