1/*

2Package store implements storage for accounts, their mailboxes, IMAP

3subscriptions and messages, and broadcasts updates (e.g. mail delivery) to

4interested sessions (e.g. IMAP connections).

5

6Layout of storage for accounts:

7

8 <DataDir>/accounts/<name>/index.db

9 <DataDir>/accounts/<name>/msg/[a-zA-Z0-9_-]+/<id>

10

11Index.db holds tables for user information, mailboxes, and messages. Messages

12are stored in the msg/ subdirectory, each in their own file. The on-disk message

13does not contain headers generated during an incoming SMTP transaction, such as

14Received and Authentication-Results headers. Those are in the database to

15prevent having to rewrite incoming messages (e.g. Authentication-Result for DKIM

16signatures can only be determined after having read the message). Messages must

17be read through MsgReader, which transparently adds the prefix from the

18database.

19*/

20package store

21

22// todo: make up a function naming scheme that indicates whether caller should broadcast changes.

23

24import (

25 "context"

26 "crypto/md5"

27 cryptorand "crypto/rand"

28 "crypto/sha1"

29 "crypto/sha256"

30 "encoding"

31 "encoding/json"

32 "errors"

33 "fmt"

34 "hash"

35 "io"

36 "log/slog"

37 "os"

38 "path/filepath"

39 "runtime/debug"

40 "slices"

41 "sort"

42 "strconv"

43 "strings"

44 "sync"

45 "time"

46

47 "golang.org/x/crypto/bcrypt"

48 "golang.org/x/text/secure/precis"

49 "golang.org/x/text/unicode/norm"

50

51 "github.com/mjl-/bstore"

52

53 "github.com/mjl-/mox/config"

54 "github.com/mjl-/mox/dns"

55 "github.com/mjl-/mox/message"

56 "github.com/mjl-/mox/metrics"

57 "github.com/mjl-/mox/mlog"

58 "github.com/mjl-/mox/mox-"

59 "github.com/mjl-/mox/moxio"

60 "github.com/mjl-/mox/publicsuffix"

61 "github.com/mjl-/mox/scram"

62 "github.com/mjl-/mox/smtp"

63)

64

65// If true, each time an account is closed its database file is checked for

66// consistency. If an inconsistency is found, panic is called. Set by default

67// because of all the packages with tests, the mox main function sets it to

68// false again.

69var CheckConsistencyOnClose = true

70

71var (

72 ErrUnknownMailbox = errors.New("no such mailbox")

73 ErrUnknownCredentials = errors.New("credentials not found")

74 ErrAccountUnknown = errors.New("no such account")

75 ErrOverQuota = errors.New("account over quota")

76)

77

78var DefaultInitialMailboxes = config.InitialMailboxes{

79 SpecialUse: config.SpecialUseMailboxes{

80 Sent: "Sent",

81 Archive: "Archive",

82 Trash: "Trash",

83 Draft: "Drafts",

84 Junk: "Junk",

85 },

86}

87

88type SCRAM struct {

89 Salt []byte

90 Iterations int

91 SaltedPassword []byte

92}

93

94// CRAMMD5 holds HMAC ipad and opad hashes that are initialized with the first

95// block with (a derivation of) the key/password, so we don't store the password in plain

96// text.

97type CRAMMD5 struct {

98 Ipad hash.Hash

99 Opad hash.Hash

100}

101

102// BinaryMarshal is used by bstore to store the ipad/opad hash states.

103func (c CRAMMD5) MarshalBinary() ([]byte, error) {

104 if c.Ipad == nil || c.Opad == nil {

105 return nil, nil

106 }

107

108 ipad, err := c.Ipad.(encoding.BinaryMarshaler).MarshalBinary()

109 if err != nil {

110 return nil, fmt.Errorf("marshal ipad: %v", err)

111 }

112 opad, err := c.Opad.(encoding.BinaryMarshaler).MarshalBinary()

113 if err != nil {

114 return nil, fmt.Errorf("marshal opad: %v", err)

115 }

116 buf := make([]byte, 2+len(ipad)+len(opad))

117 ipadlen := uint16(len(ipad))

118 buf[0] = byte(ipadlen >> 8)

119 buf[1] = byte(ipadlen >> 0)

120 copy(buf[2:], ipad)

121 copy(buf[2+len(ipad):], opad)

122 return buf, nil

123}

124

125// BinaryUnmarshal is used by bstore to restore the ipad/opad hash states.

126func (c *CRAMMD5) UnmarshalBinary(buf []byte) error {

127 if len(buf) == 0 {

128 *c = CRAMMD5{}

129 return nil

130 }

131 if len(buf) < 2 {

132 return fmt.Errorf("short buffer")

133 }

134 ipadlen := int(uint16(buf[0])<<8 | uint16(buf[1])<<0)

135 if len(buf) < 2+ipadlen {

136 return fmt.Errorf("buffer too short for ipadlen")

137 }

138 ipad := md5.New()

139 opad := md5.New()

140 if err := ipad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2 : 2+ipadlen]); err != nil {

141 return fmt.Errorf("unmarshal ipad: %v", err)

142 }

143 if err := opad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2+ipadlen:]); err != nil {

144 return fmt.Errorf("unmarshal opad: %v", err)

145 }

146 *c = CRAMMD5{ipad, opad}

147 return nil

148}

149

150// Password holds credentials in various forms, for logging in with SMTP/IMAP.

151type Password struct {

152 Hash string // bcrypt hash for IMAP LOGIN, SASL PLAIN and HTTP basic authentication.

153 CRAMMD5 CRAMMD5 // For SASL CRAM-MD5.

154 SCRAMSHA1 SCRAM // For SASL SCRAM-SHA-1.

155 SCRAMSHA256 SCRAM // For SASL SCRAM-SHA-256.

156}

157

158// Subjectpass holds the secret key used to sign subjectpass tokens.

159type Subjectpass struct {

160 Email string // Our destination address (canonical, with catchall localpart stripped).

161 Key string

162}

163

164// NextUIDValidity is a singleton record in the database with the next UIDValidity

165// to use for the next mailbox.

166type NextUIDValidity struct {

167 ID int // Just a single record with ID 1.

168 Next uint32

169}

170

171// SyncState track ModSeqs.

172type SyncState struct {

173 ID int // Just a single record with ID 1.

174

175 // Last used, next assigned will be one higher. The first value we hand out is 2.

176 // That's because 0 (the default value for old existing messages, from before the

177 // Message.ModSeq field) is special in IMAP, so we return it as 1.

178 LastModSeq ModSeq `bstore:"nonzero"`

179

180 // Highest ModSeq of expunged record that we deleted. When a clients synchronizes

181 // and requests changes based on a modseq before this one, we don't have the

182 // history to provide information about deletions. We normally keep these expunged

183 // records around, but we may periodically truly delete them to reclaim storage

184 // space. Initially set to -1 because we don't want to match with any ModSeq in the

185 // database, which can be zero values.

186 HighestDeletedModSeq ModSeq

187}

188

189// Mailbox is collection of messages, e.g. Inbox or Sent.

190type Mailbox struct {

191 ID int64

192

193 // "Inbox" is the name for the special IMAP "INBOX". Slash separated

194 // for hierarchy.

195 Name string `bstore:"nonzero,unique"`

196

197 // If UIDs are invalidated, e.g. when renaming a mailbox to a previously existing

198 // name, UIDValidity must be changed. Used by IMAP for synchronization.

199 UIDValidity uint32

200

201 // UID likely to be assigned to next message. Used by IMAP to detect messages

202 // delivered to a mailbox.

203 UIDNext UID

204

205 SpecialUse

206

207 // Keywords as used in messages. Storing a non-system keyword for a message

208 // automatically adds it to this list. Used in the IMAP FLAGS response. Only

209 // "atoms" are allowed (IMAP syntax), keywords are case-insensitive, only stored in

210 // lower case (for JMAP), sorted.

211 Keywords []string

212

213 HaveCounts bool // Whether MailboxCounts have been initialized.

214 MailboxCounts // Statistics about messages, kept up to date whenever a change happens.

215}

216

217// MailboxCounts tracks statistics about messages for a mailbox.

218type MailboxCounts struct {

219 Total int64 // Total number of messages, excluding \Deleted. For JMAP.

220 Deleted int64 // Number of messages with \Deleted flag. Used for IMAP message count that includes messages with \Deleted.

221 Unread int64 // Messages without \Seen, excluding those with \Deleted, for JMAP.

222 Unseen int64 // Messages without \Seen, including those with \Deleted, for IMAP.

223 Size int64 // Number of bytes for all messages.

224}

225

226func (mc MailboxCounts) String() string {

227 return fmt.Sprintf("%d total, %d deleted, %d unread, %d unseen, size %d bytes", mc.Total, mc.Deleted, mc.Unread, mc.Unseen, mc.Size)

228}

229

230// Add increases mailbox counts mc with those of delta.

231func (mc *MailboxCounts) Add(delta MailboxCounts) {

232 mc.Total += delta.Total

233 mc.Deleted += delta.Deleted

234 mc.Unread += delta.Unread

235 mc.Unseen += delta.Unseen

236 mc.Size += delta.Size

237}

238

239// Add decreases mailbox counts mc with those of delta.

240func (mc *MailboxCounts) Sub(delta MailboxCounts) {

241 mc.Total -= delta.Total

242 mc.Deleted -= delta.Deleted

243 mc.Unread -= delta.Unread

244 mc.Unseen -= delta.Unseen

245 mc.Size -= delta.Size

246}

247

248// SpecialUse identifies a specific role for a mailbox, used by clients to

249// understand where messages should go.

250type SpecialUse struct {

251 Archive bool

252 Draft bool

253 Junk bool

254 Sent bool

255 Trash bool

256}

257

258// CalculateCounts calculates the full current counts for messages in the mailbox.

259func (mb *Mailbox) CalculateCounts(tx *bstore.Tx) (mc MailboxCounts, err error) {

260 q := bstore.QueryTx[Message](tx)

261 q.FilterNonzero(Message{MailboxID: mb.ID})

262 q.FilterEqual("Expunged", false)

263 err = q.ForEach(func(m Message) error {

264 mc.Add(m.MailboxCounts())

265 return nil

266 })

267 return

268}

269

270// ChangeSpecialUse returns a change for special-use flags, for broadcasting to

271// other connections.

272func (mb Mailbox) ChangeSpecialUse() ChangeMailboxSpecialUse {

273 return ChangeMailboxSpecialUse{mb.ID, mb.Name, mb.SpecialUse}

274}

275

276// ChangeKeywords returns a change with new keywords for a mailbox (e.g. after

277// setting a new keyword on a message in the mailbox), for broadcasting to other

278// connections.

279func (mb Mailbox) ChangeKeywords() ChangeMailboxKeywords {

280 return ChangeMailboxKeywords{mb.ID, mb.Name, mb.Keywords}

281}

282

283// KeywordsChanged returns whether the keywords in a mailbox have changed.

284func (mb Mailbox) KeywordsChanged(origmb Mailbox) bool {

285 if len(mb.Keywords) != len(origmb.Keywords) {

286 return true

287 }

288 // Keywords are stored sorted.

289 for i, kw := range mb.Keywords {

290 if origmb.Keywords[i] != kw {

291 return true

292 }

293 }

294 return false

295}

296

297// CountsChange returns a change with mailbox counts.

298func (mb Mailbox) ChangeCounts() ChangeMailboxCounts {

299 return ChangeMailboxCounts{mb.ID, mb.Name, mb.MailboxCounts}

300}

301

302// Subscriptions are separate from existence of mailboxes.

303type Subscription struct {

304 Name string

305}

306

307// Flags for a mail message.

308type Flags struct {

309 Seen bool

310 Answered bool

311 Flagged bool

312 Forwarded bool

313 Junk bool

314 Notjunk bool

315 Deleted bool

316 Draft bool

317 Phishing bool

318 MDNSent bool

319}

320

321// FlagsAll is all flags set, for use as mask.

322var FlagsAll = Flags{true, true, true, true, true, true, true, true, true, true}

323

324// Validation of "message From" domain.

325type Validation uint8

326

327const (

328 ValidationUnknown Validation = 0

329 ValidationStrict Validation = 1 // Like DMARC, with strict policies.

330 ValidationDMARC Validation = 2 // Actual DMARC policy.

331 ValidationRelaxed Validation = 3 // Like DMARC, with relaxed policies.

332 ValidationPass Validation = 4 // For SPF.

333 ValidationNeutral Validation = 5 // For SPF.

334 ValidationTemperror Validation = 6

335 ValidationPermerror Validation = 7

336 ValidationFail Validation = 8

337 ValidationSoftfail Validation = 9 // For SPF.

338 ValidationNone Validation = 10 // E.g. No records.

339)

340

341// Message stored in database and per-message file on disk.

342//

343// Contents are always the combined data from MsgPrefix and the on-disk file named

344// based on ID.

345//

346// Messages always have a header section, even if empty. Incoming messages without

347// header section must get an empty header section added before inserting.

348type Message struct {

349 // ID, unchanged over lifetime, determines path to on-disk msg file.

350 // Set during deliver.

351 ID int64

352

353 UID UID `bstore:"nonzero"` // UID, for IMAP. Set during deliver.

354 MailboxID int64 `bstore:"nonzero,unique MailboxID+UID,index MailboxID+Received,index MailboxID+ModSeq,ref Mailbox"`

355

356 // Modification sequence, for faster syncing with IMAP QRESYNC and JMAP.

357 // ModSeq is the last modification. CreateSeq is the Seq the message was inserted,

358 // always <= ModSeq. If Expunged is set, the message has been removed and should not

359 // be returned to the user. In this case, ModSeq is the Seq where the message is

360 // removed, and will never be changed again.

361 // We have an index on both ModSeq (for JMAP that synchronizes per account) and

362 // MailboxID+ModSeq (for IMAP that synchronizes per mailbox).

363 // The index on CreateSeq helps efficiently finding created messages for JMAP.

364 // The value of ModSeq is special for IMAP. Messages that existed before ModSeq was

365 // added have 0 as value. But modseq 0 in IMAP is special, so we return it as 1. If

366 // we get modseq 1 from a client, the IMAP server will translate it to 0. When we

367 // return modseq to clients, we turn 0 into 1.

368 ModSeq ModSeq `bstore:"index"`

369 CreateSeq ModSeq `bstore:"index"`

370 Expunged bool

371

372 // If set, this message was delivered to a Rejects mailbox. When it is moved to a

373 // different mailbox, its MailboxOrigID is set to the destination mailbox and this

374 // flag cleared.

375 IsReject bool

376

377 // If set, this is a forwarded message (through a ruleset with IsForward). This

378 // causes fields used during junk analysis to be moved to their Orig variants, and

379 // masked IP fields cleared, so they aren't used in junk classifications for

380 // incoming messages. This ensures the forwarded messages don't cause negative

381 // reputation for the forwarding mail server, which may also be sending regular

382 // messages.

383 IsForward bool

384

385 // MailboxOrigID is the mailbox the message was originally delivered to. Typically

386 // Inbox or Rejects, but can also be a mailbox configured in a Ruleset, or

387 // Postmaster, TLS/DMARC reporting addresses. MailboxOrigID is not changed when the

388 // message is moved to another mailbox, e.g. Archive/Trash/Junk. Used for

389 // per-mailbox reputation.

390 //

391 // MailboxDestinedID is normally 0, but when a message is delivered to the Rejects

392 // mailbox, it is set to the intended mailbox according to delivery rules,

393 // typically that of Inbox. When such a message is moved out of Rejects, the

394 // MailboxOrigID is corrected by setting it to MailboxDestinedID. This ensures the

395 // message is used for reputation calculation for future deliveries to that

396 // mailbox.

397 //

398 // These are not bstore references to prevent having to update all messages in a

399 // mailbox when the original mailbox is removed. Use of these fields requires

400 // checking if the mailbox still exists.

401 MailboxOrigID int64

402 MailboxDestinedID int64

403

404 Received time.Time `bstore:"default now,index"`

405

406 // Full IP address of remote SMTP server. Empty if not delivered over SMTP. The

407 // masked IPs are used to classify incoming messages. They are left empty for

408 // messages matching a ruleset for forwarded messages.

409 RemoteIP string

410 RemoteIPMasked1 string `bstore:"index RemoteIPMasked1+Received"` // For IPv4 /32, for IPv6 /64, for reputation.

411 RemoteIPMasked2 string `bstore:"index RemoteIPMasked2+Received"` // For IPv4 /26, for IPv6 /48.

412 RemoteIPMasked3 string `bstore:"index RemoteIPMasked3+Received"` // For IPv4 /21, for IPv6 /32.

413

414 // Only set if present and not an IP address. Unicode string. Empty for forwarded

415 // messages.

416 EHLODomain string `bstore:"index EHLODomain+Received"`

417 MailFrom string // With localpart and domain. Can be empty.

418 MailFromLocalpart smtp.Localpart // SMTP "MAIL FROM", can be empty.

419 // Only set if it is a domain, not an IP. Unicode string. Empty for forwarded

420 // messages, but see OrigMailFromDomain.

421 MailFromDomain string `bstore:"index MailFromDomain+Received"`

422 RcptToLocalpart smtp.Localpart // SMTP "RCPT TO", can be empty.

423 RcptToDomain string // Unicode string.

424

425 // Parsed "From" message header, used for reputation along with domain validation.

426 MsgFromLocalpart smtp.Localpart

427 MsgFromDomain string `bstore:"index MsgFromDomain+Received"` // Unicode string.

428 MsgFromOrgDomain string `bstore:"index MsgFromOrgDomain+Received"` // Unicode string.

429

430 // Simplified statements of the Validation fields below, used for incoming messages

431 // to check reputation.

432 EHLOValidated bool

433 MailFromValidated bool

434 MsgFromValidated bool

435

436 EHLOValidation Validation // Validation can also take reverse IP lookup into account, not only SPF.

437 MailFromValidation Validation // Can have SPF-specific validations like ValidationSoftfail.

438 MsgFromValidation Validation // Desirable validations: Strict, DMARC, Relaxed. Will not be just Pass.

439

440 // Domains with verified DKIM signatures. Unicode string. For forwarded messages, a

441 // DKIM domain that matched a ruleset's verified domain is left out, but included

442 // in OrigDKIMDomains.

443 DKIMDomains []string `bstore:"index DKIMDomains+Received"`

444

445 // For forwarded messages,

446 OrigEHLODomain string

447 OrigDKIMDomains []string

448

449 // Canonicalized Message-Id, always lower-case and normalized quoting, without

450 // <>'s. Empty if missing. Used for matching message threads, and to prevent

451 // duplicate reject delivery.

452 MessageID string `bstore:"index"`

453 // lower-case: ../rfc/5256:495

454

455 // For matching threads in case there is no References/In-Reply-To header. It is

456 // lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed.

457 SubjectBase string `bstore:"index"`

458 // ../rfc/5256:90

459

460 // Hash of message. For rejects delivery in case there is no Message-ID, only set

461 // when delivered as reject.

462 MessageHash []byte

463

464 // ID of message starting this thread.

465 ThreadID int64 `bstore:"index"`

466 // IDs of parent messages, from closest parent to the root message. Parent messages

467 // may be in a different mailbox, or may no longer exist. ThreadParentIDs must

468 // never contain the message id itself (a cycle), and parent messages must

469 // reference the same ancestors.

470 ThreadParentIDs []int64

471 // ThreadMissingLink is true if there is no match with a direct parent. E.g. first

472 // ID in ThreadParentIDs is not the direct ancestor (an intermediate message may

473 // have been deleted), or subject-based matching was done.

474 ThreadMissingLink bool

475 // If set, newly delivered child messages are automatically marked as read. This

476 // field is copied to new child messages. Changes are propagated to the webmail

477 // client.

478 ThreadMuted bool

479 // If set, this (sub)thread is collapsed in the webmail client, for threading mode

480 // "on" (mode "unread" ignores it). This field is copied to new child message.

481 // Changes are propagated to the webmail client.

482 ThreadCollapsed bool

483

484 // If received message was known to match a mailing list rule (with modified junk

485 // filtering).

486 IsMailingList bool

487

488 // If this message is a DSN, generated by us or received. For DSNs, we don't look

489 // at the subject when matching threads.

490 DSN bool

491

492 ReceivedTLSVersion uint16 // 0 if unknown, 1 if plaintext/no TLS, otherwise TLS cipher suite.

493 ReceivedTLSCipherSuite uint16

494 ReceivedRequireTLS bool // Whether RequireTLS was known to be used for incoming delivery.

495

496 Flags

497 // For keywords other than system flags or the basic well-known $-flags. Only in

498 // "atom" syntax (IMAP), they are case-insensitive, always stored in lower-case

499 // (for JMAP), sorted.

500 Keywords []string `bstore:"index"`

501 Size int64

502 TrainedJunk *bool // If nil, no training done yet. Otherwise, true is trained as junk, false trained as nonjunk.

503 MsgPrefix []byte // Typically holds received headers and/or header separator.

504

505 // ParsedBuf message structure. Currently saved as JSON of message.Part because bstore

506 // cannot yet store recursive types. Created when first needed, and saved in the

507 // database.

508 // todo: once replaced with non-json storage, remove date fixup in ../message/part.go.

509 ParsedBuf []byte

510}

511

512// MailboxCounts returns the delta to counts this message means for its

513// mailbox.

514func (m Message) MailboxCounts() (mc MailboxCounts) {

515 if m.Expunged {

516 return

517 }

518 if m.Deleted {

519 mc.Deleted++

520 } else {

521 mc.Total++

522 }

523 if !m.Seen {

524 mc.Unseen++

525 if !m.Deleted {

526 mc.Unread++

527 }

528 }

529 mc.Size += m.Size

530 return

531}

532

533func (m Message) ChangeAddUID() ChangeAddUID {

534 return ChangeAddUID{m.MailboxID, m.UID, m.ModSeq, m.Flags, m.Keywords}

535}

536

537func (m Message) ChangeFlags(orig Flags) ChangeFlags {

538 mask := m.Flags.Changed(orig)

539 return ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, ModSeq: m.ModSeq, Mask: mask, Flags: m.Flags, Keywords: m.Keywords}

540}

541

542func (m Message) ChangeThread() ChangeThread {

543 return ChangeThread{[]int64{m.ID}, m.ThreadMuted, m.ThreadCollapsed}

544}

545

546// ModSeq represents a modseq as stored in the database. ModSeq 0 in the

547// database is sent to the client as 1, because modseq 0 is special in IMAP.

548// ModSeq coming from the client are of type int64.

549type ModSeq int64

550

551func (ms ModSeq) Client() int64 {

552 if ms == 0 {

553 return 1

554 }

555 return int64(ms)

556}

557

558// ModSeqFromClient converts a modseq from a client to a modseq for internal

559// use, e.g. in a database query.

560// ModSeq 1 is turned into 0 (the Go zero value for ModSeq).

561func ModSeqFromClient(modseq int64) ModSeq {

562 if modseq == 1 {

563 return 0

564 }

565 return ModSeq(modseq)

566}

567

568// PrepareExpunge clears fields that are no longer needed after an expunge, so

569// almost all fields. Does not change ModSeq, but does set Expunged.

570func (m *Message) PrepareExpunge() {

571 *m = Message{

572 ID: m.ID,

573 UID: m.UID,

574 MailboxID: m.MailboxID,

575 CreateSeq: m.CreateSeq,

576 ModSeq: m.ModSeq,

577 Expunged: true,

578 ThreadID: m.ThreadID,

579 }

580}

581

582// PrepareThreading sets MessageID, SubjectBase and DSN (used in threading) based

583// on the part.

584func (m *Message) PrepareThreading(log mlog.Log, part *message.Part) {

585 m.DSN = part.IsDSN()

586

587 if part.Envelope == nil {

588 return

589 }

590 messageID, raw, err := message.MessageIDCanonical(part.Envelope.MessageID)

591 if err != nil {

592 log.Debugx("parsing message-id, ignoring", err, slog.String("messageid", part.Envelope.MessageID))

593 } else if raw {

594 log.Debug("could not parse message-id as address, continuing with raw value", slog.String("messageid", part.Envelope.MessageID))

595 }

596 m.MessageID = messageID

597 m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false)

598}

599

600// LoadPart returns a message.Part by reading from m.ParsedBuf.

601func (m Message) LoadPart(r io.ReaderAt) (message.Part, error) {

602 if m.ParsedBuf == nil {

603 return message.Part{}, fmt.Errorf("message not parsed")

604 }

605 var p message.Part

606 err := json.Unmarshal(m.ParsedBuf, &p)

607 if err != nil {

608 return p, fmt.Errorf("unmarshal message part")

609 }

610 p.SetReaderAt(r)

611 return p, nil

612}

613

614// NeedsTraining returns whether message needs a training update, based on

615// TrainedJunk (current training status) and new Junk/Notjunk flags.

616func (m Message) NeedsTraining() bool {

617 untrain := m.TrainedJunk != nil

618 untrainJunk := untrain && *m.TrainedJunk

619 train := m.Junk || m.Notjunk && !(m.Junk && m.Notjunk)

620 trainJunk := m.Junk

621 return untrain != train || untrain && train && untrainJunk != trainJunk

622}

623

624// JunkFlagsForMailbox sets Junk and Notjunk flags based on mailbox name if configured. Often

625// used when delivering/moving/copying messages to a mailbox. Mail clients are not

626// very helpful with setting junk/notjunk flags. But clients can move/copy messages

627// to other mailboxes. So we set flags when clients move a message.

628func (m *Message) JunkFlagsForMailbox(mb Mailbox, conf config.Account) {

629 if mb.Junk {

630 m.Junk = true

631 m.Notjunk = false

632 return

633 }

634

635 if !conf.AutomaticJunkFlags.Enabled {

636 return

637 }

638

639 lmailbox := strings.ToLower(mb.Name)

640

641 if conf.JunkMailbox != nil && conf.JunkMailbox.MatchString(lmailbox) {

642 m.Junk = true

643 m.Notjunk = false

644 } else if conf.NeutralMailbox != nil && conf.NeutralMailbox.MatchString(lmailbox) {

645 m.Junk = false

646 m.Notjunk = false

647 } else if conf.NotJunkMailbox != nil && conf.NotJunkMailbox.MatchString(lmailbox) {

648 m.Junk = false

649 m.Notjunk = true

650 } else if conf.JunkMailbox == nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox != nil {

651 m.Junk = true

652 m.Notjunk = false

653 } else if conf.JunkMailbox != nil && conf.NeutralMailbox == nil && conf.NotJunkMailbox != nil {

654 m.Junk = false

655 m.Notjunk = false

656 } else if conf.JunkMailbox != nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox == nil {

657 m.Junk = false

658 m.Notjunk = true

659 }

660}

661

662// Recipient represents the recipient of a message. It is tracked to allow

663// first-time incoming replies from users this account has sent messages to. When a

664// mailbox is added to the Sent mailbox the message is parsed and recipients are

665// inserted as recipient. Recipients are never removed other than for removing the

666// message. On move/copy of a message, recipients aren't modified either. For IMAP,

667// this assumes a client simply appends messages to the Sent mailbox (as opposed to

668// copying messages from some place).

669type Recipient struct {

670 ID int64

671 MessageID int64 `bstore:"nonzero,ref Message"` // Ref gives it its own index, useful for fast removal as well.

672 Localpart string `bstore:"nonzero"` // Encoded localpart.

673 Domain string `bstore:"nonzero,index Domain+Localpart"` // Unicode string.

674 OrgDomain string `bstore:"nonzero,index"` // Unicode string.

675 Sent time.Time `bstore:"nonzero"`

676}

677

678// Outgoing is a message submitted for delivery from the queue. Used to enforce

679// maximum outgoing messages.

680type Outgoing struct {

681 ID int64

682 Recipient string `bstore:"nonzero,index"` // Canonical international address with utf8 domain.

683 Submitted time.Time `bstore:"nonzero,default now"`

684}

685

686// RecipientDomainTLS stores TLS capabilities of a recipient domain as encountered

687// during most recent connection (delivery attempt).

688type RecipientDomainTLS struct {

689 Domain string // Unicode.

690 Updated time.Time `bstore:"default now"`

691 STARTTLS bool // Supports STARTTLS.

692 RequireTLS bool // Supports RequireTLS SMTP extension.

693}

694

695// DiskUsage tracks quota use.

696type DiskUsage struct {

697 ID int64 // Always one record with ID 1.

698 MessageSize int64 // Sum of all messages, for quota accounting.

699}

700

701// SessionToken and CSRFToken are types to prevent mixing them up.

702// Base64 raw url encoded.

703type SessionToken string

704type CSRFToken string

705

706// LoginSession represents a login session. We keep a limited number of sessions

707// for a user, removing the oldest session when a new one is created.

708type LoginSession struct {

709 ID int64

710 Created time.Time `bstore:"nonzero,default now"` // Of original login.

711 Expires time.Time `bstore:"nonzero"` // Extended each time it is used.

712 SessionTokenBinary [16]byte `bstore:"nonzero"` // Stored in cookie, like "webmailsession" or "webaccountsession".

713 CSRFTokenBinary [16]byte // For API requests, in "x-mox-csrf" header.

714 AccountName string `bstore:"nonzero"`

715 LoginAddress string `bstore:"nonzero"`

716

717 // Set when loading from database.

718 sessionToken SessionToken

719 csrfToken CSRFToken

720}

721

722// Quoting is a setting for how to quote in replies/forwards.

723type Quoting string

724

725const (

726 Default Quoting = "" // Bottom-quote if text is selected, top-quote otherwise.

727 Bottom Quoting = "bottom"

728 Top Quoting = "top"

729)

730

731// Settings are webmail client settings.

732type Settings struct {

733 ID uint8 // Singleton ID 1.

734

735 Signature string

736 Quoting Quoting

737

738 // Whether to show the bars underneath the address input fields indicating

739 // starttls/dnssec/dane/mtasts/requiretls support by address.

740 ShowAddressSecurity bool

741}

742

743// ViewMode how a message should be viewed: its text parts, html parts, or html

744// with loading external resources.

745type ViewMode string

746

747const (

748 ModeDefault ViewMode = ""

749 ModeText ViewMode = "text"

750 ModeHTML ViewMode = "html"

751 ModeHTMLExt ViewMode = "htmlext" // HTML with external resources.

752)

753

754// FromAddressSettings are webmail client settings per "From" address.

755type FromAddressSettings struct {

756 FromAddress string // Unicode.

757 ViewMode ViewMode

758}

759

760// RulesetNoListID records a user "no" response to the question of

761// creating/removing a ruleset after moving a message with list-id header from/to

762// the inbox.

763type RulesetNoListID struct {

764 ID int64

765 RcptToAddress string `bstore:"nonzero"`

766 ListID string `bstore:"nonzero"`

767 ToInbox bool // Otherwise from Inbox to other mailbox.

768}

769

770// RulesetNoMsgFrom records a user "no" response to the question of

771// creating/moveing a ruleset after moving a mesage with message "from" address

772// from/to the inbox.

773type RulesetNoMsgFrom struct {

774 ID int64

775 RcptToAddress string `bstore:"nonzero"`

776 MsgFromAddress string `bstore:"nonzero"` // Unicode.

777 ToInbox bool // Otherwise from Inbox to other mailbox.

778}

779

780// RulesetNoMailbox represents a "never from/to this mailbox" response to the

781// question of adding/removing a ruleset after moving a message.

782type RulesetNoMailbox struct {

783 ID int64

784

785 // The mailbox from/to which the move has happened.

786 // Not a references, if mailbox is deleted, an entry becomes ineffective.

787 MailboxID int64 `bstore:"nonzero"`

788 ToMailbox bool // Whether MailboxID is the destination of the move (instead of source).

789}

790

791// Types stored in DB.

792var DBTypes = []any{

793 NextUIDValidity{},

794 Message{},

795 Recipient{},

796 Mailbox{},

797 Subscription{},

798 Outgoing{},

799 Password{},

800 Subjectpass{},

801 SyncState{},

802 Upgrade{},

803 RecipientDomainTLS{},

804 DiskUsage{},

805 LoginSession{},

806 Settings{},

807 FromAddressSettings{},

808 RulesetNoListID{},

809 RulesetNoMsgFrom{},

810 RulesetNoMailbox{},

811}

812

813// Account holds the information about a user, includings mailboxes, messages, imap subscriptions.

814type Account struct {

815 Name string // Name, according to configuration.

816 Dir string // Directory where account files, including the database, bloom filter, and mail messages, are stored for this account.

817 DBPath string // Path to database with mailboxes, messages, etc.

818 DB *bstore.DB // Open database connection.

819

820 // Channel that is closed if/when account has/gets "threads" accounting (see

821 // Upgrade.Threads).

822 threadsCompleted chan struct{}

823 // If threads upgrade completed with error, this is set. Used for warning during

824 // delivery, or aborting when importing.

825 threadsErr error

826

827 // Write lock must be held for account/mailbox modifications including message delivery.

828 // Read lock for reading mailboxes/messages.

829 // When making changes to mailboxes/messages, changes must be broadcasted before

830 // releasing the lock to ensure proper UID ordering.

831 sync.RWMutex

832

833 nused int // Reference count, while >0, this account is alive and shared.

834}

835

836type Upgrade struct {

837 ID byte

838 Threads byte // 0: None, 1: Adding MessageID's completed, 2: Adding ThreadID's completed.

839}

840

841// InitialUIDValidity returns a UIDValidity used for initializing an account.

842// It can be replaced during tests with a predictable value.

843var InitialUIDValidity = func() uint32 {

844 return uint32(time.Now().Unix() >> 1) // A 2-second resolution will get us far enough beyond 2038.

845}

846

847var openAccounts = struct {

848 names map[string]*Account

849 sync.Mutex

850}{

851 names: map[string]*Account{},

852}

853

854func closeAccount(acc *Account) (rerr error) {

855 openAccounts.Lock()

856 acc.nused--

857 defer openAccounts.Unlock()

858 if acc.nused == 0 {

859 // threadsCompleted must be closed now because it increased nused.

860 rerr = acc.DB.Close()

861 acc.DB = nil

862 delete(openAccounts.names, acc.Name)

863 }

864 return

865}

866

867// OpenAccount opens an account by name.

868//

869// No additional data path prefix or ".db" suffix should be added to the name.

870// A single shared account exists per name.

871func OpenAccount(log mlog.Log, name string) (*Account, error) {

872 openAccounts.Lock()

873 defer openAccounts.Unlock()

874 if acc, ok := openAccounts.names[name]; ok {

875 acc.nused++

876 return acc, nil

877 }

878

879 if _, ok := mox.Conf.Account(name); !ok {

880 return nil, ErrAccountUnknown

881 }

882

883 acc, err := openAccount(log, name)

884 if err != nil {

885 return nil, err

886 }

887 openAccounts.names[name] = acc

888 return acc, nil

889}

890

891// openAccount opens an existing account, or creates it if it is missing.

892func openAccount(log mlog.Log, name string) (a *Account, rerr error) {

893 dir := filepath.Join(mox.DataDirPath("accounts"), name)

894 return OpenAccountDB(log, dir, name)

895}

896

897// OpenAccountDB opens an account database file and returns an initialized account

898// or error. Only exported for use by subcommands that verify the database file.

899// Almost all account opens must go through OpenAccount/OpenEmail/OpenEmailAuth.

900func OpenAccountDB(log mlog.Log, accountDir, accountName string) (a *Account, rerr error) {

901 dbpath := filepath.Join(accountDir, "index.db")

902

903 // Create account if it doesn't exist yet.

904 isNew := false

905 if _, err := os.Stat(dbpath); err != nil && os.IsNotExist(err) {

906 isNew = true

907 os.MkdirAll(accountDir, 0770)

908 }

909

910 db, err := bstore.Open(context.TODO(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)

911 if err != nil {

912 return nil, err

913 }

914

915 defer func() {

916 if rerr != nil {

917 db.Close()

918 if isNew {

919 os.Remove(dbpath)

920 }

921 }

922 }()

923

924 acc := &Account{

925 Name: accountName,

926 Dir: accountDir,

927 DBPath: dbpath,

928 DB: db,

929 nused: 1,

930 threadsCompleted: make(chan struct{}),

931 }

932

933 if isNew {

934 if err := initAccount(db); err != nil {

935 return nil, fmt.Errorf("initializing account: %v", err)

936 }

937 close(acc.threadsCompleted)

938 return acc, nil

939 }

940

941 // Ensure singletons are present. Mailbox counts and total message size, Settings.

942 var mentioned bool

943 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

944 if tx.Get(&Settings{ID: 1}) == bstore.ErrAbsent {

945 if err := tx.Insert(&Settings{ID: 1, ShowAddressSecurity: true}); err != nil {

946 return err

947 }

948 }

949

950 err := bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error {

951 if !mentioned {

952 mentioned = true

953 log.Info("first calculation of mailbox counts for account", slog.String("account", accountName))

954 }

955 mc, err := mb.CalculateCounts(tx)

956 if err != nil {

957 return err

958 }

959 mb.HaveCounts = true

960 mb.MailboxCounts = mc

961 return tx.Update(&mb)

962 })

963 if err != nil {

964 return err

965 }

966

967 du := DiskUsage{ID: 1}

968 err = tx.Get(&du)

969 if err == nil || !errors.Is(err, bstore.ErrAbsent) {

970 return err

971 }

972 // No DiskUsage record yet, calculate total size and insert.

973 err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {

974 du.MessageSize += mb.Size

975 return nil

976 })

977 if err != nil {

978 return err

979 }

980 return tx.Insert(&du)

981 })

982 if err != nil {

983 return nil, fmt.Errorf("calculating counts for mailbox or inserting settings: %v", err)

984 }

985

986 // Start adding threading if needed.

987 up := Upgrade{ID: 1}

988 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

989 err := tx.Get(&up)

990 if err == bstore.ErrAbsent {

991 if err := tx.Insert(&up); err != nil {

992 return fmt.Errorf("inserting initial upgrade record: %v", err)

993 }

994 err = nil

995 }

996 return err

997 })

998 if err != nil {

999 return nil, fmt.Errorf("checking message threading: %v", err)

1000 }

1001 if up.Threads == 2 {

1002 close(acc.threadsCompleted)

1003 return acc, nil

1004 }

1005

1006 // Increase account use before holding on to account in background.

1007 // Caller holds the lock. The goroutine below decreases nused by calling

1008 // closeAccount.

1009 acc.nused++

1010

1011 // Ensure all messages have a MessageID and SubjectBase, which are needed when

1012 // matching threads.

1013 // Then assign messages to threads, in the same way we do during imports.

1014 log.Info("upgrading account for threading, in background", slog.String("account", acc.Name))

1015 go func() {

1016 defer func() {

1017 err := closeAccount(acc)

1018 log.Check(err, "closing use of account after upgrading account storage for threads", slog.String("account", a.Name))

1019

1020 // Mark that upgrade has finished, possibly error is indicated in threadsErr.

1021 close(acc.threadsCompleted)

1022 }()

1023

1024 defer func() {

1025 x := recover() // Should not happen, but don't take program down if it does.

1026 if x != nil {

1027 log.Error("upgradeThreads panic", slog.Any("err", x))

1028 debug.PrintStack()

1029 metrics.PanicInc(metrics.Upgradethreads)

1030 acc.threadsErr = fmt.Errorf("panic during upgradeThreads: %v", x)

1031 }

1032 }()

1033

1034 err := upgradeThreads(mox.Shutdown, log, acc, &up)

1035 if err != nil {

1036 a.threadsErr = err

1037 log.Errorx("upgrading account for threading, aborted", err, slog.String("account", a.Name))

1038 } else {

1039 log.Info("upgrading account for threading, completed", slog.String("account", a.Name))

1040 }

1041 }()

1042 return acc, nil

1043}

1044

1045// ThreadingWait blocks until the one-time account threading upgrade for the

1046// account has completed, and returns an error if not successful.

1047//

1048// To be used before starting an import of messages.

1049func (a *Account) ThreadingWait(log mlog.Log) error {

1050 select {

1051 case <-a.threadsCompleted:

1052 return a.threadsErr

1053 default:

1054 }

1055 log.Debug("waiting for account upgrade to complete")

1056

1057 <-a.threadsCompleted

1058 return a.threadsErr

1059}

1060

1061func initAccount(db *bstore.DB) error {

1062 return db.Write(context.TODO(), func(tx *bstore.Tx) error {

1063 uidvalidity := InitialUIDValidity()

1064

1065 if err := tx.Insert(&Upgrade{ID: 1, Threads: 2}); err != nil {

1066 return err

1067 }

1068 if err := tx.Insert(&DiskUsage{ID: 1}); err != nil {

1069 return err

1070 }

1071 if err := tx.Insert(&Settings{ID: 1}); err != nil {

1072 return err

1073 }

1074

1075 if len(mox.Conf.Static.DefaultMailboxes) > 0 {

1076 // Deprecated in favor of InitialMailboxes.

1077 defaultMailboxes := mox.Conf.Static.DefaultMailboxes

1078 mailboxes := []string{"Inbox"}

1079 for _, name := range defaultMailboxes {

1080 if strings.EqualFold(name, "Inbox") {

1081 continue

1082 }

1083 mailboxes = append(mailboxes, name)

1084 }

1085 for _, name := range mailboxes {

1086 mb := Mailbox{Name: name, UIDValidity: uidvalidity, UIDNext: 1, HaveCounts: true}

1087 if strings.HasPrefix(name, "Archive") {

1088 mb.Archive = true

1089 } else if strings.HasPrefix(name, "Drafts") {

1090 mb.Draft = true

1091 } else if strings.HasPrefix(name, "Junk") {

1092 mb.Junk = true

1093 } else if strings.HasPrefix(name, "Sent") {

1094 mb.Sent = true

1095 } else if strings.HasPrefix(name, "Trash") {

1096 mb.Trash = true

1097 }

1098 if err := tx.Insert(&mb); err != nil {

1099 return fmt.Errorf("creating mailbox: %w", err)

1100 }

1101 if err := tx.Insert(&Subscription{name}); err != nil {

1102 return fmt.Errorf("adding subscription: %w", err)

1103 }

1104 }

1105 } else {

1106 mailboxes := mox.Conf.Static.InitialMailboxes

1107 var zerouse config.SpecialUseMailboxes

1108 if mailboxes.SpecialUse == zerouse && len(mailboxes.Regular) == 0 {

1109 mailboxes = DefaultInitialMailboxes

1110 }

1111

1112 add := func(name string, use SpecialUse) error {

1113 mb := Mailbox{Name: name, UIDValidity: uidvalidity, UIDNext: 1, SpecialUse: use, HaveCounts: true}

1114 if err := tx.Insert(&mb); err != nil {

1115 return fmt.Errorf("creating mailbox: %w", err)

1116 }

1117 if err := tx.Insert(&Subscription{name}); err != nil {

1118 return fmt.Errorf("adding subscription: %w", err)

1119 }

1120 return nil

1121 }

1122 addSpecialOpt := func(nameOpt string, use SpecialUse) error {

1123 if nameOpt == "" {

1124 return nil

1125 }

1126 return add(nameOpt, use)

1127 }

1128 l := []struct {

1129 nameOpt string

1130 use SpecialUse

1131 }{

1132 {"Inbox", SpecialUse{}},

1133 {mailboxes.SpecialUse.Archive, SpecialUse{Archive: true}},

1134 {mailboxes.SpecialUse.Draft, SpecialUse{Draft: true}},

1135 {mailboxes.SpecialUse.Junk, SpecialUse{Junk: true}},

1136 {mailboxes.SpecialUse.Sent, SpecialUse{Sent: true}},

1137 {mailboxes.SpecialUse.Trash, SpecialUse{Trash: true}},

1138 }

1139 for _, e := range l {

1140 if err := addSpecialOpt(e.nameOpt, e.use); err != nil {

1141 return err

1142 }

1143 }

1144 for _, name := range mailboxes.Regular {

1145 if err := add(name, SpecialUse{}); err != nil {

1146 return err

1147 }

1148 }

1149 }

1150

1151 uidvalidity++

1152 if err := tx.Insert(&NextUIDValidity{1, uidvalidity}); err != nil {

1153 return fmt.Errorf("inserting nextuidvalidity: %w", err)

1154 }

1155 return nil

1156 })

1157}

1158

1159// CheckClosed asserts that the account has a zero reference count. For use in tests.

1160func (a *Account) CheckClosed() {

1161 openAccounts.Lock()

1162 defer openAccounts.Unlock()

1163 if a.nused != 0 {

1164 panic(fmt.Sprintf("account still in use, %d refs", a.nused))

1165 }

1166}

1167

1168// Close reduces the reference count, and closes the database connection when

1169// it was the last user.

1170func (a *Account) Close() error {

1171 if CheckConsistencyOnClose {

1172 xerr := a.CheckConsistency()

1173 err := closeAccount(a)

1174 if xerr != nil {

1175 panic(xerr)

1176 }

1177 return err

1178 }

1179 return closeAccount(a)

1180}

1181

1182// CheckConsistency checks the consistency of the database and returns a non-nil

1183// error for these cases:

1184//

1185// - Missing on-disk file for message.

1186// - Mismatch between message size and length of MsgPrefix and on-disk file.

1187// - Missing HaveCounts.

1188// - Incorrect mailbox counts.

1189// - Incorrect total message size.

1190// - Message with UID >= mailbox uid next.

1191// - Mailbox uidvalidity >= account uid validity.

1192// - ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq.

1193// - All messages have a nonzero ThreadID, and no cycles in ThreadParentID, and parent messages the same ThreadParentIDs tail.

1194func (a *Account) CheckConsistency() error {

1195 var uidErrors []string // With a limit, could be many.

1196 var modseqErrors []string // With limit.

1197 var fileErrors []string // With limit.

1198 var threadidErrors []string // With limit.

1199 var threadParentErrors []string // With limit.

1200 var threadAncestorErrors []string // With limit.

1201 var errors []string

1202

1203 err := a.DB.Read(context.Background(), func(tx *bstore.Tx) error {

1204 nuv := NextUIDValidity{ID: 1}

1205 err := tx.Get(&nuv)

1206 if err != nil {

1207 return fmt.Errorf("fetching next uid validity: %v", err)

1208 }

1209

1210 mailboxes := map[int64]Mailbox{}

1211 err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {

1212 mailboxes[mb.ID] = mb

1213

1214 if mb.UIDValidity >= nuv.Next {

1215 errmsg := fmt.Sprintf("mailbox %q (id %d) has uidvalidity %d >= account next uidvalidity %d", mb.Name, mb.ID, mb.UIDValidity, nuv.Next)

1216 errors = append(errors, errmsg)

1217 }

1218 return nil

1219 })

1220 if err != nil {

1221 return fmt.Errorf("listing mailboxes: %v", err)

1222 }

1223

1224 counts := map[int64]MailboxCounts{}

1225 err = bstore.QueryTx[Message](tx).ForEach(func(m Message) error {

1226 mc := counts[m.MailboxID]

1227 mc.Add(m.MailboxCounts())

1228 counts[m.MailboxID] = mc

1229

1230 mb := mailboxes[m.MailboxID]

1231

1232 if (m.ModSeq == 0 || m.CreateSeq == 0 || m.CreateSeq > m.ModSeq) && len(modseqErrors) < 20 {

1233 modseqerr := fmt.Sprintf("message %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and createseq <= modseq", m.ID, mb.Name, mb.ID, m.ModSeq, m.CreateSeq)

1234 modseqErrors = append(modseqErrors, modseqerr)

1235 }

1236 if m.UID >= mb.UIDNext && len(uidErrors) < 20 {

1237 uiderr := fmt.Sprintf("message %d in mailbox %q (id %d) has uid %d >= mailbox uidnext %d", m.ID, mb.Name, mb.ID, m.UID, mb.UIDNext)

1238 uidErrors = append(uidErrors, uiderr)

1239 }

1240 if m.Expunged {

1241 return nil

1242 }

1243 p := a.MessagePath(m.ID)

1244 st, err := os.Stat(p)

1245 if err != nil {

1246 existserr := fmt.Sprintf("message %d in mailbox %q (id %d) on-disk file %s: %v", m.ID, mb.Name, mb.ID, p, err)

1247 fileErrors = append(fileErrors, existserr)

1248 } else if len(fileErrors) < 20 && m.Size != int64(len(m.MsgPrefix))+st.Size() {

1249 sizeerr := fmt.Sprintf("message %d in mailbox %q (id %d) has size %d != len msgprefix %d + on-disk file size %d = %d", m.ID, mb.Name, mb.ID, m.Size, len(m.MsgPrefix), st.Size(), int64(len(m.MsgPrefix))+st.Size())

1250 fileErrors = append(fileErrors, sizeerr)

1251 }

1252

1253 if m.ThreadID <= 0 && len(threadidErrors) < 20 {

1254 err := fmt.Sprintf("message %d in mailbox %q (id %d) has threadid 0", m.ID, mb.Name, mb.ID)

1255 threadidErrors = append(threadidErrors, err)

1256 }

1257 if slices.Contains(m.ThreadParentIDs, m.ID) && len(threadParentErrors) < 20 {

1258 err := fmt.Sprintf("message %d in mailbox %q (id %d) references itself in threadparentids", m.ID, mb.Name, mb.ID)

1259 threadParentErrors = append(threadParentErrors, err)

1260 }

1261 for i, pid := range m.ThreadParentIDs {

1262 am := Message{ID: pid}

1263 if err := tx.Get(&am); err == bstore.ErrAbsent {

1264 continue

1265 } else if err != nil {

1266 return fmt.Errorf("get ancestor message: %v", err)

1267 } else if !slices.Equal(m.ThreadParentIDs[i+1:], am.ThreadParentIDs) && len(threadAncestorErrors) < 20 {

1268 err := fmt.Sprintf("message %d, thread %d has ancestor ids %v, and ancestor at index %d with id %d should have the same tail but has %v\n", m.ID, m.ThreadID, m.ThreadParentIDs, i, am.ID, am.ThreadParentIDs)

1269 threadAncestorErrors = append(threadAncestorErrors, err)

1270 } else {

1271 break

1272 }

1273 }

1274 return nil

1275 })

1276 if err != nil {

1277 return fmt.Errorf("reading messages: %v", err)

1278 }

1279

1280 var totalSize int64

1281 for _, mb := range mailboxes {

1282 totalSize += mb.Size

1283 if !mb.HaveCounts {

1284 errmsg := fmt.Sprintf("mailbox %q (id %d) does not have counts, should be %#v", mb.Name, mb.ID, counts[mb.ID])

1285 errors = append(errors, errmsg)

1286 } else if mb.MailboxCounts != counts[mb.ID] {

1287 mbcounterr := fmt.Sprintf("mailbox %q (id %d) has wrong counts %s, should be %s", mb.Name, mb.ID, mb.MailboxCounts, counts[mb.ID])

1288 errors = append(errors, mbcounterr)

1289 }

1290 }

1291

1292 du := DiskUsage{ID: 1}

1293 if err := tx.Get(&du); err != nil {

1294 return fmt.Errorf("get diskusage")

1295 }

1296 if du.MessageSize != totalSize {

1297 errmsg := fmt.Sprintf("total message size in database is %d, sum of mailbox message sizes is %d", du.MessageSize, totalSize)

1298 errors = append(errors, errmsg)

1299 }

1300

1301 return nil

1302 })

1303 if err != nil {

1304 return err

1305 }

1306 errors = append(errors, uidErrors...)

1307 errors = append(errors, modseqErrors...)

1308 errors = append(errors, fileErrors...)

1309 errors = append(errors, threadidErrors...)

1310 errors = append(errors, threadParentErrors...)

1311 errors = append(errors, threadAncestorErrors...)

1312 if len(errors) > 0 {

1313 return fmt.Errorf("%s", strings.Join(errors, "; "))

1314 }

1315 return nil

1316}

1317

1318// Conf returns the configuration for this account if it still exists. During

1319// an SMTP session, a configuration update may drop an account.

1320func (a *Account) Conf() (config.Account, bool) {

1321 return mox.Conf.Account(a.Name)

1322}

1323

1324// NextUIDValidity returns the next new/unique uidvalidity to use for this account.

1325func (a *Account) NextUIDValidity(tx *bstore.Tx) (uint32, error) {

1326 nuv := NextUIDValidity{ID: 1}

1327 if err := tx.Get(&nuv); err != nil {

1328 return 0, err

1329 }

1330 v := nuv.Next

1331 nuv.Next++

1332 if err := tx.Update(&nuv); err != nil {

1333 return 0, err

1334 }

1335 return v, nil

1336}

1337

1338// NextModSeq returns the next modification sequence, which is global per account,

1339// over all types.

1340func (a *Account) NextModSeq(tx *bstore.Tx) (ModSeq, error) {

1341 v := SyncState{ID: 1}

1342 if err := tx.Get(&v); err == bstore.ErrAbsent {

1343 // We start assigning from modseq 2. Modseq 0 is not usable, so returned as 1, so

1344 // already used.

1345 // HighestDeletedModSeq is -1 so comparison against the default ModSeq zero value

1346 // makes sense.

1347 v = SyncState{1, 2, -1}

1348 return v.LastModSeq, tx.Insert(&v)

1349 } else if err != nil {

1350 return 0, err

1351 }

1352 v.LastModSeq++

1353 return v.LastModSeq, tx.Update(&v)

1354}

1355

1356func (a *Account) HighestDeletedModSeq(tx *bstore.Tx) (ModSeq, error) {

1357 v := SyncState{ID: 1}

1358 err := tx.Get(&v)

1359 if err == bstore.ErrAbsent {

1360 return 0, nil

1361 }

1362 return v.HighestDeletedModSeq, err

1363}

1364

1365// WithWLock runs fn with account writelock held. Necessary for account/mailbox

1366// modification. For message delivery, a read lock is required.

1367func (a *Account) WithWLock(fn func()) {

1368 a.Lock()

1369 defer a.Unlock()

1370 fn()

1371}

1372

1373// WithRLock runs fn with account read lock held. Needed for message delivery.

1374func (a *Account) WithRLock(fn func()) {

1375 a.RLock()

1376 defer a.RUnlock()

1377 fn()

1378}

1379

1380// DeliverMessage delivers a mail message to the account.

1381//

1382// The message, with msg.MsgPrefix and msgFile combined, must have a header

1383// section. The caller is responsible for adding a header separator to

1384// msg.MsgPrefix if missing from an incoming message.

1385//

1386// If the destination mailbox has the Sent special-use flag, the message is parsed

1387// for its recipients (to/cc/bcc). Their domains are added to Recipients for use in

1388// dmarc reputation.

1389//

1390// If sync is true, the message file and its directory are synced. Should be true

1391// for regular mail delivery, but can be false when importing many messages.

1392//

1393// If updateDiskUsage is true, the account total message size (for quota) is

1394// updated. Callers must check if a message can be added within quota before

1395// calling DeliverMessage.

1396//

1397// If CreateSeq/ModSeq is not set, it is assigned automatically.

1398//

1399// Must be called with account rlock or wlock.

1400//

1401// Caller must broadcast new message.

1402//

1403// Caller must update mailbox counts.

1404func (a *Account) DeliverMessage(log mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, sync, notrain, nothreads, updateDiskUsage bool) error {

1405 if m.Expunged {

1406 return fmt.Errorf("cannot deliver expunged message")

1407 }

1408

1409 mb := Mailbox{ID: m.MailboxID}

1410 if err := tx.Get(&mb); err != nil {

1411 return fmt.Errorf("get mailbox: %w", err)

1412 }

1413 m.UID = mb.UIDNext

1414 mb.UIDNext++

1415 if err := tx.Update(&mb); err != nil {

1416 return fmt.Errorf("updating mailbox nextuid: %w", err)

1417 }

1418

1419 if updateDiskUsage {

1420 du := DiskUsage{ID: 1}

1421 if err := tx.Get(&du); err != nil {

1422 return fmt.Errorf("get disk usage: %v", err)

1423 }

1424 du.MessageSize += m.Size

1425 if err := tx.Update(&du); err != nil {

1426 return fmt.Errorf("update disk usage: %v", err)

1427 }

1428 }

1429

1430 conf, _ := a.Conf()

1431 m.JunkFlagsForMailbox(mb, conf)

1432

1433 mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile.

1434 var part *message.Part

1435 if m.ParsedBuf == nil {

1436 p, err := message.EnsurePart(log.Logger, false, mr, m.Size)

1437 if err != nil {

1438 log.Infox("parsing delivered message", err, slog.String("parse", ""), slog.Int64("message", m.ID))

1439 // We continue, p is still valid.

1440 }

1441 part = &p

1442 buf, err := json.Marshal(part)

1443 if err != nil {

1444 return fmt.Errorf("marshal parsed message: %w", err)

1445 }

1446 m.ParsedBuf = buf

1447 } else {

1448 var p message.Part

1449 if err := json.Unmarshal(m.ParsedBuf, &p); err != nil {

1450 log.Errorx("unmarshal parsed message, continuing", err, slog.String("parse", ""))

1451 } else {

1452 part = &p

1453 }

1454 }

1455

1456 // If we are delivering to the originally intended mailbox, no need to store the mailbox ID again.

1457 if m.MailboxDestinedID != 0 && m.MailboxDestinedID == m.MailboxOrigID {

1458 m.MailboxDestinedID = 0

1459 }

1460 if m.CreateSeq == 0 || m.ModSeq == 0 {

1461 modseq, err := a.NextModSeq(tx)

1462 if err != nil {

1463 return fmt.Errorf("assigning next modseq: %w", err)

1464 }

1465 m.CreateSeq = modseq

1466 m.ModSeq = modseq

1467 }

1468

1469 if part != nil && m.MessageID == "" && m.SubjectBase == "" {

1470 m.PrepareThreading(log, part)

1471 }

1472

1473 // Assign to thread (if upgrade has completed).

1474 noThreadID := nothreads

1475 if m.ThreadID == 0 && !nothreads && part != nil {

1476 select {

1477 case <-a.threadsCompleted:

1478 if a.threadsErr != nil {

1479 log.Info("not assigning threads for new delivery, upgrading to threads failed")

1480 noThreadID = true

1481 } else {

1482 if err := assignThread(log, tx, m, part); err != nil {

1483 return fmt.Errorf("assigning thread: %w", err)

1484 }

1485 }

1486 default:

1487 // note: since we have a write transaction to get here, we can't wait for the

1488 // thread upgrade to finish.

1489 // If we don't assign a threadid the upgrade process will do it.

1490 log.Info("not assigning threads for new delivery, upgrading to threads in progress which will assign this message")

1491 noThreadID = true

1492 }

1493 }

1494

1495 if err := tx.Insert(m); err != nil {

1496 return fmt.Errorf("inserting message: %w", err)

1497 }

1498 if !noThreadID && m.ThreadID == 0 {

1499 m.ThreadID = m.ID

1500 if err := tx.Update(m); err != nil {

1501 return fmt.Errorf("updating message for its own thread id: %w", err)

1502 }

1503 }

1504

1505 // todo: perhaps we should match the recipients based on smtp submission and a matching message-id? we now miss the addresses in bcc's if the mail client doesn't save a message that includes the bcc header in the sent mailbox.

1506 if mb.Sent && part != nil && part.Envelope != nil {

1507 e := part.Envelope

1508 sent := e.Date

1509 if sent.IsZero() {

1510 sent = m.Received

1511 }

1512 if sent.IsZero() {

1513 sent = time.Now()

1514 }

1515 addrs := append(append(e.To, e.CC...), e.BCC...)

1516 for _, addr := range addrs {

1517 if addr.User == "" {

1518 // Would trigger error because Recipient.Localpart must be nonzero. todo: we could allow empty localpart in db, and filter by not using FilterNonzero.

1519 log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", slog.Any("address", addr))

1520 continue

1521 }

1522 d, err := dns.ParseDomain(addr.Host)

1523 if err != nil {

1524 log.Debugx("parsing domain in to/cc/bcc address", err, slog.Any("address", addr))

1525 continue

1526 }

1527 lp, err := smtp.ParseLocalpart(addr.User)

1528 if err != nil {

1529 log.Debugx("parsing localpart in to/cc/bcc address", err, slog.Any("address", addr))

1530 continue

1531 }

1532 mr := Recipient{

1533 MessageID: m.ID,

1534 Localpart: lp.String(),

1535 Domain: d.Name(),

1536 OrgDomain: publicsuffix.Lookup(context.TODO(), log.Logger, d).Name(),

1537 Sent: sent,

1538 }

1539 if err := tx.Insert(&mr); err != nil {

1540 return fmt.Errorf("inserting sent message recipients: %w", err)

1541 }

1542 }

1543 }

1544

1545 msgPath := a.MessagePath(m.ID)

1546 msgDir := filepath.Dir(msgPath)

1547 os.MkdirAll(msgDir, 0770)

1548

1549 // Sync file data to disk.

1550 if sync {

1551 if err := msgFile.Sync(); err != nil {

1552 return fmt.Errorf("fsync message file: %w", err)

1553 }

1554 }

1555

1556 if err := moxio.LinkOrCopy(log, msgPath, msgFile.Name(), &moxio.AtReader{R: msgFile}, true); err != nil {

1557 return fmt.Errorf("linking/copying message to new file: %w", err)

1558 }

1559

1560 if sync {

1561 if err := moxio.SyncDir(log, msgDir); err != nil {

1562 xerr := os.Remove(msgPath)

1563 log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath))

1564 return fmt.Errorf("sync directory: %w", err)

1565 }

1566 }

1567

1568 if !notrain && m.NeedsTraining() {

1569 l := []Message{*m}

1570 if err := a.RetrainMessages(context.TODO(), log, tx, l, false); err != nil {

1571 xerr := os.Remove(msgPath)

1572 log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath))

1573 return fmt.Errorf("training junkfilter: %w", err)

1574 }

1575 *m = l[0]

1576 }

1577

1578 return nil

1579}

1580

1581// SetPassword saves a new password for this account. This password is used for

1582// IMAP, SMTP (submission) sessions and the HTTP account web page.

1583func (a *Account) SetPassword(log mlog.Log, password string) error {

1584 password, err := precis.OpaqueString.String(password)

1585 if err != nil {

1586 return fmt.Errorf(`password not allowed by "precis"`)

1587 }

1588

1589 if len(password) < 8 {

1590 // We actually check for bytes...

1591 return fmt.Errorf("password must be at least 8 characters long")

1592 }

1593

1594 hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)

1595 if err != nil {

1596 return fmt.Errorf("generating password hash: %w", err)

1597 }

1598

1599 err = a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1600 if _, err := bstore.QueryTx[Password](tx).Delete(); err != nil {

1601 return fmt.Errorf("deleting existing password: %v", err)

1602 }

1603 var pw Password

1604 pw.Hash = string(hash)

1605

1606 // CRAM-MD5 calculates an HMAC-MD5, with the password as key, over a per-attempt

1607 // unique text that includes a timestamp. HMAC performs two hashes. Both times, the

1608 // first block is based on the key/password. We hash those first blocks now, and

1609 // store the hash state in the database. When we actually authenticate, we'll

1610 // complete the HMAC by hashing only the text. We cannot store crypto/hmac's hash,

1611 // because it does not expose its internal state and isn't a BinaryMarshaler.

1612 // ../rfc/2104:121

1613 pw.CRAMMD5.Ipad = md5.New()

1614 pw.CRAMMD5.Opad = md5.New()

1615 key := []byte(password)

1616 if len(key) > 64 {

1617 t := md5.Sum(key)

1618 key = t[:]

1619 }

1620 ipad := make([]byte, md5.BlockSize)

1621 opad := make([]byte, md5.BlockSize)

1622 copy(ipad, key)

1623 copy(opad, key)

1624 for i := range ipad {

1625 ipad[i] ^= 0x36

1626 opad[i] ^= 0x5c

1627 }

1628 pw.CRAMMD5.Ipad.Write(ipad)

1629 pw.CRAMMD5.Opad.Write(opad)

1630

1631 pw.SCRAMSHA1.Salt = scram.MakeRandom()

1632 pw.SCRAMSHA1.Iterations = 2 * 4096

1633 pw.SCRAMSHA1.SaltedPassword = scram.SaltPassword(sha1.New, password, pw.SCRAMSHA1.Salt, pw.SCRAMSHA1.Iterations)

1634

1635 pw.SCRAMSHA256.Salt = scram.MakeRandom()

1636 pw.SCRAMSHA256.Iterations = 4096

1637 pw.SCRAMSHA256.SaltedPassword = scram.SaltPassword(sha256.New, password, pw.SCRAMSHA256.Salt, pw.SCRAMSHA256.Iterations)

1638

1639 if err := tx.Insert(&pw); err != nil {

1640 return fmt.Errorf("inserting new password: %v", err)

1641 }

1642

1643 return sessionRemoveAll(context.TODO(), log, tx, a.Name)

1644 })

1645 if err == nil {

1646 log.Info("new password set for account", slog.String("account", a.Name))

1647 }

1648 return err

1649}

1650

1651// Subjectpass returns the signing key for use with subjectpass for the given

1652// email address with canonical localpart.

1653func (a *Account) Subjectpass(email string) (key string, err error) {

1654 return key, a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1655 v := Subjectpass{Email: email}

1656 err := tx.Get(&v)

1657 if err == nil {

1658 key = v.Key

1659 return nil

1660 }

1661 if !errors.Is(err, bstore.ErrAbsent) {

1662 return fmt.Errorf("get subjectpass key from accounts database: %w", err)

1663 }

1664 key = ""

1665 const chars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"

1666 buf := make([]byte, 16)

1667 if _, err := cryptorand.Read(buf); err != nil {

1668 return err

1669 }

1670 for _, b := range buf {

1671 key += string(chars[int(b)%len(chars)])

1672 }

1673 v.Key = key

1674 return tx.Insert(&v)

1675 })

1676}

1677

1678// Ensure mailbox is present in database, adding records for the mailbox and its

1679// parents if they aren't present.

1680//

1681// If subscribe is true, any mailboxes that were created will also be subscribed to.

1682// Caller must hold account wlock.

1683// Caller must propagate changes if any.

1684func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool) (mb Mailbox, changes []Change, rerr error) {

1685 if norm.NFC.String(name) != name {

1686 return Mailbox{}, nil, fmt.Errorf("mailbox name not normalized")

1687 }

1688

1689 // Quick sanity check.

1690 if strings.EqualFold(name, "inbox") && name != "Inbox" {

1691 return Mailbox{}, nil, fmt.Errorf("bad casing for inbox")

1692 }

1693

1694 elems := strings.Split(name, "/")

1695 q := bstore.QueryTx[Mailbox](tx)

1696 q.FilterFn(func(mb Mailbox) bool {

1697 return mb.Name == elems[0] || strings.HasPrefix(mb.Name, elems[0]+"/")

1698 })

1699 l, err := q.List()

1700 if err != nil {

1701 return Mailbox{}, nil, fmt.Errorf("list mailboxes: %v", err)

1702 }

1703

1704 mailboxes := map[string]Mailbox{}

1705 for _, xmb := range l {

1706 mailboxes[xmb.Name] = xmb

1707 }

1708

1709 p := ""

1710 for _, elem := range elems {

1711 if p != "" {

1712 p += "/"

1713 }

1714 p += elem

1715 var ok bool

1716 mb, ok = mailboxes[p]

1717 if ok {

1718 continue

1719 }

1720 uidval, err := a.NextUIDValidity(tx)

1721 if err != nil {

1722 return Mailbox{}, nil, fmt.Errorf("next uid validity: %v", err)

1723 }

1724 mb = Mailbox{

1725 Name: p,

1726 UIDValidity: uidval,

1727 UIDNext: 1,

1728 HaveCounts: true,

1729 }

1730 err = tx.Insert(&mb)

1731 if err != nil {

1732 return Mailbox{}, nil, fmt.Errorf("creating new mailbox: %v", err)

1733 }

1734

1735 var flags []string

1736 if subscribe {

1737 if tx.Get(&Subscription{p}) != nil {

1738 err := tx.Insert(&Subscription{p})

1739 if err != nil {

1740 return Mailbox{}, nil, fmt.Errorf("subscribing to mailbox: %v", err)

1741 }

1742 }

1743 flags = []string{`\Subscribed`}

1744 }

1745 changes = append(changes, ChangeAddMailbox{mb, flags})

1746 }

1747 return mb, changes, nil

1748}

1749

1750// MailboxExists checks if mailbox exists.

1751// Caller must hold account rlock.

1752func (a *Account) MailboxExists(tx *bstore.Tx, name string) (bool, error) {

1753 q := bstore.QueryTx[Mailbox](tx)

1754 q.FilterEqual("Name", name)

1755 return q.Exists()

1756}

1757

1758// MailboxFind finds a mailbox by name, returning a nil mailbox and nil error if mailbox does not exist.

1759func (a *Account) MailboxFind(tx *bstore.Tx, name string) (*Mailbox, error) {

1760 q := bstore.QueryTx[Mailbox](tx)

1761 q.FilterEqual("Name", name)

1762 mb, err := q.Get()

1763 if err == bstore.ErrAbsent {

1764 return nil, nil

1765 }

1766 if err != nil {

1767 return nil, fmt.Errorf("looking up mailbox: %w", err)

1768 }

1769 return &mb, nil

1770}

1771

1772// SubscriptionEnsure ensures a subscription for name exists. The mailbox does not

1773// have to exist. Any parents are not automatically subscribed.

1774// Changes are returned and must be broadcasted by the caller.

1775func (a *Account) SubscriptionEnsure(tx *bstore.Tx, name string) ([]Change, error) {

1776 if err := tx.Get(&Subscription{name}); err == nil {

1777 return nil, nil

1778 }

1779

1780 if err := tx.Insert(&Subscription{name}); err != nil {

1781 return nil, fmt.Errorf("inserting subscription: %w", err)

1782 }

1783

1784 q := bstore.QueryTx[Mailbox](tx)

1785 q.FilterEqual("Name", name)

1786 _, err := q.Get()

1787 if err == nil {

1788 return []Change{ChangeAddSubscription{name, nil}}, nil

1789 } else if err != bstore.ErrAbsent {

1790 return nil, fmt.Errorf("looking up mailbox for subscription: %w", err)

1791 }

1792 return []Change{ChangeAddSubscription{name, []string{`\NonExistent`}}}, nil

1793}

1794

1795// MessageRuleset returns the first ruleset (if any) that matches the message

1796// represented by msgPrefix and msgFile, with smtp and validation fields from m.

1797func MessageRuleset(log mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset {

1798 if len(dest.Rulesets) == 0 {

1799 return nil

1800 }

1801

1802 mr := FileMsgReader(msgPrefix, msgFile) // We don't close, it would close the msgFile.

1803 p, err := message.Parse(log.Logger, false, mr)

1804 if err != nil {

1805 log.Errorx("parsing message for evaluating rulesets, continuing with headers", err, slog.String("parse", ""))

1806 // note: part is still set.

1807 }

1808 // todo optimize: only parse header if needed for rulesets. and probably reuse an earlier parsing.

1809 header, err := p.Header()

1810 if err != nil {

1811 log.Errorx("parsing message headers for evaluating rulesets, delivering to default mailbox", err, slog.String("parse", ""))

1812 // todo: reject message?

1813 return nil

1814 }

1815

1816ruleset:

1817 for _, rs := range dest.Rulesets {

1818 if rs.SMTPMailFromRegexpCompiled != nil {

1819 if !rs.SMTPMailFromRegexpCompiled.MatchString(m.MailFrom) {

1820 continue ruleset

1821 }

1822 }

1823 if rs.MsgFromRegexpCompiled != nil {

1824 if m.MsgFromLocalpart == "" && m.MsgFromDomain == "" || !rs.MsgFromRegexpCompiled.MatchString(m.MsgFromLocalpart.String()+"@"+m.MsgFromDomain) {

1825 continue ruleset

1826 }

1827 }

1828

1829 if !rs.VerifiedDNSDomain.IsZero() {

1830 d := rs.VerifiedDNSDomain.Name()

1831 suffix := "." + d

1832 matchDomain := func(s string) bool {

1833 return s == d || strings.HasSuffix(s, suffix)

1834 }

1835 var ok bool

1836 if m.EHLOValidated && matchDomain(m.EHLODomain) {

1837 ok = true

1838 }

1839 if m.MailFromValidated && matchDomain(m.MailFromDomain) {

1840 ok = true

1841 }

1842 for _, d := range m.DKIMDomains {

1843 if matchDomain(d) {

1844 ok = true

1845 break

1846 }

1847 }

1848 if !ok {

1849 continue ruleset

1850 }

1851 }

1852

1853 header:

1854 for _, t := range rs.HeadersRegexpCompiled {

1855 for k, vl := range header {

1856 k = strings.ToLower(k)

1857 if !t[0].MatchString(k) {

1858 continue

1859 }

1860 for _, v := range vl {

1861 v = strings.ToLower(strings.TrimSpace(v))

1862 if t[1].MatchString(v) {

1863 continue header

1864 }

1865 }

1866 }

1867 continue ruleset

1868 }

1869 return &rs

1870 }

1871 return nil

1872}

1873

1874// MessagePath returns the file system path of a message.

1875func (a *Account) MessagePath(messageID int64) string {

1876 return strings.Join(append([]string{a.Dir, "msg"}, messagePathElems(messageID)...), string(filepath.Separator))

1877}

1878

1879// MessageReader opens a message for reading, transparently combining the

1880// message prefix with the original incoming message.

1881func (a *Account) MessageReader(m Message) *MsgReader {

1882 return &MsgReader{prefix: m.MsgPrefix, path: a.MessagePath(m.ID), size: m.Size}

1883}

1884

1885// DeliverDestination delivers an email to dest, based on the configured rulesets.

1886//

1887// Returns ErrOverQuota when account would be over quota after adding message.

1888//

1889// Caller must hold account wlock (mailbox may be created).

1890// Message delivery, possible mailbox creation, and updated mailbox counts are

1891// broadcasted.

1892func (a *Account) DeliverDestination(log mlog.Log, dest config.Destination, m *Message, msgFile *os.File) error {

1893 var mailbox string

1894 rs := MessageRuleset(log, dest, m, m.MsgPrefix, msgFile)

1895 if rs != nil {

1896 mailbox = rs.Mailbox

1897 } else if dest.Mailbox == "" {

1898 mailbox = "Inbox"

1899 } else {

1900 mailbox = dest.Mailbox

1901 }

1902 return a.DeliverMailbox(log, mailbox, m, msgFile)

1903}

1904

1905// DeliverMailbox delivers an email to the specified mailbox.

1906//

1907// Returns ErrOverQuota when account would be over quota after adding message.

1908//

1909// Caller must hold account wlock (mailbox may be created).

1910// Message delivery, possible mailbox creation, and updated mailbox counts are

1911// broadcasted.

1912func (a *Account) DeliverMailbox(log mlog.Log, mailbox string, m *Message, msgFile *os.File) error {

1913 var changes []Change

1914 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1915 if ok, _, err := a.CanAddMessageSize(tx, m.Size); err != nil {

1916 return err

1917 } else if !ok {

1918 return ErrOverQuota

1919 }

1920

1921 mb, chl, err := a.MailboxEnsure(tx, mailbox, true)

1922 if err != nil {

1923 return fmt.Errorf("ensuring mailbox: %w", err)

1924 }

1925 m.MailboxID = mb.ID

1926 m.MailboxOrigID = mb.ID

1927

1928 // Update count early, DeliverMessage will update mb too and we don't want to fetch

1929 // it again before updating.

1930 mb.MailboxCounts.Add(m.MailboxCounts())

1931 if err := tx.Update(&mb); err != nil {

1932 return fmt.Errorf("updating mailbox for delivery: %w", err)

1933 }

1934

1935 if err := a.DeliverMessage(log, tx, m, msgFile, true, false, false, true); err != nil {

1936 return err

1937 }

1938

1939 changes = append(changes, chl...)

1940 changes = append(changes, m.ChangeAddUID(), mb.ChangeCounts())

1941 return nil

1942 })

1943 // todo: if rename succeeded but transaction failed, we should remove the file.

1944 if err != nil {

1945 return err

1946 }

1947

1948 BroadcastChanges(a, changes)

1949 return nil

1950}

1951

1952// TidyRejectsMailbox removes old reject emails, and returns whether there is space for a new delivery.

1953//

1954// Caller most hold account wlock.

1955// Changes are broadcasted.

1956func (a *Account) TidyRejectsMailbox(log mlog.Log, rejectsMailbox string) (hasSpace bool, rerr error) {

1957 var changes []Change

1958

1959 var remove []Message

1960 defer func() {

1961 for _, m := range remove {

1962 p := a.MessagePath(m.ID)

1963 err := os.Remove(p)

1964 log.Check(err, "removing rejects message file", slog.String("path", p))

1965 }

1966 }()

1967

1968 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1969 mb, err := a.MailboxFind(tx, rejectsMailbox)

1970 if err != nil {

1971 return fmt.Errorf("finding mailbox: %w", err)

1972 }

1973 if mb == nil {

1974 // No messages have been delivered yet.

1975 hasSpace = true

1976 return nil

1977 }

1978

1979 // Gather old messages to remove.

1980 old := time.Now().Add(-14 * 24 * time.Hour)

1981 qdel := bstore.QueryTx[Message](tx)

1982 qdel.FilterNonzero(Message{MailboxID: mb.ID})

1983 qdel.FilterEqual("Expunged", false)

1984 qdel.FilterLess("Received", old)

1985 remove, err = qdel.List()

1986 if err != nil {

1987 return fmt.Errorf("listing old messages: %w", err)

1988 }

1989

1990 changes, err = a.rejectsRemoveMessages(context.TODO(), log, tx, mb, remove)

1991 if err != nil {

1992 return fmt.Errorf("removing messages: %w", err)

1993 }

1994

1995 // We allow up to n messages.

1996 qcount := bstore.QueryTx[Message](tx)

1997 qcount.FilterNonzero(Message{MailboxID: mb.ID})

1998 qcount.FilterEqual("Expunged", false)

1999 qcount.Limit(1000)

2000 n, err := qcount.Count()

2001 if err != nil {

2002 return fmt.Errorf("counting rejects: %w", err)

2003 }

2004 hasSpace = n < 1000

2005

2006 return nil

2007 })

2008 if err != nil {

2009 remove = nil // Don't remove files on failure.

2010 return false, err

2011 }

2012

2013 BroadcastChanges(a, changes)

2014

2015 return hasSpace, nil

2016}

2017

2018func (a *Account) rejectsRemoveMessages(ctx context.Context, log mlog.Log, tx *bstore.Tx, mb *Mailbox, l []Message) ([]Change, error) {

2019 if len(l) == 0 {

2020 return nil, nil

2021 }

2022 ids := make([]int64, len(l))

2023 anyids := make([]any, len(l))

2024 for i, m := range l {

2025 ids[i] = m.ID

2026 anyids[i] = m.ID

2027 }

2028

2029 // Remove any message recipients. Should not happen, but a user can move messages

2030 // from a Sent mailbox to the rejects mailbox...

2031 qdmr := bstore.QueryTx[Recipient](tx)

2032 qdmr.FilterEqual("MessageID", anyids...)

2033 if _, err := qdmr.Delete(); err != nil {

2034 return nil, fmt.Errorf("deleting from message recipient: %w", err)

2035 }

2036

2037 // Assign new modseq.

2038 modseq, err := a.NextModSeq(tx)

2039 if err != nil {

2040 return nil, fmt.Errorf("assign next modseq: %w", err)

2041 }

2042

2043 // Expunge the messages.

2044 qx := bstore.QueryTx[Message](tx)

2045 qx.FilterIDs(ids)

2046 var expunged []Message

2047 qx.Gather(&expunged)

2048 if _, err := qx.UpdateNonzero(Message{ModSeq: modseq, Expunged: true}); err != nil {

2049 return nil, fmt.Errorf("expunging messages: %w", err)

2050 }

2051

2052 var totalSize int64

2053 for _, m := range expunged {

2054 m.Expunged = false // Was set by update, but would cause wrong count.

2055 mb.MailboxCounts.Sub(m.MailboxCounts())

2056 totalSize += m.Size

2057 }

2058 if err := tx.Update(mb); err != nil {

2059 return nil, fmt.Errorf("updating mailbox counts: %w", err)

2060 }

2061 if err := a.AddMessageSize(log, tx, -totalSize); err != nil {

2062 return nil, fmt.Errorf("updating disk usage: %w", err)

2063 }

2064

2065 // Mark as neutral and train so junk filter gets untrained with these (junk) messages.

2066 for i := range expunged {

2067 expunged[i].Junk = false

2068 expunged[i].Notjunk = false

2069 }

2070 if err := a.RetrainMessages(ctx, log, tx, expunged, true); err != nil {

2071 return nil, fmt.Errorf("retraining expunged messages: %w", err)

2072 }

2073

2074 changes := make([]Change, len(l), len(l)+1)

2075 for i, m := range l {

2076 changes[i] = ChangeRemoveUIDs{mb.ID, []UID{m.UID}, modseq}

2077 }

2078 changes = append(changes, mb.ChangeCounts())

2079 return changes, nil

2080}

2081

2082// RejectsRemove removes a message from the rejects mailbox if present.

2083// Caller most hold account wlock.

2084// Changes are broadcasted.

2085func (a *Account) RejectsRemove(log mlog.Log, rejectsMailbox, messageID string) error {

2086 var changes []Change

2087

2088 var remove []Message

2089 defer func() {

2090 for _, m := range remove {

2091 p := a.MessagePath(m.ID)

2092 err := os.Remove(p)

2093 log.Check(err, "removing rejects message file", slog.String("path", p))

2094 }

2095 }()

2096

2097 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

2098 mb, err := a.MailboxFind(tx, rejectsMailbox)

2099 if err != nil {

2100 return fmt.Errorf("finding mailbox: %w", err)

2101 }

2102 if mb == nil {

2103 return nil

2104 }

2105

2106 q := bstore.QueryTx[Message](tx)

2107 q.FilterNonzero(Message{MailboxID: mb.ID, MessageID: messageID})

2108 q.FilterEqual("Expunged", false)

2109 remove, err = q.List()

2110 if err != nil {

2111 return fmt.Errorf("listing messages to remove: %w", err)

2112 }

2113

2114 changes, err = a.rejectsRemoveMessages(context.TODO(), log, tx, mb, remove)

2115 if err != nil {

2116 return fmt.Errorf("removing messages: %w", err)

2117 }

2118

2119 return nil

2120 })

2121 if err != nil {

2122 remove = nil // Don't remove files on failure.

2123 return err

2124 }

2125

2126 BroadcastChanges(a, changes)

2127

2128 return nil

2129}

2130

2131// AddMessageSize adjusts the DiskUsage.MessageSize by size.

2132func (a *Account) AddMessageSize(log mlog.Log, tx *bstore.Tx, size int64) error {

2133 du := DiskUsage{ID: 1}

2134 if err := tx.Get(&du); err != nil {

2135 return fmt.Errorf("get diskusage: %v", err)

2136 }

2137 du.MessageSize += size

2138 if du.MessageSize < 0 {

2139 log.Error("negative total message size", slog.Int64("delta", size), slog.Int64("newtotalsize", du.MessageSize))

2140 }

2141 if err := tx.Update(&du); err != nil {

2142 return fmt.Errorf("update total message size: %v", err)

2143 }

2144 return nil

2145}

2146

2147// QuotaMessageSize returns the effective maximum total message size for an

2148// account. Returns 0 if there is no maximum.

2149func (a *Account) QuotaMessageSize() int64 {

2150 conf, _ := a.Conf()

2151 size := conf.QuotaMessageSize

2152 if size == 0 {

2153 size = mox.Conf.Static.QuotaMessageSize

2154 }

2155 if size < 0 {

2156 size = 0

2157 }

2158 return size

2159}

2160

2161// CanAddMessageSize checks if a message of size bytes can be added, depending on

2162// total message size and configured quota for account.

2163func (a *Account) CanAddMessageSize(tx *bstore.Tx, size int64) (ok bool, maxSize int64, err error) {

2164 maxSize = a.QuotaMessageSize()

2165 if maxSize <= 0 {

2166 return true, 0, nil

2167 }

2168

2169 du := DiskUsage{ID: 1}

2170 if err := tx.Get(&du); err != nil {

2171 return false, maxSize, fmt.Errorf("get diskusage: %v", err)

2172 }

2173 return du.MessageSize+size <= maxSize, maxSize, nil

2174}

2175

2176// We keep a cache of recent successful authentications, so we don't have to bcrypt successful calls each time.

2177var authCache = struct {

2178 sync.Mutex

2179 success map[authKey]string

2180}{

2181 success: map[authKey]string{},

2182}

2183

2184type authKey struct {

2185 email, hash string

2186}

2187

2188// StartAuthCache starts a goroutine that regularly clears the auth cache.

2189func StartAuthCache() {

2190 go manageAuthCache()

2191}

2192

2193func manageAuthCache() {

2194 for {

2195 authCache.Lock()

2196 authCache.success = map[authKey]string{}

2197 authCache.Unlock()

2198 time.Sleep(15 * time.Minute)

2199 }

2200}

2201

2202// OpenEmailAuth opens an account given an email address and password.

2203//

2204// The email address may contain a catchall separator.

2205func OpenEmailAuth(log mlog.Log, email string, password string) (acc *Account, rerr error) {

2206 password, err := precis.OpaqueString.String(password)

2207 if err != nil {

2208 return nil, ErrUnknownCredentials

2209 }

2210

2211 acc, _, rerr = OpenEmail(log, email)

2212 if rerr != nil {

2213 return

2214 }

2215

2216 defer func() {

2217 if rerr != nil && acc != nil {

2218 err := acc.Close()

2219 log.Check(err, "closing account after open auth failure")

2220 acc = nil

2221 }

2222 }()

2223

2224 pw, err := bstore.QueryDB[Password](context.TODO(), acc.DB).Get()

2225 if err != nil {

2226 if err == bstore.ErrAbsent {

2227 return acc, ErrUnknownCredentials

2228 }

2229 return acc, fmt.Errorf("looking up password: %v", err)

2230 }

2231 authCache.Lock()

2232 ok := len(password) >= 8 && authCache.success[authKey{email, pw.Hash}] == password

2233 authCache.Unlock()

2234 if ok {

2235 return

2236 }

2237 if err := bcrypt.CompareHashAndPassword([]byte(pw.Hash), []byte(password)); err != nil {

2238 rerr = ErrUnknownCredentials

2239 } else {

2240 authCache.Lock()

2241 authCache.success[authKey{email, pw.Hash}] = password

2242 authCache.Unlock()

2243 }

2244 return

2245}

2246

2247// OpenEmail opens an account given an email address.

2248//

2249// The email address may contain a catchall separator.

2250func OpenEmail(log mlog.Log, email string) (*Account, config.Destination, error) {

2251 addr, err := smtp.ParseAddress(email)

2252 if err != nil {

2253 return nil, config.Destination{}, fmt.Errorf("%w: %v", ErrUnknownCredentials, err)

2254 }

2255 accountName, _, _, dest, err := mox.LookupAddress(addr.Localpart, addr.Domain, false, false)

2256 if err != nil && (errors.Is(err, mox.ErrAddressNotFound) || errors.Is(err, mox.ErrDomainNotFound)) {

2257 return nil, config.Destination{}, ErrUnknownCredentials

2258 } else if err != nil {

2259 return nil, config.Destination{}, fmt.Errorf("looking up address: %v", err)

2260 }

2261 acc, err := OpenAccount(log, accountName)

2262 if err != nil {

2263 return nil, config.Destination{}, err

2264 }

2265 return acc, dest, nil

2266}

2267

2268// 64 characters, must be power of 2 for MessagePath

2269const msgDirChars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ-_"

2270

2271// MessagePath returns the filename of the on-disk filename, relative to the

2272// containing directory such as <account>/msg or queue.

2273// Returns names like "AB/1".

2274func MessagePath(messageID int64) string {

2275 return strings.Join(messagePathElems(messageID), string(filepath.Separator))

2276}

2277

2278// messagePathElems returns the elems, for a single join without intermediate

2279// string allocations.

2280func messagePathElems(messageID int64) []string {

2281 v := messageID >> 13 // 8k files per directory.

2282 dir := ""

2283 for {

2284 dir += string(msgDirChars[int(v)&(len(msgDirChars)-1)])

2285 v >>= 6

2286 if v == 0 {

2287 break

2288 }

2289 }

2290 return []string{dir, strconv.FormatInt(messageID, 10)}

2291}

2292

2293// Set returns a copy of f, with each flag that is true in mask set to the

2294// value from flags.

2295func (f Flags) Set(mask, flags Flags) Flags {

2296 set := func(d *bool, m, v bool) {

2297 if m {

2298 *d = v

2299 }

2300 }

2301 r := f

2302 set(&r.Seen, mask.Seen, flags.Seen)

2303 set(&r.Answered, mask.Answered, flags.Answered)

2304 set(&r.Flagged, mask.Flagged, flags.Flagged)

2305 set(&r.Forwarded, mask.Forwarded, flags.Forwarded)

2306 set(&r.Junk, mask.Junk, flags.Junk)

2307 set(&r.Notjunk, mask.Notjunk, flags.Notjunk)

2308 set(&r.Deleted, mask.Deleted, flags.Deleted)

2309 set(&r.Draft, mask.Draft, flags.Draft)

2310 set(&r.Phishing, mask.Phishing, flags.Phishing)

2311 set(&r.MDNSent, mask.MDNSent, flags.MDNSent)

2312 return r

2313}

2314

2315// Changed returns a mask of flags that have been between f and other.

2316func (f Flags) Changed(other Flags) (mask Flags) {

2317 mask.Seen = f.Seen != other.Seen

2318 mask.Answered = f.Answered != other.Answered

2319 mask.Flagged = f.Flagged != other.Flagged

2320 mask.Forwarded = f.Forwarded != other.Forwarded

2321 mask.Junk = f.Junk != other.Junk

2322 mask.Notjunk = f.Notjunk != other.Notjunk

2323 mask.Deleted = f.Deleted != other.Deleted

2324 mask.Draft = f.Draft != other.Draft

2325 mask.Phishing = f.Phishing != other.Phishing

2326 mask.MDNSent = f.MDNSent != other.MDNSent

2327 return

2328}

2329

2330// Strings returns the flags that are set in their string form.

2331func (f Flags) Strings() []string {

2332 fields := []struct {

2333 word string

2334 have bool

2335 }{

2336 {`$forwarded`, f.Forwarded},

2337 {`$junk`, f.Junk},

2338 {`$mdnsent`, f.MDNSent},

2339 {`$notjunk`, f.Notjunk},

2340 {`$phishing`, f.Phishing},

2341 {`\answered`, f.Answered},

2342 {`\deleted`, f.Deleted},

2343 {`\draft`, f.Draft},

2344 {`\flagged`, f.Flagged},

2345 {`\seen`, f.Seen},

2346 }

2347 var l []string

2348 for _, fh := range fields {

2349 if fh.have {

2350 l = append(l, fh.word)

2351 }

2352 }

2353 return l

2354}

2355

2356var systemWellKnownFlags = map[string]bool{

2357 `\answered`: true,

2358 `\flagged`: true,

2359 `\deleted`: true,

2360 `\seen`: true,

2361 `\draft`: true,

2362 `$junk`: true,

2363 `$notjunk`: true,

2364 `$forwarded`: true,

2365 `$phishing`: true,

2366 `$mdnsent`: true,

2367}

2368

2369// ParseFlagsKeywords parses a list of textual flags into system/known flags, and

2370// other keywords. Keywords are lower-cased and sorted and check for valid syntax.

2371func ParseFlagsKeywords(l []string) (flags Flags, keywords []string, rerr error) {

2372 fields := map[string]*bool{

2373 `\answered`: &flags.Answered,

2374 `\flagged`: &flags.Flagged,

2375 `\deleted`: &flags.Deleted,

2376 `\seen`: &flags.Seen,

2377 `\draft`: &flags.Draft,

2378 `$junk`: &flags.Junk,

2379 `$notjunk`: &flags.Notjunk,

2380 `$forwarded`: &flags.Forwarded,

2381 `$phishing`: &flags.Phishing,

2382 `$mdnsent`: &flags.MDNSent,

2383 }

2384 seen := map[string]bool{}

2385 for _, f := range l {

2386 f = strings.ToLower(f)

2387 if field, ok := fields[f]; ok {

2388 *field = true

2389 } else if seen[f] {

2390 if mox.Pedantic {

2391 return Flags{}, nil, fmt.Errorf("duplicate keyword %s", f)

2392 }

2393 } else {

2394 if err := CheckKeyword(f); err != nil {

2395 return Flags{}, nil, fmt.Errorf("invalid keyword %s", f)

2396 }

2397 keywords = append(keywords, f)

2398 seen[f] = true

2399 }

2400 }

2401 sort.Strings(keywords)

2402 return flags, keywords, nil

2403}

2404

2405// RemoveKeywords removes keywords from l, returning whether any modifications were

2406// made, and a slice, a new slice in case of modifications. Keywords must have been

2407// validated earlier, e.g. through ParseFlagKeywords or CheckKeyword. Should only

2408// be used with valid keywords, not with system flags like \Seen.

2409func RemoveKeywords(l, remove []string) ([]string, bool) {

2410 var copied bool

2411 var changed bool

2412 for _, k := range remove {

2413 if i := slices.Index(l, k); i >= 0 {

2414 if !copied {

2415 l = append([]string{}, l...)

2416 copied = true

2417 }

2418 copy(l[i:], l[i+1:])

2419 l = l[:len(l)-1]

2420 changed = true

2421 }

2422 }

2423 return l, changed

2424}

2425

2426// MergeKeywords adds keywords from add into l, returning whether it added any

2427// keyword, and the slice with keywords, a new slice if modifications were made.

2428// Keywords are only added if they aren't already present. Should only be used with

2429// keywords, not with system flags like \Seen.

2430func MergeKeywords(l, add []string) ([]string, bool) {

2431 var copied bool

2432 var changed bool

2433 for _, k := range add {

2434 if !slices.Contains(l, k) {

2435 if !copied {

2436 l = append([]string{}, l...)

2437 copied = true

2438 }

2439 l = append(l, k)

2440 changed = true

2441 }

2442 }

2443 if changed {

2444 sort.Strings(l)

2445 }

2446 return l, changed

2447}

2448

2449// CheckKeyword returns an error if kw is not a valid keyword. Kw should

2450// already be in lower-case.

2451func CheckKeyword(kw string) error {

2452 if kw == "" {

2453 return fmt.Errorf("keyword cannot be empty")

2454 }

2455 if systemWellKnownFlags[kw] {

2456 return fmt.Errorf("cannot use well-known flag as keyword")

2457 }

2458 for _, c := range kw {

2459 // ../rfc/9051:6334

2460 if c <= ' ' || c > 0x7e || c >= 'A' && c <= 'Z' || strings.ContainsRune(`(){%*"\]`, c) {

2461 return errors.New(`not a valid keyword, must be lower-case ascii without spaces and without any of these characters: (){%*"\]`)

2462 }

2463 }

2464 return nil

2465}

2466

2467// SendLimitReached checks whether sending a message to recipients would reach

2468// the limit of outgoing messages for the account. If so, the message should

2469// not be sent. If the returned numbers are >= 0, the limit was reached and the

2470// values are the configured limits.

2471//

2472// To limit damage to the internet and our reputation in case of account

2473// compromise, we limit the max number of messages sent in a 24 hour window, both

2474// total number of messages and number of first-time recipients.

2475func (a *Account) SendLimitReached(tx *bstore.Tx, recipients []smtp.Path) (msglimit, rcptlimit int, rerr error) {

2476 conf, _ := a.Conf()

2477 msgmax := conf.MaxOutgoingMessagesPerDay

2478 if msgmax == 0 {

2479 // For human senders, 1000 recipients in a day is quite a lot.

2480 msgmax = 1000

2481 }

2482 rcptmax := conf.MaxFirstTimeRecipientsPerDay

2483 if rcptmax == 0 {

2484 // Human senders may address a new human-sized list of people once in a while. In

2485 // case of a compromise, a spammer will probably try to send to many new addresses.

2486 rcptmax = 200

2487 }

2488

2489 rcpts := map[string]time.Time{}

2490 n := 0

2491 err := bstore.QueryTx[Outgoing](tx).FilterGreater("Submitted", time.Now().Add(-24*time.Hour)).ForEach(func(o Outgoing) error {

2492 n++

2493 if rcpts[o.Recipient].IsZero() || o.Submitted.Before(rcpts[o.Recipient]) {

2494 rcpts[o.Recipient] = o.Submitted

2495 }

2496 return nil

2497 })

2498 if err != nil {

2499 return -1, -1, fmt.Errorf("querying message recipients in past 24h: %w", err)

2500 }

2501 if n+len(recipients) > msgmax {

2502 return msgmax, -1, nil

2503 }

2504

2505 // Only check if max first-time recipients is reached if there are enough messages

2506 // to trigger the limit.

2507 if n+len(recipients) < rcptmax {

2508 return -1, -1, nil

2509 }

2510

2511 isFirstTime := func(rcpt string, before time.Time) (bool, error) {

2512 exists, err := bstore.QueryTx[Outgoing](tx).FilterNonzero(Outgoing{Recipient: rcpt}).FilterLess("Submitted", before).Exists()

2513 return !exists, err

2514 }

2515

2516 firsttime := 0

2517 now := time.Now()

2518 for _, r := range recipients {

2519 if first, err := isFirstTime(r.XString(true), now); err != nil {

2520 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

2521 } else if first {

2522 firsttime++

2523 }

2524 }

2525 for r, t := range rcpts {

2526 if first, err := isFirstTime(r, t); err != nil {

2527 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

2528 } else if first {

2529 firsttime++

2530 }

2531 }

2532 if firsttime > rcptmax {

2533 return -1, rcptmax, nil

2534 }

2535 return -1, -1, nil

2536}

2537

2538// MailboxCreate creates a new mailbox, including any missing parent mailboxes,

2539// the total list of created mailboxes is returned in created. On success, if

2540// exists is false and rerr nil, the changes must be broadcasted by the caller.

2541//

2542// Name must be in normalized form.

2543func (a *Account) MailboxCreate(tx *bstore.Tx, name string) (changes []Change, created []string, exists bool, rerr error) {

2544 elems := strings.Split(name, "/")

2545 var p string

2546 for i, elem := range elems {

2547 if i > 0 {

2548 p += "/"

2549 }

2550 p += elem

2551 exists, err := a.MailboxExists(tx, p)

2552 if err != nil {

2553 return nil, nil, false, fmt.Errorf("checking if mailbox exists")

2554 }

2555 if exists {

2556 if i == len(elems)-1 {

2557 return nil, nil, true, fmt.Errorf("mailbox already exists")

2558 }

2559 continue

2560 }

2561 _, nchanges, err := a.MailboxEnsure(tx, p, true)

2562 if err != nil {

2563 return nil, nil, false, fmt.Errorf("ensuring mailbox exists")

2564 }

2565 changes = append(changes, nchanges...)

2566 created = append(created, p)

2567 }

2568 return changes, created, false, nil

2569}

2570

2571// MailboxRename renames mailbox mbsrc to dst, and any missing parents for the

2572// destination, and any children of mbsrc and the destination.

2573//

2574// Names must be normalized and cannot be Inbox.

2575func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc Mailbox, dst string) (changes []Change, isInbox, notExists, alreadyExists bool, rerr error) {

2576 if mbsrc.Name == "Inbox" || dst == "Inbox" {

2577 return nil, true, false, false, fmt.Errorf("inbox cannot be renamed")

2578 }

2579

2580 // We gather existing mailboxes that we need for deciding what to create/delete/update.

2581 q := bstore.QueryTx[Mailbox](tx)

2582 srcPrefix := mbsrc.Name + "/"

2583 dstRoot := strings.SplitN(dst, "/", 2)[0]

2584 dstRootPrefix := dstRoot + "/"

2585 q.FilterFn(func(mb Mailbox) bool {

2586 return mb.Name == mbsrc.Name || strings.HasPrefix(mb.Name, srcPrefix) || mb.Name == dstRoot || strings.HasPrefix(mb.Name, dstRootPrefix)

2587 })

2588 q.SortAsc("Name") // We'll rename the parents before children.

2589 l, err := q.List()

2590 if err != nil {

2591 return nil, false, false, false, fmt.Errorf("listing relevant mailboxes: %v", err)

2592 }

2593

2594 mailboxes := map[string]Mailbox{}

2595 for _, mb := range l {

2596 mailboxes[mb.Name] = mb

2597 }

2598

2599 if _, ok := mailboxes[mbsrc.Name]; !ok {

2600 return nil, false, true, false, fmt.Errorf("mailbox does not exist")

2601 }

2602

2603 uidval, err := a.NextUIDValidity(tx)

2604 if err != nil {

2605 return nil, false, false, false, fmt.Errorf("next uid validity: %v", err)

2606 }

2607

2608 // Ensure parent mailboxes for the destination paths exist.

2609 var parent string

2610 dstElems := strings.Split(dst, "/")

2611 for i, elem := range dstElems[:len(dstElems)-1] {

2612 if i > 0 {

2613 parent += "/"

2614 }

2615 parent += elem

2616

2617 mb, ok := mailboxes[parent]

2618 if ok {

2619 continue

2620 }

2621 omb := mb

2622 mb = Mailbox{

2623 ID: omb.ID,

2624 Name: parent,

2625 UIDValidity: uidval,

2626 UIDNext: 1,

2627 HaveCounts: true,

2628 }

2629 if err := tx.Insert(&mb); err != nil {

2630 return nil, false, false, false, fmt.Errorf("creating parent mailbox %q: %v", mb.Name, err)

2631 }

2632 if err := tx.Get(&Subscription{Name: parent}); err != nil {

2633 if err := tx.Insert(&Subscription{Name: parent}); err != nil {

2634 return nil, false, false, false, fmt.Errorf("creating subscription for %q: %v", parent, err)

2635 }

2636 }

2637 changes = append(changes, ChangeAddMailbox{Mailbox: mb, Flags: []string{`\Subscribed`}})

2638 }

2639

2640 // Process src mailboxes, renaming them to dst.

2641 for _, srcmb := range l {

2642 if srcmb.Name != mbsrc.Name && !strings.HasPrefix(srcmb.Name, srcPrefix) {

2643 continue

2644 }

2645 srcName := srcmb.Name

2646 dstName := dst + srcmb.Name[len(mbsrc.Name):]

2647 if _, ok := mailboxes[dstName]; ok {

2648 return nil, false, false, true, fmt.Errorf("destination mailbox %q already exists", dstName)

2649 }

2650

2651 srcmb.Name = dstName

2652 srcmb.UIDValidity = uidval

2653 if err := tx.Update(&srcmb); err != nil {

2654 return nil, false, false, false, fmt.Errorf("renaming mailbox: %v", err)

2655 }

2656

2657 var dstFlags []string

2658 if tx.Get(&Subscription{Name: dstName}) == nil {

2659 dstFlags = []string{`\Subscribed`}

2660 }

2661 changes = append(changes, ChangeRenameMailbox{MailboxID: srcmb.ID, OldName: srcName, NewName: dstName, Flags: dstFlags})

2662 }

2663

2664 // If we renamed e.g. a/b to a/b/c/d, and a/b/c to a/b/c/d/c, we'll have to recreate a/b and a/b/c.

2665 srcElems := strings.Split(mbsrc.Name, "/")

2666 xsrc := mbsrc.Name

2667 for i := 0; i < len(dstElems) && strings.HasPrefix(dst, xsrc+"/"); i++ {

2668 mb := Mailbox{

2669 UIDValidity: uidval,

2670 UIDNext: 1,

2671 Name: xsrc,

2672 HaveCounts: true,

2673 }

2674 if err := tx.Insert(&mb); err != nil {

2675 return nil, false, false, false, fmt.Errorf("creating mailbox at old path %q: %v", mb.Name, err)

2676 }

2677 xsrc += "/" + dstElems[len(srcElems)+i]

2678 }

2679 return changes, false, false, false, nil

2680}

2681

2682// MailboxDelete deletes a mailbox by ID. If it has children, the return value

2683// indicates that and an error is returned.

2684//

2685// Caller should broadcast the changes and remove files for the removed message IDs.

2686func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx, mailbox Mailbox) (changes []Change, removeMessageIDs []int64, hasChildren bool, rerr error) {

2687 // Look for existence of child mailboxes. There is a lot of text in the IMAP RFCs about

2688 // NoInferior and NoSelect. We just require only leaf mailboxes are deleted.

2689 qmb := bstore.QueryTx[Mailbox](tx)

2690 mbprefix := mailbox.Name + "/"

2691 qmb.FilterFn(func(mb Mailbox) bool {

2692 return strings.HasPrefix(mb.Name, mbprefix)

2693 })

2694 if childExists, err := qmb.Exists(); err != nil {

2695 return nil, nil, false, fmt.Errorf("checking if mailbox has child: %v", err)

2696 } else if childExists {

2697 return nil, nil, true, fmt.Errorf("mailbox has a child, only leaf mailboxes can be deleted")

2698 }

2699

2700 // todo jmap: instead of completely deleting a mailbox and its messages, we need to mark them all as expunged.

2701

2702 qm := bstore.QueryTx[Message](tx)

2703 qm.FilterNonzero(Message{MailboxID: mailbox.ID})

2704 remove, err := qm.List()

2705 if err != nil {

2706 return nil, nil, false, fmt.Errorf("listing messages to remove: %v", err)

2707 }

2708

2709 if len(remove) > 0 {

2710 removeIDs := make([]any, len(remove))

2711 for i, m := range remove {

2712 removeIDs[i] = m.ID

2713 }

2714 qmr := bstore.QueryTx[Recipient](tx)

2715 qmr.FilterEqual("MessageID", removeIDs...)

2716 if _, err = qmr.Delete(); err != nil {

2717 return nil, nil, false, fmt.Errorf("removing message recipients for messages: %v", err)

2718 }

2719

2720 qm = bstore.QueryTx[Message](tx)

2721 qm.FilterNonzero(Message{MailboxID: mailbox.ID})

2722 if _, err := qm.Delete(); err != nil {

2723 return nil, nil, false, fmt.Errorf("removing messages: %v", err)

2724 }

2725

2726 var totalSize int64

2727 for _, m := range remove {

2728 if !m.Expunged {

2729 removeMessageIDs = append(removeMessageIDs, m.ID)

2730 totalSize += m.Size

2731 }

2732 }

2733 if err := a.AddMessageSize(log, tx, -totalSize); err != nil {

2734 return nil, nil, false, fmt.Errorf("updating disk usage: %v", err)

2735 }

2736

2737 // Mark messages as not needing training. Then retrain them, so they are untrained if they were.

2738 n := 0

2739 o := 0

2740 for _, m := range remove {

2741 if !m.Expunged {

2742 remove[o] = m

2743 remove[o].Junk = false

2744 remove[o].Notjunk = false

2745 n++

2746 }

2747 }

2748 remove = remove[:n]

2749 if err := a.RetrainMessages(ctx, log, tx, remove, true); err != nil {

2750 return nil, nil, false, fmt.Errorf("untraining deleted messages: %v", err)

2751 }

2752 }

2753

2754 if err := tx.Delete(&Mailbox{ID: mailbox.ID}); err != nil {

2755 return nil, nil, false, fmt.Errorf("removing mailbox: %v", err)

2756 }

2757 return []Change{ChangeRemoveMailbox{MailboxID: mailbox.ID, Name: mailbox.Name}}, removeMessageIDs, false, nil

2758}

2759

2760// CheckMailboxName checks if name is valid, returning an INBOX-normalized name.

2761// I.e. it changes various casings of INBOX and INBOX/* to Inbox and Inbox/*.

2762// Name is invalid if it contains leading/trailing/double slashes, or when it isn't

2763// unicode-normalized, or when empty or has special characters.

2764//

2765// If name is the inbox, and allowInbox is false, this is indicated with the isInbox return parameter.

2766// For that case, and for other invalid names, an error is returned.

2767func CheckMailboxName(name string, allowInbox bool) (normalizedName string, isInbox bool, rerr error) {

2768 first := strings.SplitN(name, "/", 2)[0]

2769 if strings.EqualFold(first, "inbox") {

2770 if len(name) == len("inbox") && !allowInbox {

2771 return "", true, fmt.Errorf("special mailbox name Inbox not allowed")

2772 }

2773 name = "Inbox" + name[len("Inbox"):]

2774 }

2775

2776 if norm.NFC.String(name) != name {

2777 return "", false, errors.New("non-unicode-normalized mailbox names not allowed")

2778 }

2779

2780 if name == "" {

2781 return "", false, errors.New("empty mailbox name")

2782 }

2783 if strings.HasPrefix(name, "/") || strings.HasSuffix(name, "/") || strings.Contains(name, "//") {

2784 return "", false, errors.New("bad slashes in mailbox name")

2785 }

2786

2787 // "%" and "*" are difficult to use with the IMAP LIST command, but we allow mostly

2788 // allow them. ../rfc/3501:1002 ../rfc/9051:983

2789 if strings.HasPrefix(name, "#") {

2790 return "", false, errors.New("mailbox name cannot start with hash due to conflict with imap namespaces")

2791 }

2792

2793 // "#" and "&" are special in IMAP mailbox names. "#" for namespaces, "&" for

2794 // IMAP-UTF-7 encoding. We do allow them. ../rfc/3501:1018 ../rfc/9051:991

2795

2796 for _, c := range name {

2797 // ../rfc/3501:999 ../rfc/6855:192 ../rfc/9051:979

2798 if c <= 0x1f || c >= 0x7f && c <= 0x9f || c == 0x2028 || c == 0x2029 {

2799 return "", false, errors.New("control characters not allowed in mailbox name")

2800 }

2801 }

2802 return name, false, nil

2803}

2804