1/*

2Package store implements storage for accounts, their mailboxes, IMAP

3subscriptions and messages, and broadcasts updates (e.g. mail delivery) to

4interested sessions (e.g. IMAP connections).

6Layout of storage for accounts:

8 <DataDir>/accounts/<name>/index.db

9 <DataDir>/accounts/<name>/msg/[a-zA-Z0-9_-]+/<id>

11Index.db holds tables for user information, mailboxes, and messages. Messages

12are stored in the msg/ subdirectory, each in their own file. The on-disk message

13does not contain headers generated during an incoming SMTP transaction, such as

14Received and Authentication-Results headers. Those are in the database to

15prevent having to rewrite incoming messages (e.g. Authentication-Result for DKIM

16signatures can only be determined after having read the message). Messages must

17be read through MsgReader, which transparently adds the prefix from the

18database.

19*/

20package store

22// todo: make up a function naming scheme that indicates whether caller should broadcast changes.

24import (

25 "context"

26 "crypto/md5"

27 cryptorand "crypto/rand"

28 "crypto/sha1"

29 "crypto/sha256"

30 "encoding"

31 "encoding/json"

32 "errors"

33 "fmt"

34 "hash"

35 "io"

36 "log/slog"

37 "os"

38 "path/filepath"

39 "runtime/debug"

40 "slices"

41 "sort"

42 "strconv"

43 "strings"

44 "sync"

45 "time"

47 "golang.org/x/crypto/bcrypt"

48 "golang.org/x/text/secure/precis"

49 "golang.org/x/text/unicode/norm"

51 "github.com/mjl-/bstore"

53 "github.com/mjl-/mox/config"

54 "github.com/mjl-/mox/dns"

55 "github.com/mjl-/mox/message"

56 "github.com/mjl-/mox/metrics"

57 "github.com/mjl-/mox/mlog"

58 "github.com/mjl-/mox/mox-"

59 "github.com/mjl-/mox/moxio"

60 "github.com/mjl-/mox/moxvar"

61 "github.com/mjl-/mox/publicsuffix"

62 "github.com/mjl-/mox/scram"

63 "github.com/mjl-/mox/smtp"

64)

66// If true, each time an account is closed its database file is checked for

67// consistency. If an inconsistency is found, panic is called. Set by default

68// because of all the packages with tests, the mox main function sets it to

69// false again.

70var CheckConsistencyOnClose = true

72var (

73 ErrUnknownMailbox = errors.New("no such mailbox")

74 ErrUnknownCredentials = errors.New("credentials not found")

75 ErrAccountUnknown = errors.New("no such account")

76 ErrOverQuota = errors.New("account over quota")

77 ErrLoginDisabled = errors.New("login disabled for account")

78)

80var DefaultInitialMailboxes = config.InitialMailboxes{

81 SpecialUse: config.SpecialUseMailboxes{

82 Sent: "Sent",

83 Archive: "Archive",

84 Trash: "Trash",

85 Draft: "Drafts",

86 Junk: "Junk",

87 },

88}

90type SCRAM struct {

91 Salt []byte

92 Iterations int

93 SaltedPassword []byte

94}

96// CRAMMD5 holds HMAC ipad and opad hashes that are initialized with the first

97// block with (a derivation of) the key/password, so we don't store the password in plain

98// text.

99type CRAMMD5 struct {

100 Ipad hash.Hash

101 Opad hash.Hash

102}

103

104// BinaryMarshal is used by bstore to store the ipad/opad hash states.

105func (c CRAMMD5) MarshalBinary() ([]byte, error) {

106 if c.Ipad == nil || c.Opad == nil {

107 return nil, nil

108 }

109

110 ipad, err := c.Ipad.(encoding.BinaryMarshaler).MarshalBinary()

111 if err != nil {

112 return nil, fmt.Errorf("marshal ipad: %v", err)

113 }

114 opad, err := c.Opad.(encoding.BinaryMarshaler).MarshalBinary()

115 if err != nil {

116 return nil, fmt.Errorf("marshal opad: %v", err)

117 }

118 buf := make([]byte, 2+len(ipad)+len(opad))

119 ipadlen := uint16(len(ipad))

120 buf[0] = byte(ipadlen >> 8)

121 buf[1] = byte(ipadlen >> 0)

122 copy(buf[2:], ipad)

123 copy(buf[2+len(ipad):], opad)

124 return buf, nil

125}

126

127// BinaryUnmarshal is used by bstore to restore the ipad/opad hash states.

128func (c *CRAMMD5) UnmarshalBinary(buf []byte) error {

129 if len(buf) == 0 {

130 *c = CRAMMD5{}

131 return nil

132 }

133 if len(buf) < 2 {

134 return fmt.Errorf("short buffer")

135 }

136 ipadlen := int(uint16(buf[0])<<8 | uint16(buf[1])<<0)

137 if len(buf) < 2+ipadlen {

138 return fmt.Errorf("buffer too short for ipadlen")

139 }

140 ipad := md5.New()

141 opad := md5.New()

142 if err := ipad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2 : 2+ipadlen]); err != nil {

143 return fmt.Errorf("unmarshal ipad: %v", err)

144 }

145 if err := opad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2+ipadlen:]); err != nil {

146 return fmt.Errorf("unmarshal opad: %v", err)

147 }

148 *c = CRAMMD5{ipad, opad}

149 return nil

150}

151

152// Password holds credentials in various forms, for logging in with SMTP/IMAP.

153type Password struct {

154 Hash string // bcrypt hash for IMAP LOGIN, SASL PLAIN and HTTP basic authentication.

155 CRAMMD5 CRAMMD5 // For SASL CRAM-MD5.

156 SCRAMSHA1 SCRAM // For SASL SCRAM-SHA-1.

157 SCRAMSHA256 SCRAM // For SASL SCRAM-SHA-256.

158}

159

160// Subjectpass holds the secret key used to sign subjectpass tokens.

161type Subjectpass struct {

162 Email string // Our destination address (canonical, with catchall localpart stripped).

163 Key string

164}

165

166// NextUIDValidity is a singleton record in the database with the next UIDValidity

167// to use for the next mailbox.

168type NextUIDValidity struct {

169 ID int // Just a single record with ID 1.

170 Next uint32

171}

172

173// SyncState track ModSeqs.

174type SyncState struct {

175 ID int // Just a single record with ID 1.

176

177 // Last used, next assigned will be one higher. The first value we hand out is 2.

178 // That's because 0 (the default value for old existing messages, from before the

179 // Message.ModSeq field) is special in IMAP, so we return it as 1.

180 LastModSeq ModSeq `bstore:"nonzero"`

181

182 // Highest ModSeq of expunged record that we deleted. When a clients synchronizes

183 // and requests changes based on a modseq before this one, we don't have the

184 // history to provide information about deletions. We normally keep these expunged

185 // records around, but we may periodically truly delete them to reclaim storage

186 // space. Initially set to -1 because we don't want to match with any ModSeq in the

187 // database, which can be zero values.

188 HighestDeletedModSeq ModSeq

189}

190

191// Mailbox is collection of messages, e.g. Inbox or Sent.

192type Mailbox struct {

193 ID int64

194

195 // "Inbox" is the name for the special IMAP "INBOX". Slash separated

196 // for hierarchy.

197 Name string `bstore:"nonzero,unique"`

198

199 // If UIDs are invalidated, e.g. when renaming a mailbox to a previously existing

200 // name, UIDValidity must be changed. Used by IMAP for synchronization.

201 UIDValidity uint32

202

203 // UID likely to be assigned to next message. Used by IMAP to detect messages

204 // delivered to a mailbox.

205 UIDNext UID

206

207 SpecialUse

208

209 // Keywords as used in messages. Storing a non-system keyword for a message

210 // automatically adds it to this list. Used in the IMAP FLAGS response. Only

211 // "atoms" are allowed (IMAP syntax), keywords are case-insensitive, only stored in

212 // lower case (for JMAP), sorted.

213 Keywords []string

214

215 HaveCounts bool // Whether MailboxCounts have been initialized.

216 MailboxCounts // Statistics about messages, kept up to date whenever a change happens.

217}

218

219// Annotation is a per-mailbox or global (per-account) annotation for the IMAP

220// metadata extension, currently always a private annotation.

221type Annotation struct {

222 ID int64

223

224 // Can be zero, indicates global (per-account) annotation.

225 MailboxID int64 `bstore:"ref Mailbox,unique MailboxID+Key"`

226

227 // "Entry name", always starts with "/private/". Stored lower-case, comparisons

228 // must be done case-insensitively.

229 Key string `bstore:"nonzero"`

230

231 IsString bool // If true, the value is a string instead of bytes.

232 Value []byte

233}

234

235// Change returns a broadcastable change for the annotation.

236func (a Annotation) Change(mailboxName string) ChangeAnnotation {

237 return ChangeAnnotation{a.MailboxID, mailboxName, a.Key}

238}

239

240// MailboxCounts tracks statistics about messages for a mailbox.

241type MailboxCounts struct {

242 Total int64 // Total number of messages, excluding \Deleted. For JMAP.

243 Deleted int64 // Number of messages with \Deleted flag. Used for IMAP message count that includes messages with \Deleted.

244 Unread int64 // Messages without \Seen, excluding those with \Deleted, for JMAP.

245 Unseen int64 // Messages without \Seen, including those with \Deleted, for IMAP.

246 Size int64 // Number of bytes for all messages.

247}

248

249func (mc MailboxCounts) String() string {

250 return fmt.Sprintf("%d total, %d deleted, %d unread, %d unseen, size %d bytes", mc.Total, mc.Deleted, mc.Unread, mc.Unseen, mc.Size)

251}

252

253// Add increases mailbox counts mc with those of delta.

254func (mc *MailboxCounts) Add(delta MailboxCounts) {

255 mc.Total += delta.Total

256 mc.Deleted += delta.Deleted

257 mc.Unread += delta.Unread

258 mc.Unseen += delta.Unseen

259 mc.Size += delta.Size

260}

261

262// Add decreases mailbox counts mc with those of delta.

263func (mc *MailboxCounts) Sub(delta MailboxCounts) {

264 mc.Total -= delta.Total

265 mc.Deleted -= delta.Deleted

266 mc.Unread -= delta.Unread

267 mc.Unseen -= delta.Unseen

268 mc.Size -= delta.Size

269}

270

271// SpecialUse identifies a specific role for a mailbox, used by clients to

272// understand where messages should go.

273type SpecialUse struct {

274 Archive bool

275 Draft bool // "Drafts"

276 Junk bool

277 Sent bool

278 Trash bool

279}

280

281// CalculateCounts calculates the full current counts for messages in the mailbox.

282func (mb *Mailbox) CalculateCounts(tx *bstore.Tx) (mc MailboxCounts, err error) {

283 q := bstore.QueryTx[Message](tx)

284 q.FilterNonzero(Message{MailboxID: mb.ID})

285 q.FilterEqual("Expunged", false)

286 err = q.ForEach(func(m Message) error {

287 mc.Add(m.MailboxCounts())

288 return nil

289 })

290 return

291}

292

293// ChangeSpecialUse returns a change for special-use flags, for broadcasting to

294// other connections.

295func (mb Mailbox) ChangeSpecialUse() ChangeMailboxSpecialUse {

296 return ChangeMailboxSpecialUse{mb.ID, mb.Name, mb.SpecialUse}

297}

298

299// ChangeKeywords returns a change with new keywords for a mailbox (e.g. after

300// setting a new keyword on a message in the mailbox), for broadcasting to other

301// connections.

302func (mb Mailbox) ChangeKeywords() ChangeMailboxKeywords {

303 return ChangeMailboxKeywords{mb.ID, mb.Name, mb.Keywords}

304}

305

306// KeywordsChanged returns whether the keywords in a mailbox have changed.

307func (mb Mailbox) KeywordsChanged(origmb Mailbox) bool {

308 if len(mb.Keywords) != len(origmb.Keywords) {

309 return true

310 }

311 // Keywords are stored sorted.

312 for i, kw := range mb.Keywords {

313 if origmb.Keywords[i] != kw {

314 return true

315 }

316 }

317 return false

318}

319

320// CountsChange returns a change with mailbox counts.

321func (mb Mailbox) ChangeCounts() ChangeMailboxCounts {

322 return ChangeMailboxCounts{mb.ID, mb.Name, mb.MailboxCounts}

323}

324

325// Subscriptions are separate from existence of mailboxes.

326type Subscription struct {

327 Name string

328}

329

330// Flags for a mail message.

331type Flags struct {

332 Seen bool

333 Answered bool

334 Flagged bool

335 Forwarded bool

336 Junk bool

337 Notjunk bool

338 Deleted bool

339 Draft bool

340 Phishing bool

341 MDNSent bool

342}

343

344// FlagsAll is all flags set, for use as mask.

345var FlagsAll = Flags{true, true, true, true, true, true, true, true, true, true}

346

347// Validation of "message From" domain.

348type Validation uint8

349

350const (

351 ValidationUnknown Validation = 0

352 ValidationStrict Validation = 1 // Like DMARC, with strict policies.

353 ValidationDMARC Validation = 2 // Actual DMARC policy.

354 ValidationRelaxed Validation = 3 // Like DMARC, with relaxed policies.

355 ValidationPass Validation = 4 // For SPF.

356 ValidationNeutral Validation = 5 // For SPF.

357 ValidationTemperror Validation = 6

358 ValidationPermerror Validation = 7

359 ValidationFail Validation = 8

360 ValidationSoftfail Validation = 9 // For SPF.

361 ValidationNone Validation = 10 // E.g. No records.

362)

363

364// Message stored in database and per-message file on disk.

365//

366// Contents are always the combined data from MsgPrefix and the on-disk file named

367// based on ID.

368//

369// Messages always have a header section, even if empty. Incoming messages without

370// header section must get an empty header section added before inserting.

371type Message struct {

372 // ID, unchanged over lifetime, determines path to on-disk msg file.

373 // Set during deliver.

374 ID int64

375

376 UID UID `bstore:"nonzero"` // UID, for IMAP. Set during deliver.

377 MailboxID int64 `bstore:"nonzero,unique MailboxID+UID,index MailboxID+Received,index MailboxID+ModSeq,ref Mailbox"`

378

379 // Modification sequence, for faster syncing with IMAP QRESYNC and JMAP.

380 // ModSeq is the last modification. CreateSeq is the Seq the message was inserted,

381 // always <= ModSeq. If Expunged is set, the message has been removed and should not

382 // be returned to the user. In this case, ModSeq is the Seq where the message is

383 // removed, and will never be changed again.

384 // We have an index on both ModSeq (for JMAP that synchronizes per account) and

385 // MailboxID+ModSeq (for IMAP that synchronizes per mailbox).

386 // The index on CreateSeq helps efficiently finding created messages for JMAP.

387 // The value of ModSeq is special for IMAP. Messages that existed before ModSeq was

388 // added have 0 as value. But modseq 0 in IMAP is special, so we return it as 1. If

389 // we get modseq 1 from a client, the IMAP server will translate it to 0. When we

390 // return modseq to clients, we turn 0 into 1.

391 ModSeq ModSeq `bstore:"index"`

392 CreateSeq ModSeq `bstore:"index"`

393 Expunged bool

394

395 // If set, this message was delivered to a Rejects mailbox. When it is moved to a

396 // different mailbox, its MailboxOrigID is set to the destination mailbox and this

397 // flag cleared.

398 IsReject bool

399

400 // If set, this is a forwarded message (through a ruleset with IsForward). This

401 // causes fields used during junk analysis to be moved to their Orig variants, and

402 // masked IP fields cleared, so they aren't used in junk classifications for

403 // incoming messages. This ensures the forwarded messages don't cause negative

404 // reputation for the forwarding mail server, which may also be sending regular

405 // messages.

406 IsForward bool

407

408 // MailboxOrigID is the mailbox the message was originally delivered to. Typically

409 // Inbox or Rejects, but can also be a mailbox configured in a Ruleset, or

410 // Postmaster, TLS/DMARC reporting addresses. MailboxOrigID is not changed when the

411 // message is moved to another mailbox, e.g. Archive/Trash/Junk. Used for

412 // per-mailbox reputation.

413 //

414 // MailboxDestinedID is normally 0, but when a message is delivered to the Rejects

415 // mailbox, it is set to the intended mailbox according to delivery rules,

416 // typically that of Inbox. When such a message is moved out of Rejects, the

417 // MailboxOrigID is corrected by setting it to MailboxDestinedID. This ensures the

418 // message is used for reputation calculation for future deliveries to that

419 // mailbox.

420 //

421 // These are not bstore references to prevent having to update all messages in a

422 // mailbox when the original mailbox is removed. Use of these fields requires

423 // checking if the mailbox still exists.

424 MailboxOrigID int64

425 MailboxDestinedID int64

426

427 // Received indicates time of receival over SMTP, or of IMAP APPEND.

428 Received time.Time `bstore:"default now,index"`

429

430 // SaveDate is the time of copy/move/save to a mailbox, used with IMAP SAVEDATE

431 // extension. Must be updated each time a message is copied/moved to another

432 // mailbox. Can be nil for messages from before this functionality was introduced.

433 SaveDate *time.Time `bstore:"default now"`

434

435 // Full IP address of remote SMTP server. Empty if not delivered over SMTP. The

436 // masked IPs are used to classify incoming messages. They are left empty for

437 // messages matching a ruleset for forwarded messages.

438 RemoteIP string

439 RemoteIPMasked1 string `bstore:"index RemoteIPMasked1+Received"` // For IPv4 /32, for IPv6 /64, for reputation.

440 RemoteIPMasked2 string `bstore:"index RemoteIPMasked2+Received"` // For IPv4 /26, for IPv6 /48.

441 RemoteIPMasked3 string `bstore:"index RemoteIPMasked3+Received"` // For IPv4 /21, for IPv6 /32.

442

443 // Only set if present and not an IP address. Unicode string. Empty for forwarded

444 // messages.

445 EHLODomain string `bstore:"index EHLODomain+Received"`

446 MailFrom string // With localpart and domain. Can be empty.

447 MailFromLocalpart smtp.Localpart // SMTP "MAIL FROM", can be empty.

448 // Only set if it is a domain, not an IP. Unicode string. Empty for forwarded

449 // messages, but see OrigMailFromDomain.

450 MailFromDomain string `bstore:"index MailFromDomain+Received"`

451 RcptToLocalpart smtp.Localpart // SMTP "RCPT TO", can be empty.

452 RcptToDomain string // Unicode string.

453

454 // Parsed "From" message header, used for reputation along with domain validation.

455 MsgFromLocalpart smtp.Localpart

456 MsgFromDomain string `bstore:"index MsgFromDomain+Received"` // Unicode string.

457 MsgFromOrgDomain string `bstore:"index MsgFromOrgDomain+Received"` // Unicode string.

458

459 // Simplified statements of the Validation fields below, used for incoming messages

460 // to check reputation.

461 EHLOValidated bool

462 MailFromValidated bool

463 MsgFromValidated bool

464

465 EHLOValidation Validation // Validation can also take reverse IP lookup into account, not only SPF.

466 MailFromValidation Validation // Can have SPF-specific validations like ValidationSoftfail.

467 MsgFromValidation Validation // Desirable validations: Strict, DMARC, Relaxed. Will not be just Pass.

468

469 // Domains with verified DKIM signatures. Unicode string. For forwarded messages, a

470 // DKIM domain that matched a ruleset's verified domain is left out, but included

471 // in OrigDKIMDomains.

472 DKIMDomains []string `bstore:"index DKIMDomains+Received"`

473

474 // For forwarded messages,

475 OrigEHLODomain string

476 OrigDKIMDomains []string

477

478 // Canonicalized Message-Id, always lower-case and normalized quoting, without

479 // <>'s. Empty if missing. Used for matching message threads, and to prevent

480 // duplicate reject delivery.

481 MessageID string `bstore:"index"`

482 // lower-case: ../rfc/5256:495

483

484 // For matching threads in case there is no References/In-Reply-To header. It is

485 // lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed.

486 SubjectBase string `bstore:"index"`

487 // ../rfc/5256:90

488

489 // Hash of message. For rejects delivery in case there is no Message-ID, only set

490 // when delivered as reject.

491 MessageHash []byte

492

493 // ID of message starting this thread.

494 ThreadID int64 `bstore:"index"`

495 // IDs of parent messages, from closest parent to the root message. Parent messages

496 // may be in a different mailbox, or may no longer exist. ThreadParentIDs must

497 // never contain the message id itself (a cycle), and parent messages must

498 // reference the same ancestors.

499 ThreadParentIDs []int64

500 // ThreadMissingLink is true if there is no match with a direct parent. E.g. first

501 // ID in ThreadParentIDs is not the direct ancestor (an intermediate message may

502 // have been deleted), or subject-based matching was done.

503 ThreadMissingLink bool

504 // If set, newly delivered child messages are automatically marked as read. This

505 // field is copied to new child messages. Changes are propagated to the webmail

506 // client.

507 ThreadMuted bool

508 // If set, this (sub)thread is collapsed in the webmail client, for threading mode

509 // "on" (mode "unread" ignores it). This field is copied to new child message.

510 // Changes are propagated to the webmail client.

511 ThreadCollapsed bool

512

513 // If received message was known to match a mailing list rule (with modified junk

514 // filtering).

515 IsMailingList bool

516

517 // If this message is a DSN, generated by us or received. For DSNs, we don't look

518 // at the subject when matching threads.

519 DSN bool

520

521 ReceivedTLSVersion uint16 // 0 if unknown, 1 if plaintext/no TLS, otherwise TLS cipher suite.

522 ReceivedTLSCipherSuite uint16

523 ReceivedRequireTLS bool // Whether RequireTLS was known to be used for incoming delivery.

524

525 Flags

526 // For keywords other than system flags or the basic well-known $-flags. Only in

527 // "atom" syntax (IMAP), they are case-insensitive, always stored in lower-case

528 // (for JMAP), sorted.

529 Keywords []string `bstore:"index"`

530 Size int64

531 TrainedJunk *bool // If nil, no training done yet. Otherwise, true is trained as junk, false trained as nonjunk.

532 MsgPrefix []byte // Typically holds received headers and/or header separator.

533

534 // ParsedBuf message structure. Currently saved as JSON of message.Part because bstore

535 // cannot yet store recursive types. Created when first needed, and saved in the

536 // database.

537 // todo: once replaced with non-json storage, remove date fixup in ../message/part.go.

538 ParsedBuf []byte

539}

540

541// MailboxCounts returns the delta to counts this message means for its

542// mailbox.

543func (m Message) MailboxCounts() (mc MailboxCounts) {

544 if m.Expunged {

545 return

546 }

547 if m.Deleted {

548 mc.Deleted++

549 } else {

550 mc.Total++

551 }

552 if !m.Seen {

553 mc.Unseen++

554 if !m.Deleted {

555 mc.Unread++

556 }

557 }

558 mc.Size += m.Size

559 return

560}

561

562func (m Message) ChangeAddUID() ChangeAddUID {

563 return ChangeAddUID{m.MailboxID, m.UID, m.ModSeq, m.Flags, m.Keywords}

564}

565

566func (m Message) ChangeFlags(orig Flags) ChangeFlags {

567 mask := m.Flags.Changed(orig)

568 return ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, ModSeq: m.ModSeq, Mask: mask, Flags: m.Flags, Keywords: m.Keywords}

569}

570

571func (m Message) ChangeThread() ChangeThread {

572 return ChangeThread{[]int64{m.ID}, m.ThreadMuted, m.ThreadCollapsed}

573}

574

575// ModSeq represents a modseq as stored in the database. ModSeq 0 in the

576// database is sent to the client as 1, because modseq 0 is special in IMAP.

577// ModSeq coming from the client are of type int64.

578type ModSeq int64

579

580func (ms ModSeq) Client() int64 {

581 if ms == 0 {

582 return 1

583 }

584 return int64(ms)

585}

586

587// ModSeqFromClient converts a modseq from a client to a modseq for internal

588// use, e.g. in a database query.

589// ModSeq 1 is turned into 0 (the Go zero value for ModSeq).

590func ModSeqFromClient(modseq int64) ModSeq {

591 if modseq == 1 {

592 return 0

593 }

594 return ModSeq(modseq)

595}

596

597// PrepareExpunge clears fields that are no longer needed after an expunge, so

598// almost all fields. Does not change ModSeq, but does set Expunged.

599func (m *Message) PrepareExpunge() {

600 *m = Message{

601 ID: m.ID,

602 UID: m.UID,

603 MailboxID: m.MailboxID,

604 CreateSeq: m.CreateSeq,

605 ModSeq: m.ModSeq,

606 Expunged: true,

607 ThreadID: m.ThreadID,

608 }

609}

610

611// PrepareThreading sets MessageID, SubjectBase and DSN (used in threading) based

612// on the part.

613func (m *Message) PrepareThreading(log mlog.Log, part *message.Part) {

614 m.DSN = part.IsDSN()

615

616 if part.Envelope == nil {

617 return

618 }

619 messageID, raw, err := message.MessageIDCanonical(part.Envelope.MessageID)

620 if err != nil {

621 log.Debugx("parsing message-id, ignoring", err, slog.String("messageid", part.Envelope.MessageID))

622 } else if raw {

623 log.Debug("could not parse message-id as address, continuing with raw value", slog.String("messageid", part.Envelope.MessageID))

624 }

625 m.MessageID = messageID

626 m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false)

627}

628

629// LoadPart returns a message.Part by reading from m.ParsedBuf.

630func (m Message) LoadPart(r io.ReaderAt) (message.Part, error) {

631 if m.ParsedBuf == nil {

632 return message.Part{}, fmt.Errorf("message not parsed")

633 }

634 var p message.Part

635 err := json.Unmarshal(m.ParsedBuf, &p)

636 if err != nil {

637 return p, fmt.Errorf("unmarshal message part")

638 }

639 p.SetReaderAt(r)

640 return p, nil

641}

642

643// NeedsTraining returns whether message needs a training update, based on

644// TrainedJunk (current training status) and new Junk/Notjunk flags.

645func (m Message) NeedsTraining() bool {

646 untrain := m.TrainedJunk != nil

647 untrainJunk := untrain && *m.TrainedJunk

648 train := m.Junk || m.Notjunk && !(m.Junk && m.Notjunk)

649 trainJunk := m.Junk

650 return untrain != train || untrain && train && untrainJunk != trainJunk

651}

652

653// JunkFlagsForMailbox sets Junk and Notjunk flags based on mailbox name if configured. Often

654// used when delivering/moving/copying messages to a mailbox. Mail clients are not

655// very helpful with setting junk/notjunk flags. But clients can move/copy messages

656// to other mailboxes. So we set flags when clients move a message.

657func (m *Message) JunkFlagsForMailbox(mb Mailbox, conf config.Account) {

658 if mb.Junk {

659 m.Junk = true

660 m.Notjunk = false

661 return

662 }

663

664 if !conf.AutomaticJunkFlags.Enabled {

665 return

666 }

667

668 lmailbox := strings.ToLower(mb.Name)

669

670 if conf.JunkMailbox != nil && conf.JunkMailbox.MatchString(lmailbox) {

671 m.Junk = true

672 m.Notjunk = false

673 } else if conf.NeutralMailbox != nil && conf.NeutralMailbox.MatchString(lmailbox) {

674 m.Junk = false

675 m.Notjunk = false

676 } else if conf.NotJunkMailbox != nil && conf.NotJunkMailbox.MatchString(lmailbox) {

677 m.Junk = false

678 m.Notjunk = true

679 } else if conf.JunkMailbox == nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox != nil {

680 m.Junk = true

681 m.Notjunk = false

682 } else if conf.JunkMailbox != nil && conf.NeutralMailbox == nil && conf.NotJunkMailbox != nil {

683 m.Junk = false

684 m.Notjunk = false

685 } else if conf.JunkMailbox != nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox == nil {

686 m.Junk = false

687 m.Notjunk = true

688 }

689}

690

691// Recipient represents the recipient of a message. It is tracked to allow

692// first-time incoming replies from users this account has sent messages to. When a

693// mailbox is added to the Sent mailbox the message is parsed and recipients are

694// inserted as recipient. Recipients are never removed other than for removing the

695// message. On move/copy of a message, recipients aren't modified either. For IMAP,

696// this assumes a client simply appends messages to the Sent mailbox (as opposed to

697// copying messages from some place).

698type Recipient struct {

699 ID int64

700 MessageID int64 `bstore:"nonzero,ref Message"` // Ref gives it its own index, useful for fast removal as well.

701 Localpart string `bstore:"nonzero"` // Encoded localpart.

702 Domain string `bstore:"nonzero,index Domain+Localpart"` // Unicode string.

703 OrgDomain string `bstore:"nonzero,index"` // Unicode string.

704 Sent time.Time `bstore:"nonzero"`

705}

706

707// Outgoing is a message submitted for delivery from the queue. Used to enforce

708// maximum outgoing messages.

709type Outgoing struct {

710 ID int64

711 Recipient string `bstore:"nonzero,index"` // Canonical international address with utf8 domain.

712 Submitted time.Time `bstore:"nonzero,default now"`

713}

714

715// RecipientDomainTLS stores TLS capabilities of a recipient domain as encountered

716// during most recent connection (delivery attempt).

717type RecipientDomainTLS struct {

718 Domain string // Unicode.

719 Updated time.Time `bstore:"default now"`

720 STARTTLS bool // Supports STARTTLS.

721 RequireTLS bool // Supports RequireTLS SMTP extension.

722}

723

724// DiskUsage tracks quota use.

725type DiskUsage struct {

726 ID int64 // Always one record with ID 1.

727 MessageSize int64 // Sum of all messages, for quota accounting.

728}

729

730// SessionToken and CSRFToken are types to prevent mixing them up.

731// Base64 raw url encoded.

732type SessionToken string

733type CSRFToken string

734

735// LoginSession represents a login session. We keep a limited number of sessions

736// for a user, removing the oldest session when a new one is created.

737type LoginSession struct {

738 ID int64

739 Created time.Time `bstore:"nonzero,default now"` // Of original login.

740 Expires time.Time `bstore:"nonzero"` // Extended each time it is used.

741 SessionTokenBinary [16]byte `bstore:"nonzero"` // Stored in cookie, like "webmailsession" or "webaccountsession".

742 CSRFTokenBinary [16]byte // For API requests, in "x-mox-csrf" header.

743 AccountName string `bstore:"nonzero"`

744 LoginAddress string `bstore:"nonzero"`

745

746 // Set when loading from database.

747 sessionToken SessionToken

748 csrfToken CSRFToken

749}

750

751// Quoting is a setting for how to quote in replies/forwards.

752type Quoting string

753

754const (

755 Default Quoting = "" // Bottom-quote if text is selected, top-quote otherwise.

756 Bottom Quoting = "bottom"

757 Top Quoting = "top"

758)

759

760// Settings are webmail client settings.

761type Settings struct {

762 ID uint8 // Singleton ID 1.

763

764 Signature string

765 Quoting Quoting

766

767 // Whether to show the bars underneath the address input fields indicating

768 // starttls/dnssec/dane/mtasts/requiretls support by address.

769 ShowAddressSecurity bool

770

771 // Show HTML version of message by default, instead of plain text.

772 ShowHTML bool

773

774 // If true, don't show shortcuts in webmail after mouse interaction.

775 NoShowShortcuts bool

776

777 // Additional headers to display in message view. E.g. Delivered-To, User-Agent, X-Mox-Reason.

778 ShowHeaders []string

779}

780

781// ViewMode how a message should be viewed: its text parts, html parts, or html

782// with loading external resources.

783type ViewMode string

784

785const (

786 ModeText ViewMode = "text"

787 ModeHTML ViewMode = "html"

788 ModeHTMLExt ViewMode = "htmlext" // HTML with external resources.

789)

790

791// FromAddressSettings are webmail client settings per "From" address.

792type FromAddressSettings struct {

793 FromAddress string // Unicode.

794 ViewMode ViewMode

795}

796

797// RulesetNoListID records a user "no" response to the question of

798// creating/removing a ruleset after moving a message with list-id header from/to

799// the inbox.

800type RulesetNoListID struct {

801 ID int64

802 RcptToAddress string `bstore:"nonzero"`

803 ListID string `bstore:"nonzero"`

804 ToInbox bool // Otherwise from Inbox to other mailbox.

805}

806

807// RulesetNoMsgFrom records a user "no" response to the question of

808// creating/moveing a ruleset after moving a mesage with message "from" address

809// from/to the inbox.

810type RulesetNoMsgFrom struct {

811 ID int64

812 RcptToAddress string `bstore:"nonzero"`

813 MsgFromAddress string `bstore:"nonzero"` // Unicode.

814 ToInbox bool // Otherwise from Inbox to other mailbox.

815}

816

817// RulesetNoMailbox represents a "never from/to this mailbox" response to the

818// question of adding/removing a ruleset after moving a message.

819type RulesetNoMailbox struct {

820 ID int64

821

822 // The mailbox from/to which the move has happened.

823 // Not a references, if mailbox is deleted, an entry becomes ineffective.

824 MailboxID int64 `bstore:"nonzero"`

825 ToMailbox bool // Whether MailboxID is the destination of the move (instead of source).

826}

827

828// Types stored in DB.

829var DBTypes = []any{

830 NextUIDValidity{},

831 Message{},

832 Recipient{},

833 Mailbox{},

834 Subscription{},

835 Outgoing{},

836 Password{},

837 Subjectpass{},

838 SyncState{},

839 Upgrade{},

840 RecipientDomainTLS{},

841 DiskUsage{},

842 LoginSession{},

843 Settings{},

844 FromAddressSettings{},

845 RulesetNoListID{},

846 RulesetNoMsgFrom{},

847 RulesetNoMailbox{},

848 Annotation{},

849}

850

851// Account holds the information about a user, includings mailboxes, messages, imap subscriptions.

852type Account struct {

853 Name string // Name, according to configuration.

854 Dir string // Directory where account files, including the database, bloom filter, and mail messages, are stored for this account.

855 DBPath string // Path to database with mailboxes, messages, etc.

856 DB *bstore.DB // Open database connection.

857

858 // Channel that is closed if/when account has/gets "threads" accounting (see

859 // Upgrade.Threads).

860 threadsCompleted chan struct{}

861 // If threads upgrade completed with error, this is set. Used for warning during

862 // delivery, or aborting when importing.

863 threadsErr error

864

865 // Write lock must be held for account/mailbox modifications including message delivery.

866 // Read lock for reading mailboxes/messages.

867 // When making changes to mailboxes/messages, changes must be broadcasted before

868 // releasing the lock to ensure proper UID ordering.

869 sync.RWMutex

870

871 nused int // Reference count, while >0, this account is alive and shared.

872}

873

874type Upgrade struct {

875 ID byte

876 Threads byte // 0: None, 1: Adding MessageID's completed, 2: Adding ThreadID's completed.

877}

878

879// InitialUIDValidity returns a UIDValidity used for initializing an account.

880// It can be replaced during tests with a predictable value.

881var InitialUIDValidity = func() uint32 {

882 return uint32(time.Now().Unix() >> 1) // A 2-second resolution will get us far enough beyond 2038.

883}

884

885var openAccounts = struct {

886 names map[string]*Account

887 sync.Mutex

888}{

889 names: map[string]*Account{},

890}

891

892func closeAccount(acc *Account) (rerr error) {

893 openAccounts.Lock()

894 acc.nused--

895 defer openAccounts.Unlock()

896 if acc.nused == 0 {

897 // threadsCompleted must be closed now because it increased nused.

898 rerr = acc.DB.Close()

899 acc.DB = nil

900 delete(openAccounts.names, acc.Name)

901 }

902 return

903}

904

905// OpenAccount opens an account by name.

906//

907// No additional data path prefix or ".db" suffix should be added to the name.

908// A single shared account exists per name.

909func OpenAccount(log mlog.Log, name string, checkLoginDisabled bool) (*Account, error) {

910 openAccounts.Lock()

911 defer openAccounts.Unlock()

912 if acc, ok := openAccounts.names[name]; ok {

913 acc.nused++

914 return acc, nil

915 }

916

917 if a, ok := mox.Conf.Account(name); !ok {

918 return nil, ErrAccountUnknown

919 } else if checkLoginDisabled && a.LoginDisabled != "" {

920 return nil, fmt.Errorf("%w: %s", ErrLoginDisabled, a.LoginDisabled)

921 }

922

923 acc, err := openAccount(log, name)

924 if err != nil {

925 return nil, err

926 }

927 openAccounts.names[name] = acc

928 return acc, nil

929}

930

931// openAccount opens an existing account, or creates it if it is missing.

932func openAccount(log mlog.Log, name string) (a *Account, rerr error) {

933 dir := filepath.Join(mox.DataDirPath("accounts"), name)

934 return OpenAccountDB(log, dir, name)

935}

936

937// OpenAccountDB opens an account database file and returns an initialized account

938// or error. Only exported for use by subcommands that verify the database file.

939// Almost all account opens must go through OpenAccount/OpenEmail/OpenEmailAuth.

940func OpenAccountDB(log mlog.Log, accountDir, accountName string) (a *Account, rerr error) {

941 dbpath := filepath.Join(accountDir, "index.db")

942

943 // Create account if it doesn't exist yet.

944 isNew := false

945 if _, err := os.Stat(dbpath); err != nil && os.IsNotExist(err) {

946 isNew = true

947 os.MkdirAll(accountDir, 0770)

948 }

949

950 opts := bstore.Options{Timeout: 5 * time.Second, Perm: 0660, RegisterLogger: moxvar.RegisterLogger(dbpath, log.Logger)}

951 db, err := bstore.Open(context.TODO(), dbpath, &opts, DBTypes...)

952 if err != nil {

953 return nil, err

954 }

955

956 defer func() {

957 if rerr != nil {

958 db.Close()

959 if isNew {

960 os.Remove(dbpath)

961 }

962 }

963 }()

964

965 acc := &Account{

966 Name: accountName,

967 Dir: accountDir,

968 DBPath: dbpath,

969 DB: db,

970 nused: 1,

971 threadsCompleted: make(chan struct{}),

972 }

973

974 if isNew {

975 if err := initAccount(db); err != nil {

976 return nil, fmt.Errorf("initializing account: %v", err)

977 }

978 close(acc.threadsCompleted)

979 return acc, nil

980 }

981

982 // Ensure singletons are present. Mailbox counts and total message size, Settings.

983 var mentioned bool

984 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

985 if tx.Get(&Settings{ID: 1}) == bstore.ErrAbsent {

986 if err := tx.Insert(&Settings{ID: 1, ShowAddressSecurity: true}); err != nil {

987 return err

988 }

989 }

990

991 err := bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error {

992 if !mentioned {

993 mentioned = true

994 log.Info("first calculation of mailbox counts for account", slog.String("account", accountName))

995 }

996 mc, err := mb.CalculateCounts(tx)

997 if err != nil {

998 return err

999 }

1000 mb.HaveCounts = true

1001 mb.MailboxCounts = mc

1002 return tx.Update(&mb)

1003 })

1004 if err != nil {

1005 return err

1006 }

1007

1008 du := DiskUsage{ID: 1}

1009 err = tx.Get(&du)

1010 if err == nil || !errors.Is(err, bstore.ErrAbsent) {

1011 return err

1012 }

1013 // No DiskUsage record yet, calculate total size and insert.

1014 err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {

1015 du.MessageSize += mb.Size

1016 return nil

1017 })

1018 if err != nil {

1019 return err

1020 }

1021 return tx.Insert(&du)

1022 })

1023 if err != nil {

1024 return nil, fmt.Errorf("calculating counts for mailbox or inserting settings: %v", err)

1025 }

1026

1027 // Start adding threading if needed.

1028 up := Upgrade{ID: 1}

1029 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

1030 err := tx.Get(&up)

1031 if err == bstore.ErrAbsent {

1032 if err := tx.Insert(&up); err != nil {

1033 return fmt.Errorf("inserting initial upgrade record: %v", err)

1034 }

1035 err = nil

1036 }

1037 return err

1038 })

1039 if err != nil {

1040 return nil, fmt.Errorf("checking message threading: %v", err)

1041 }

1042 if up.Threads == 2 {

1043 close(acc.threadsCompleted)

1044 return acc, nil

1045 }

1046

1047 // Increase account use before holding on to account in background.

1048 // Caller holds the lock. The goroutine below decreases nused by calling

1049 // closeAccount.

1050 acc.nused++

1051

1052 // Ensure all messages have a MessageID and SubjectBase, which are needed when

1053 // matching threads.

1054 // Then assign messages to threads, in the same way we do during imports.

1055 log.Info("upgrading account for threading, in background", slog.String("account", acc.Name))

1056 go func() {

1057 defer func() {

1058 err := closeAccount(acc)

1059 log.Check(err, "closing use of account after upgrading account storage for threads", slog.String("account", a.Name))

1060

1061 // Mark that upgrade has finished, possibly error is indicated in threadsErr.

1062 close(acc.threadsCompleted)

1063 }()

1064

1065 defer func() {

1066 x := recover() // Should not happen, but don't take program down if it does.

1067 if x != nil {

1068 log.Error("upgradeThreads panic", slog.Any("err", x))

1069 debug.PrintStack()

1070 metrics.PanicInc(metrics.Upgradethreads)

1071 acc.threadsErr = fmt.Errorf("panic during upgradeThreads: %v", x)

1072 }

1073 }()

1074

1075 err := upgradeThreads(mox.Shutdown, log, acc, &up)

1076 if err != nil {

1077 a.threadsErr = err

1078 log.Errorx("upgrading account for threading, aborted", err, slog.String("account", a.Name))

1079 } else {

1080 log.Info("upgrading account for threading, completed", slog.String("account", a.Name))

1081 }

1082 }()

1083 return acc, nil

1084}

1085

1086// ThreadingWait blocks until the one-time account threading upgrade for the

1087// account has completed, and returns an error if not successful.

1088//

1089// To be used before starting an import of messages.

1090func (a *Account) ThreadingWait(log mlog.Log) error {

1091 select {

1092 case <-a.threadsCompleted:

1093 return a.threadsErr

1094 default:

1095 }

1096 log.Debug("waiting for account upgrade to complete")

1097

1098 <-a.threadsCompleted

1099 return a.threadsErr

1100}

1101

1102func initAccount(db *bstore.DB) error {

1103 return db.Write(context.TODO(), func(tx *bstore.Tx) error {

1104 uidvalidity := InitialUIDValidity()

1105

1106 if err := tx.Insert(&Upgrade{ID: 1, Threads: 2}); err != nil {

1107 return err

1108 }

1109 if err := tx.Insert(&DiskUsage{ID: 1}); err != nil {

1110 return err

1111 }

1112 if err := tx.Insert(&Settings{ID: 1}); err != nil {

1113 return err

1114 }

1115

1116 if len(mox.Conf.Static.DefaultMailboxes) > 0 {

1117 // Deprecated in favor of InitialMailboxes.

1118 defaultMailboxes := mox.Conf.Static.DefaultMailboxes

1119 mailboxes := []string{"Inbox"}

1120 for _, name := range defaultMailboxes {

1121 if strings.EqualFold(name, "Inbox") {

1122 continue

1123 }

1124 mailboxes = append(mailboxes, name)

1125 }

1126 for _, name := range mailboxes {

1127 mb := Mailbox{Name: name, UIDValidity: uidvalidity, UIDNext: 1, HaveCounts: true}

1128 if strings.HasPrefix(name, "Archive") {

1129 mb.Archive = true

1130 } else if strings.HasPrefix(name, "Drafts") {

1131 mb.Draft = true

1132 } else if strings.HasPrefix(name, "Junk") {

1133 mb.Junk = true

1134 } else if strings.HasPrefix(name, "Sent") {

1135 mb.Sent = true

1136 } else if strings.HasPrefix(name, "Trash") {

1137 mb.Trash = true

1138 }

1139 if err := tx.Insert(&mb); err != nil {

1140 return fmt.Errorf("creating mailbox: %w", err)

1141 }

1142 if err := tx.Insert(&Subscription{name}); err != nil {

1143 return fmt.Errorf("adding subscription: %w", err)

1144 }

1145 }

1146 } else {

1147 mailboxes := mox.Conf.Static.InitialMailboxes

1148 var zerouse config.SpecialUseMailboxes

1149 if mailboxes.SpecialUse == zerouse && len(mailboxes.Regular) == 0 {

1150 mailboxes = DefaultInitialMailboxes

1151 }

1152

1153 add := func(name string, use SpecialUse) error {

1154 mb := Mailbox{Name: name, UIDValidity: uidvalidity, UIDNext: 1, SpecialUse: use, HaveCounts: true}

1155 if err := tx.Insert(&mb); err != nil {

1156 return fmt.Errorf("creating mailbox: %w", err)

1157 }

1158 if err := tx.Insert(&Subscription{name}); err != nil {

1159 return fmt.Errorf("adding subscription: %w", err)

1160 }

1161 return nil

1162 }

1163 addSpecialOpt := func(nameOpt string, use SpecialUse) error {

1164 if nameOpt == "" {

1165 return nil

1166 }

1167 return add(nameOpt, use)

1168 }

1169 l := []struct {

1170 nameOpt string

1171 use SpecialUse

1172 }{

1173 {"Inbox", SpecialUse{}},

1174 {mailboxes.SpecialUse.Archive, SpecialUse{Archive: true}},

1175 {mailboxes.SpecialUse.Draft, SpecialUse{Draft: true}},

1176 {mailboxes.SpecialUse.Junk, SpecialUse{Junk: true}},

1177 {mailboxes.SpecialUse.Sent, SpecialUse{Sent: true}},

1178 {mailboxes.SpecialUse.Trash, SpecialUse{Trash: true}},

1179 }

1180 for _, e := range l {

1181 if err := addSpecialOpt(e.nameOpt, e.use); err != nil {

1182 return err

1183 }

1184 }

1185 for _, name := range mailboxes.Regular {

1186 if err := add(name, SpecialUse{}); err != nil {

1187 return err

1188 }

1189 }

1190 }

1191

1192 uidvalidity++

1193 if err := tx.Insert(&NextUIDValidity{1, uidvalidity}); err != nil {

1194 return fmt.Errorf("inserting nextuidvalidity: %w", err)

1195 }

1196 return nil

1197 })

1198}

1199

1200// CheckClosed asserts that the account has a zero reference count. For use in tests.

1201func (a *Account) CheckClosed() {

1202 openAccounts.Lock()

1203 defer openAccounts.Unlock()

1204 if a.nused != 0 {

1205 panic(fmt.Sprintf("account still in use, %d refs", a.nused))

1206 }

1207}

1208

1209// Close reduces the reference count, and closes the database connection when

1210// it was the last user.

1211func (a *Account) Close() error {

1212 if CheckConsistencyOnClose {

1213 xerr := a.CheckConsistency()

1214 err := closeAccount(a)

1215 if xerr != nil {

1216 panic(xerr)

1217 }

1218 return err

1219 }

1220 return closeAccount(a)

1221}

1222

1223// CheckConsistency checks the consistency of the database and returns a non-nil

1224// error for these cases:

1225//

1226// - Missing on-disk file for message.

1227// - Mismatch between message size and length of MsgPrefix and on-disk file.

1228// - Missing HaveCounts.

1229// - Incorrect mailbox counts.

1230// - Incorrect total message size.

1231// - Message with UID >= mailbox uid next.

1232// - Mailbox uidvalidity >= account uid validity.

1233// - ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq.

1234// - All messages have a nonzero ThreadID, and no cycles in ThreadParentID, and parent messages the same ThreadParentIDs tail.

1235func (a *Account) CheckConsistency() error {

1236 var uidErrors []string // With a limit, could be many.

1237 var modseqErrors []string // With limit.

1238 var fileErrors []string // With limit.

1239 var threadidErrors []string // With limit.

1240 var threadParentErrors []string // With limit.

1241 var threadAncestorErrors []string // With limit.

1242 var errors []string

1243

1244 err := a.DB.Read(context.Background(), func(tx *bstore.Tx) error {

1245 nuv := NextUIDValidity{ID: 1}

1246 err := tx.Get(&nuv)

1247 if err != nil {

1248 return fmt.Errorf("fetching next uid validity: %v", err)

1249 }

1250

1251 mailboxes := map[int64]Mailbox{}

1252 err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {

1253 mailboxes[mb.ID] = mb

1254

1255 if mb.UIDValidity >= nuv.Next {

1256 errmsg := fmt.Sprintf("mailbox %q (id %d) has uidvalidity %d >= account next uidvalidity %d", mb.Name, mb.ID, mb.UIDValidity, nuv.Next)

1257 errors = append(errors, errmsg)

1258 }

1259 return nil

1260 })

1261 if err != nil {

1262 return fmt.Errorf("listing mailboxes: %v", err)

1263 }

1264

1265 counts := map[int64]MailboxCounts{}

1266 err = bstore.QueryTx[Message](tx).ForEach(func(m Message) error {

1267 mc := counts[m.MailboxID]

1268 mc.Add(m.MailboxCounts())

1269 counts[m.MailboxID] = mc

1270

1271 mb := mailboxes[m.MailboxID]

1272

1273 if (m.ModSeq == 0 || m.CreateSeq == 0 || m.CreateSeq > m.ModSeq) && len(modseqErrors) < 20 {

1274 modseqerr := fmt.Sprintf("message %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and createseq <= modseq", m.ID, mb.Name, mb.ID, m.ModSeq, m.CreateSeq)

1275 modseqErrors = append(modseqErrors, modseqerr)

1276 }

1277 if m.UID >= mb.UIDNext && len(uidErrors) < 20 {

1278 uiderr := fmt.Sprintf("message %d in mailbox %q (id %d) has uid %d >= mailbox uidnext %d", m.ID, mb.Name, mb.ID, m.UID, mb.UIDNext)

1279 uidErrors = append(uidErrors, uiderr)

1280 }

1281 if m.Expunged {

1282 return nil

1283 }

1284 p := a.MessagePath(m.ID)

1285 st, err := os.Stat(p)

1286 if err != nil {

1287 existserr := fmt.Sprintf("message %d in mailbox %q (id %d) on-disk file %s: %v", m.ID, mb.Name, mb.ID, p, err)

1288 fileErrors = append(fileErrors, existserr)

1289 } else if len(fileErrors) < 20 && m.Size != int64(len(m.MsgPrefix))+st.Size() {

1290 sizeerr := fmt.Sprintf("message %d in mailbox %q (id %d) has size %d != len msgprefix %d + on-disk file size %d = %d", m.ID, mb.Name, mb.ID, m.Size, len(m.MsgPrefix), st.Size(), int64(len(m.MsgPrefix))+st.Size())

1291 fileErrors = append(fileErrors, sizeerr)

1292 }

1293

1294 if m.ThreadID <= 0 && len(threadidErrors) < 20 {

1295 err := fmt.Sprintf("message %d in mailbox %q (id %d) has threadid 0", m.ID, mb.Name, mb.ID)

1296 threadidErrors = append(threadidErrors, err)

1297 }

1298 if slices.Contains(m.ThreadParentIDs, m.ID) && len(threadParentErrors) < 20 {

1299 err := fmt.Sprintf("message %d in mailbox %q (id %d) references itself in threadparentids", m.ID, mb.Name, mb.ID)

1300 threadParentErrors = append(threadParentErrors, err)

1301 }

1302 for i, pid := range m.ThreadParentIDs {

1303 am := Message{ID: pid}

1304 if err := tx.Get(&am); err == bstore.ErrAbsent {

1305 continue

1306 } else if err != nil {

1307 return fmt.Errorf("get ancestor message: %v", err)

1308 } else if !slices.Equal(m.ThreadParentIDs[i+1:], am.ThreadParentIDs) && len(threadAncestorErrors) < 20 {

1309 err := fmt.Sprintf("message %d, thread %d has ancestor ids %v, and ancestor at index %d with id %d should have the same tail but has %v\n", m.ID, m.ThreadID, m.ThreadParentIDs, i, am.ID, am.ThreadParentIDs)

1310 threadAncestorErrors = append(threadAncestorErrors, err)

1311 } else {

1312 break

1313 }

1314 }

1315 return nil

1316 })

1317 if err != nil {

1318 return fmt.Errorf("reading messages: %v", err)

1319 }

1320

1321 var totalSize int64

1322 for _, mb := range mailboxes {

1323 totalSize += mb.Size

1324 if !mb.HaveCounts {

1325 errmsg := fmt.Sprintf("mailbox %q (id %d) does not have counts, should be %#v", mb.Name, mb.ID, counts[mb.ID])

1326 errors = append(errors, errmsg)

1327 } else if mb.MailboxCounts != counts[mb.ID] {

1328 mbcounterr := fmt.Sprintf("mailbox %q (id %d) has wrong counts %s, should be %s", mb.Name, mb.ID, mb.MailboxCounts, counts[mb.ID])

1329 errors = append(errors, mbcounterr)

1330 }

1331 }

1332

1333 du := DiskUsage{ID: 1}

1334 if err := tx.Get(&du); err != nil {

1335 return fmt.Errorf("get diskusage")

1336 }

1337 if du.MessageSize != totalSize {

1338 errmsg := fmt.Sprintf("total message size in database is %d, sum of mailbox message sizes is %d", du.MessageSize, totalSize)

1339 errors = append(errors, errmsg)

1340 }

1341

1342 return nil

1343 })

1344 if err != nil {

1345 return err

1346 }

1347 errors = append(errors, uidErrors...)

1348 errors = append(errors, modseqErrors...)

1349 errors = append(errors, fileErrors...)

1350 errors = append(errors, threadidErrors...)

1351 errors = append(errors, threadParentErrors...)

1352 errors = append(errors, threadAncestorErrors...)

1353 if len(errors) > 0 {

1354 return fmt.Errorf("%s", strings.Join(errors, "; "))

1355 }

1356 return nil

1357}

1358

1359// Conf returns the configuration for this account if it still exists. During

1360// an SMTP session, a configuration update may drop an account.

1361func (a *Account) Conf() (config.Account, bool) {

1362 return mox.Conf.Account(a.Name)

1363}

1364

1365// NextUIDValidity returns the next new/unique uidvalidity to use for this account.

1366func (a *Account) NextUIDValidity(tx *bstore.Tx) (uint32, error) {

1367 nuv := NextUIDValidity{ID: 1}

1368 if err := tx.Get(&nuv); err != nil {

1369 return 0, err

1370 }

1371 v := nuv.Next

1372 nuv.Next++

1373 if err := tx.Update(&nuv); err != nil {

1374 return 0, err

1375 }

1376 return v, nil

1377}

1378

1379// NextModSeq returns the next modification sequence, which is global per account,

1380// over all types.

1381func (a *Account) NextModSeq(tx *bstore.Tx) (ModSeq, error) {

1382 v := SyncState{ID: 1}

1383 if err := tx.Get(&v); err == bstore.ErrAbsent {

1384 // We start assigning from modseq 2. Modseq 0 is not usable, so returned as 1, so

1385 // already used.

1386 // HighestDeletedModSeq is -1 so comparison against the default ModSeq zero value

1387 // makes sense.

1388 v = SyncState{1, 2, -1}

1389 return v.LastModSeq, tx.Insert(&v)

1390 } else if err != nil {

1391 return 0, err

1392 }

1393 v.LastModSeq++

1394 return v.LastModSeq, tx.Update(&v)

1395}

1396

1397func (a *Account) HighestDeletedModSeq(tx *bstore.Tx) (ModSeq, error) {

1398 v := SyncState{ID: 1}

1399 err := tx.Get(&v)

1400 if err == bstore.ErrAbsent {

1401 return 0, nil

1402 }

1403 return v.HighestDeletedModSeq, err

1404}

1405

1406// WithWLock runs fn with account writelock held. Necessary for account/mailbox

1407// modification. For message delivery, a read lock is required.

1408func (a *Account) WithWLock(fn func()) {

1409 a.Lock()

1410 defer a.Unlock()

1411 fn()

1412}

1413

1414// WithRLock runs fn with account read lock held. Needed for message delivery.

1415func (a *Account) WithRLock(fn func()) {

1416 a.RLock()

1417 defer a.RUnlock()

1418 fn()

1419}

1420

1421// DeliverMessage delivers a mail message to the account.

1422//

1423// The message, with msg.MsgPrefix and msgFile combined, must have a header

1424// section. The caller is responsible for adding a header separator to

1425// msg.MsgPrefix if missing from an incoming message.

1426//

1427// If the destination mailbox has the Sent special-use flag, the message is parsed

1428// for its recipients (to/cc/bcc). Their domains are added to Recipients for use in

1429// dmarc reputation.

1430//

1431// If sync is true, the message file and its directory are synced. Should be true

1432// for regular mail delivery, but can be false when importing many messages.

1433//

1434// If updateDiskUsage is true, the account total message size (for quota) is

1435// updated. Callers must check if a message can be added within quota before

1436// calling DeliverMessage.

1437//

1438// If CreateSeq/ModSeq is not set, it is assigned automatically.

1439//

1440// Must be called with account rlock or wlock.

1441//

1442// Caller must broadcast new message.

1443//

1444// Caller must update mailbox counts.

1445func (a *Account) DeliverMessage(log mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, sync, notrain, nothreads, updateDiskUsage bool) error {

1446 if m.Expunged {

1447 return fmt.Errorf("cannot deliver expunged message")

1448 }

1449

1450 mb := Mailbox{ID: m.MailboxID}

1451 if err := tx.Get(&mb); err != nil {

1452 return fmt.Errorf("get mailbox: %w", err)

1453 }

1454 m.UID = mb.UIDNext

1455 mb.UIDNext++

1456 if err := tx.Update(&mb); err != nil {

1457 return fmt.Errorf("updating mailbox nextuid: %w", err)

1458 }

1459

1460 if updateDiskUsage {

1461 du := DiskUsage{ID: 1}

1462 if err := tx.Get(&du); err != nil {

1463 return fmt.Errorf("get disk usage: %v", err)

1464 }

1465 du.MessageSize += m.Size

1466 if err := tx.Update(&du); err != nil {

1467 return fmt.Errorf("update disk usage: %v", err)

1468 }

1469 }

1470

1471 conf, _ := a.Conf()

1472 m.JunkFlagsForMailbox(mb, conf)

1473

1474 mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile.

1475 var part *message.Part

1476 if m.ParsedBuf == nil {

1477 p, err := message.EnsurePart(log.Logger, false, mr, m.Size)

1478 if err != nil {

1479 log.Infox("parsing delivered message", err, slog.String("parse", ""), slog.Int64("message", m.ID))

1480 // We continue, p is still valid.

1481 }

1482 part = &p

1483 buf, err := json.Marshal(part)

1484 if err != nil {

1485 return fmt.Errorf("marshal parsed message: %w", err)

1486 }

1487 m.ParsedBuf = buf

1488 } else {

1489 var p message.Part

1490 if err := json.Unmarshal(m.ParsedBuf, &p); err != nil {

1491 log.Errorx("unmarshal parsed message, continuing", err, slog.String("parse", ""))

1492 } else {

1493 part = &p

1494 }

1495 }

1496

1497 // If we are delivering to the originally intended mailbox, no need to store the mailbox ID again.

1498 if m.MailboxDestinedID != 0 && m.MailboxDestinedID == m.MailboxOrigID {

1499 m.MailboxDestinedID = 0

1500 }

1501 if m.CreateSeq == 0 || m.ModSeq == 0 {

1502 modseq, err := a.NextModSeq(tx)

1503 if err != nil {

1504 return fmt.Errorf("assigning next modseq: %w", err)

1505 }

1506 m.CreateSeq = modseq

1507 m.ModSeq = modseq

1508 }

1509

1510 if part != nil && m.MessageID == "" && m.SubjectBase == "" {

1511 m.PrepareThreading(log, part)

1512 }

1513

1514 // Assign to thread (if upgrade has completed).

1515 noThreadID := nothreads

1516 if m.ThreadID == 0 && !nothreads && part != nil {

1517 select {

1518 case <-a.threadsCompleted:

1519 if a.threadsErr != nil {

1520 log.Info("not assigning threads for new delivery, upgrading to threads failed")

1521 noThreadID = true

1522 } else {

1523 if err := assignThread(log, tx, m, part); err != nil {

1524 return fmt.Errorf("assigning thread: %w", err)

1525 }

1526 }

1527 default:

1528 // note: since we have a write transaction to get here, we can't wait for the

1529 // thread upgrade to finish.

1530 // If we don't assign a threadid the upgrade process will do it.

1531 log.Info("not assigning threads for new delivery, upgrading to threads in progress which will assign this message")

1532 noThreadID = true

1533 }

1534 }

1535

1536 if err := tx.Insert(m); err != nil {

1537 return fmt.Errorf("inserting message: %w", err)

1538 }

1539 if !noThreadID && m.ThreadID == 0 {

1540 m.ThreadID = m.ID

1541 if err := tx.Update(m); err != nil {

1542 return fmt.Errorf("updating message for its own thread id: %w", err)

1543 }

1544 }

1545

1546 // todo: perhaps we should match the recipients based on smtp submission and a matching message-id? we now miss the addresses in bcc's if the mail client doesn't save a message that includes the bcc header in the sent mailbox.

1547 if mb.Sent && part != nil && part.Envelope != nil {

1548 e := part.Envelope

1549 sent := e.Date

1550 if sent.IsZero() {

1551 sent = m.Received

1552 }

1553 if sent.IsZero() {

1554 sent = time.Now()

1555 }

1556 addrs := append(append(e.To, e.CC...), e.BCC...)

1557 for _, addr := range addrs {

1558 if addr.User == "" {

1559 // Would trigger error because Recipient.Localpart must be nonzero. todo: we could allow empty localpart in db, and filter by not using FilterNonzero.

1560 log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", slog.Any("address", addr))

1561 continue

1562 }

1563 d, err := dns.ParseDomain(addr.Host)

1564 if err != nil {

1565 log.Debugx("parsing domain in to/cc/bcc address", err, slog.Any("address", addr))

1566 continue

1567 }

1568 lp, err := smtp.ParseLocalpart(addr.User)

1569 if err != nil {

1570 log.Debugx("parsing localpart in to/cc/bcc address", err, slog.Any("address", addr))

1571 continue

1572 }

1573 mr := Recipient{

1574 MessageID: m.ID,

1575 Localpart: lp.String(),

1576 Domain: d.Name(),

1577 OrgDomain: publicsuffix.Lookup(context.TODO(), log.Logger, d).Name(),

1578 Sent: sent,

1579 }

1580 if err := tx.Insert(&mr); err != nil {

1581 return fmt.Errorf("inserting sent message recipients: %w", err)

1582 }

1583 }

1584 }

1585

1586 msgPath := a.MessagePath(m.ID)

1587 msgDir := filepath.Dir(msgPath)

1588 os.MkdirAll(msgDir, 0770)

1589

1590 // Sync file data to disk.

1591 if sync {

1592 if err := msgFile.Sync(); err != nil {

1593 return fmt.Errorf("fsync message file: %w", err)

1594 }

1595 }

1596

1597 if err := moxio.LinkOrCopy(log, msgPath, msgFile.Name(), &moxio.AtReader{R: msgFile}, true); err != nil {

1598 return fmt.Errorf("linking/copying message to new file: %w", err)

1599 }

1600

1601 if sync {

1602 if err := moxio.SyncDir(log, msgDir); err != nil {

1603 xerr := os.Remove(msgPath)

1604 log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath))

1605 return fmt.Errorf("sync directory: %w", err)

1606 }

1607 }

1608

1609 if !notrain && m.NeedsTraining() {

1610 l := []Message{*m}

1611 if err := a.RetrainMessages(context.TODO(), log, tx, l, false); err != nil {

1612 xerr := os.Remove(msgPath)

1613 log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath))

1614 return fmt.Errorf("training junkfilter: %w", err)

1615 }

1616 *m = l[0]

1617 }

1618

1619 return nil

1620}

1621

1622// SetPassword saves a new password for this account. This password is used for

1623// IMAP, SMTP (submission) sessions and the HTTP account web page.

1624//

1625// Callers are responsible for checking if the account has NoCustomPassword set.

1626func (a *Account) SetPassword(log mlog.Log, password string) error {

1627 password, err := precis.OpaqueString.String(password)

1628 if err != nil {

1629 return fmt.Errorf(`password not allowed by "precis"`)

1630 }

1631

1632 if len(password) < 8 {

1633 // We actually check for bytes...

1634 return fmt.Errorf("password must be at least 8 characters long")

1635 }

1636

1637 hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)

1638 if err != nil {

1639 return fmt.Errorf("generating password hash: %w", err)

1640 }

1641

1642 err = a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1643 if _, err := bstore.QueryTx[Password](tx).Delete(); err != nil {

1644 return fmt.Errorf("deleting existing password: %v", err)

1645 }

1646 var pw Password

1647 pw.Hash = string(hash)

1648

1649 // CRAM-MD5 calculates an HMAC-MD5, with the password as key, over a per-attempt

1650 // unique text that includes a timestamp. HMAC performs two hashes. Both times, the

1651 // first block is based on the key/password. We hash those first blocks now, and

1652 // store the hash state in the database. When we actually authenticate, we'll

1653 // complete the HMAC by hashing only the text. We cannot store crypto/hmac's hash,

1654 // because it does not expose its internal state and isn't a BinaryMarshaler.

1655 // ../rfc/2104:121

1656 pw.CRAMMD5.Ipad = md5.New()

1657 pw.CRAMMD5.Opad = md5.New()

1658 key := []byte(password)

1659 if len(key) > 64 {

1660 t := md5.Sum(key)

1661 key = t[:]

1662 }

1663 ipad := make([]byte, md5.BlockSize)

1664 opad := make([]byte, md5.BlockSize)

1665 copy(ipad, key)

1666 copy(opad, key)

1667 for i := range ipad {

1668 ipad[i] ^= 0x36

1669 opad[i] ^= 0x5c

1670 }

1671 pw.CRAMMD5.Ipad.Write(ipad)

1672 pw.CRAMMD5.Opad.Write(opad)

1673

1674 pw.SCRAMSHA1.Salt = scram.MakeRandom()

1675 pw.SCRAMSHA1.Iterations = 2 * 4096

1676 pw.SCRAMSHA1.SaltedPassword = scram.SaltPassword(sha1.New, password, pw.SCRAMSHA1.Salt, pw.SCRAMSHA1.Iterations)

1677

1678 pw.SCRAMSHA256.Salt = scram.MakeRandom()

1679 pw.SCRAMSHA256.Iterations = 4096

1680 pw.SCRAMSHA256.SaltedPassword = scram.SaltPassword(sha256.New, password, pw.SCRAMSHA256.Salt, pw.SCRAMSHA256.Iterations)

1681

1682 if err := tx.Insert(&pw); err != nil {

1683 return fmt.Errorf("inserting new password: %v", err)

1684 }

1685

1686 return sessionRemoveAll(context.TODO(), log, tx, a.Name)

1687 })

1688 if err == nil {

1689 log.Info("new password set for account", slog.String("account", a.Name))

1690 }

1691 return err

1692}

1693

1694// SessionsClear invalidates all (web) login sessions for the account.

1695func (a *Account) SessionsClear(ctx context.Context, log mlog.Log) error {

1696 return a.DB.Write(ctx, func(tx *bstore.Tx) error {

1697 return sessionRemoveAll(ctx, log, tx, a.Name)

1698 })

1699}

1700

1701// Subjectpass returns the signing key for use with subjectpass for the given

1702// email address with canonical localpart.

1703func (a *Account) Subjectpass(email string) (key string, err error) {

1704 return key, a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1705 v := Subjectpass{Email: email}

1706 err := tx.Get(&v)

1707 if err == nil {

1708 key = v.Key

1709 return nil

1710 }

1711 if !errors.Is(err, bstore.ErrAbsent) {

1712 return fmt.Errorf("get subjectpass key from accounts database: %w", err)

1713 }

1714 key = ""

1715 const chars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"

1716 buf := make([]byte, 16)

1717 if _, err := cryptorand.Read(buf); err != nil {

1718 return err

1719 }

1720 for _, b := range buf {

1721 key += string(chars[int(b)%len(chars)])

1722 }

1723 v.Key = key

1724 return tx.Insert(&v)

1725 })

1726}

1727

1728// Ensure mailbox is present in database, adding records for the mailbox and its

1729// parents if they aren't present.

1730//

1731// If subscribe is true, any mailboxes that were created will also be subscribed to.

1732//

1733// The leaf mailbox is created with special-use flags, taking the flags away from

1734// other mailboxes, and reflecting that in the returned changes.

1735//

1736// Caller must hold account wlock.

1737// Caller must propagate changes if any.

1738func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool, specialUse SpecialUse) (mb Mailbox, changes []Change, rerr error) {

1739 if norm.NFC.String(name) != name {

1740 return Mailbox{}, nil, fmt.Errorf("mailbox name not normalized")

1741 }

1742

1743 // Quick sanity check.

1744 if strings.EqualFold(name, "inbox") && name != "Inbox" {

1745 return Mailbox{}, nil, fmt.Errorf("bad casing for inbox")

1746 }

1747

1748 elems := strings.Split(name, "/")

1749 q := bstore.QueryTx[Mailbox](tx)

1750 q.FilterFn(func(mb Mailbox) bool {

1751 return mb.Name == elems[0] || strings.HasPrefix(mb.Name, elems[0]+"/")

1752 })

1753 l, err := q.List()

1754 if err != nil {

1755 return Mailbox{}, nil, fmt.Errorf("list mailboxes: %v", err)

1756 }

1757

1758 mailboxes := map[string]Mailbox{}

1759 for _, xmb := range l {

1760 mailboxes[xmb.Name] = xmb

1761 }

1762

1763 p := ""

1764 var existed bool

1765 for _, elem := range elems {

1766 if p != "" {

1767 p += "/"

1768 }

1769 p += elem

1770 mb, existed = mailboxes[p]

1771 if existed {

1772 continue

1773 }

1774 uidval, err := a.NextUIDValidity(tx)

1775 if err != nil {

1776 return Mailbox{}, nil, fmt.Errorf("next uid validity: %v", err)

1777 }

1778 mb = Mailbox{

1779 Name: p,

1780 UIDValidity: uidval,

1781 UIDNext: 1,

1782 HaveCounts: true,

1783 }

1784 err = tx.Insert(&mb)

1785 if err != nil {

1786 return Mailbox{}, nil, fmt.Errorf("creating new mailbox: %v", err)

1787 }

1788

1789 var flags []string

1790 if subscribe {

1791 if tx.Get(&Subscription{p}) != nil {

1792 err := tx.Insert(&Subscription{p})

1793 if err != nil {

1794 return Mailbox{}, nil, fmt.Errorf("subscribing to mailbox: %v", err)

1795 }

1796 }

1797 flags = []string{`\Subscribed`}

1798 }

1799 changes = append(changes, ChangeAddMailbox{mb, flags})

1800 }

1801

1802 // Clear any special-use flags from existing mailboxes and assign them to this mailbox.

1803 var zeroSpecialUse SpecialUse

1804 if !existed && specialUse != zeroSpecialUse {

1805 var qerr error

1806 clearSpecialUse := func(b bool, fn func(*Mailbox) *bool) {

1807 if !b || qerr != nil {

1808 return

1809 }

1810 qs := bstore.QueryTx[Mailbox](tx)

1811 qs.FilterFn(func(xmb Mailbox) bool {

1812 return *fn(&xmb)

1813 })

1814 xmb, err := qs.Get()

1815 if err == bstore.ErrAbsent {

1816 return

1817 } else if err != nil {

1818 qerr = fmt.Errorf("looking up mailbox with special-use flag: %v", err)

1819 return

1820 }

1821 p := fn(&xmb)

1822 *p = false

1823 if err := tx.Update(&xmb); err != nil {

1824 qerr = fmt.Errorf("clearing special-use flag: %v", err)

1825 } else {

1826 changes = append(changes, ChangeMailboxSpecialUse{xmb.ID, xmb.Name, xmb.SpecialUse})

1827 }

1828 }

1829 clearSpecialUse(specialUse.Archive, func(xmb *Mailbox) *bool { return &xmb.Archive })

1830 clearSpecialUse(specialUse.Draft, func(xmb *Mailbox) *bool { return &xmb.Draft })

1831 clearSpecialUse(specialUse.Junk, func(xmb *Mailbox) *bool { return &xmb.Junk })

1832 clearSpecialUse(specialUse.Sent, func(xmb *Mailbox) *bool { return &xmb.Sent })

1833 clearSpecialUse(specialUse.Trash, func(xmb *Mailbox) *bool { return &xmb.Trash })

1834 if qerr != nil {

1835 return Mailbox{}, nil, qerr

1836 }

1837

1838 mb.SpecialUse = specialUse

1839 if err := tx.Update(&mb); err != nil {

1840 return Mailbox{}, nil, fmt.Errorf("setting special-use flag for new mailbox: %v", err)

1841 }

1842 changes = append(changes, ChangeMailboxSpecialUse{mb.ID, mb.Name, mb.SpecialUse})

1843 }

1844 return mb, changes, nil

1845}

1846

1847// MailboxExists checks if mailbox exists.

1848// Caller must hold account rlock.

1849func (a *Account) MailboxExists(tx *bstore.Tx, name string) (bool, error) {

1850 q := bstore.QueryTx[Mailbox](tx)

1851 q.FilterEqual("Name", name)

1852 return q.Exists()

1853}

1854

1855// MailboxFind finds a mailbox by name, returning a nil mailbox and nil error if mailbox does not exist.

1856func (a *Account) MailboxFind(tx *bstore.Tx, name string) (*Mailbox, error) {

1857 q := bstore.QueryTx[Mailbox](tx)

1858 q.FilterEqual("Name", name)

1859 mb, err := q.Get()

1860 if err == bstore.ErrAbsent {

1861 return nil, nil

1862 }

1863 if err != nil {

1864 return nil, fmt.Errorf("looking up mailbox: %w", err)

1865 }

1866 return &mb, nil

1867}

1868

1869// SubscriptionEnsure ensures a subscription for name exists. The mailbox does not

1870// have to exist. Any parents are not automatically subscribed.

1871// Changes are returned and must be broadcasted by the caller.

1872func (a *Account) SubscriptionEnsure(tx *bstore.Tx, name string) ([]Change, error) {

1873 if err := tx.Get(&Subscription{name}); err == nil {

1874 return nil, nil

1875 }

1876

1877 if err := tx.Insert(&Subscription{name}); err != nil {

1878 return nil, fmt.Errorf("inserting subscription: %w", err)

1879 }

1880

1881 q := bstore.QueryTx[Mailbox](tx)

1882 q.FilterEqual("Name", name)

1883 _, err := q.Get()

1884 if err == nil {

1885 return []Change{ChangeAddSubscription{name, nil}}, nil

1886 } else if err != bstore.ErrAbsent {

1887 return nil, fmt.Errorf("looking up mailbox for subscription: %w", err)

1888 }

1889 return []Change{ChangeAddSubscription{name, []string{`\NonExistent`}}}, nil

1890}

1891

1892// MessageRuleset returns the first ruleset (if any) that matches the message

1893// represented by msgPrefix and msgFile, with smtp and validation fields from m.

1894func MessageRuleset(log mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset {

1895 if len(dest.Rulesets) == 0 {

1896 return nil

1897 }

1898

1899 mr := FileMsgReader(msgPrefix, msgFile) // We don't close, it would close the msgFile.

1900 p, err := message.Parse(log.Logger, false, mr)

1901 if err != nil {

1902 log.Errorx("parsing message for evaluating rulesets, continuing with headers", err, slog.String("parse", ""))

1903 // note: part is still set.

1904 }

1905 // todo optimize: only parse header if needed for rulesets. and probably reuse an earlier parsing.

1906 header, err := p.Header()

1907 if err != nil {

1908 log.Errorx("parsing message headers for evaluating rulesets, delivering to default mailbox", err, slog.String("parse", ""))

1909 // todo: reject message?

1910 return nil

1911 }

1912

1913ruleset:

1914 for _, rs := range dest.Rulesets {

1915 if rs.SMTPMailFromRegexpCompiled != nil {

1916 if !rs.SMTPMailFromRegexpCompiled.MatchString(m.MailFrom) {

1917 continue ruleset

1918 }

1919 }

1920 if rs.MsgFromRegexpCompiled != nil {

1921 if m.MsgFromLocalpart == "" && m.MsgFromDomain == "" || !rs.MsgFromRegexpCompiled.MatchString(m.MsgFromLocalpart.String()+"@"+m.MsgFromDomain) {

1922 continue ruleset

1923 }

1924 }

1925

1926 if !rs.VerifiedDNSDomain.IsZero() {

1927 d := rs.VerifiedDNSDomain.Name()

1928 suffix := "." + d

1929 matchDomain := func(s string) bool {

1930 return s == d || strings.HasSuffix(s, suffix)

1931 }

1932 var ok bool

1933 if m.EHLOValidated && matchDomain(m.EHLODomain) {

1934 ok = true

1935 }

1936 if m.MailFromValidated && matchDomain(m.MailFromDomain) {

1937 ok = true

1938 }

1939 for _, d := range m.DKIMDomains {

1940 if matchDomain(d) {

1941 ok = true

1942 break

1943 }

1944 }

1945 if !ok {

1946 continue ruleset

1947 }

1948 }

1949

1950 header:

1951 for _, t := range rs.HeadersRegexpCompiled {

1952 for k, vl := range header {

1953 k = strings.ToLower(k)

1954 if !t[0].MatchString(k) {

1955 continue

1956 }

1957 for _, v := range vl {

1958 v = strings.ToLower(strings.TrimSpace(v))

1959 if t[1].MatchString(v) {

1960 continue header

1961 }

1962 }

1963 }

1964 continue ruleset

1965 }

1966 return &rs

1967 }

1968 return nil

1969}

1970

1971// MessagePath returns the file system path of a message.

1972func (a *Account) MessagePath(messageID int64) string {

1973 return strings.Join(append([]string{a.Dir, "msg"}, messagePathElems(messageID)...), string(filepath.Separator))

1974}

1975

1976// MessageReader opens a message for reading, transparently combining the

1977// message prefix with the original incoming message.

1978func (a *Account) MessageReader(m Message) *MsgReader {

1979 return &MsgReader{prefix: m.MsgPrefix, path: a.MessagePath(m.ID), size: m.Size}

1980}

1981

1982// DeliverDestination delivers an email to dest, based on the configured rulesets.

1983//

1984// Returns ErrOverQuota when account would be over quota after adding message.

1985//

1986// Caller must hold account wlock (mailbox may be created).

1987// Message delivery, possible mailbox creation, and updated mailbox counts are

1988// broadcasted.

1989func (a *Account) DeliverDestination(log mlog.Log, dest config.Destination, m *Message, msgFile *os.File) error {

1990 var mailbox string

1991 rs := MessageRuleset(log, dest, m, m.MsgPrefix, msgFile)

1992 if rs != nil {

1993 mailbox = rs.Mailbox

1994 } else if dest.Mailbox == "" {

1995 mailbox = "Inbox"

1996 } else {

1997 mailbox = dest.Mailbox

1998 }

1999 return a.DeliverMailbox(log, mailbox, m, msgFile)

2000}

2001

2002// DeliverMailbox delivers an email to the specified mailbox.

2003//

2004// Returns ErrOverQuota when account would be over quota after adding message.

2005//

2006// Caller must hold account wlock (mailbox may be created).

2007// Message delivery, possible mailbox creation, and updated mailbox counts are

2008// broadcasted.

2009func (a *Account) DeliverMailbox(log mlog.Log, mailbox string, m *Message, msgFile *os.File) error {

2010 var changes []Change

2011 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

2012 if ok, _, err := a.CanAddMessageSize(tx, m.Size); err != nil {

2013 return err

2014 } else if !ok {

2015 return ErrOverQuota

2016 }

2017

2018 mb, chl, err := a.MailboxEnsure(tx, mailbox, true, SpecialUse{})

2019 if err != nil {

2020 return fmt.Errorf("ensuring mailbox: %w", err)

2021 }

2022 m.MailboxID = mb.ID

2023 m.MailboxOrigID = mb.ID

2024

2025 // Update count early, DeliverMessage will update mb too and we don't want to fetch

2026 // it again before updating.

2027 mb.MailboxCounts.Add(m.MailboxCounts())

2028 if err := tx.Update(&mb); err != nil {

2029 return fmt.Errorf("updating mailbox for delivery: %w", err)

2030 }

2031

2032 if err := a.DeliverMessage(log, tx, m, msgFile, true, false, false, true); err != nil {

2033 return err

2034 }

2035

2036 changes = append(changes, chl...)

2037 changes = append(changes, m.ChangeAddUID(), mb.ChangeCounts())

2038 return nil

2039 })

2040 // todo: if rename succeeded but transaction failed, we should remove the file.

2041 if err != nil {

2042 return err

2043 }

2044

2045 BroadcastChanges(a, changes)

2046 return nil

2047}

2048

2049// TidyRejectsMailbox removes old reject emails, and returns whether there is space for a new delivery.

2050//

2051// Caller most hold account wlock.

2052// Changes are broadcasted.

2053func (a *Account) TidyRejectsMailbox(log mlog.Log, rejectsMailbox string) (hasSpace bool, rerr error) {

2054 var changes []Change

2055

2056 var remove []Message

2057 defer func() {

2058 for _, m := range remove {

2059 p := a.MessagePath(m.ID)

2060 err := os.Remove(p)

2061 log.Check(err, "removing rejects message file", slog.String("path", p))

2062 }

2063 }()

2064

2065 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

2066 mb, err := a.MailboxFind(tx, rejectsMailbox)

2067 if err != nil {

2068 return fmt.Errorf("finding mailbox: %w", err)

2069 }

2070 if mb == nil {

2071 // No messages have been delivered yet.

2072 hasSpace = true

2073 return nil

2074 }

2075

2076 // Gather old messages to remove.

2077 old := time.Now().Add(-14 * 24 * time.Hour)

2078 qdel := bstore.QueryTx[Message](tx)

2079 qdel.FilterNonzero(Message{MailboxID: mb.ID})

2080 qdel.FilterEqual("Expunged", false)

2081 qdel.FilterLess("Received", old)

2082 remove, err = qdel.List()

2083 if err != nil {

2084 return fmt.Errorf("listing old messages: %w", err)

2085 }

2086

2087 changes, err = a.rejectsRemoveMessages(context.TODO(), log, tx, mb, remove)

2088 if err != nil {

2089 return fmt.Errorf("removing messages: %w", err)

2090 }

2091

2092 // We allow up to n messages.

2093 qcount := bstore.QueryTx[Message](tx)

2094 qcount.FilterNonzero(Message{MailboxID: mb.ID})

2095 qcount.FilterEqual("Expunged", false)

2096 qcount.Limit(1000)

2097 n, err := qcount.Count()

2098 if err != nil {

2099 return fmt.Errorf("counting rejects: %w", err)

2100 }

2101 hasSpace = n < 1000

2102

2103 return nil

2104 })

2105 if err != nil {

2106 remove = nil // Don't remove files on failure.

2107 return false, err

2108 }

2109

2110 BroadcastChanges(a, changes)

2111

2112 return hasSpace, nil

2113}

2114

2115func (a *Account) rejectsRemoveMessages(ctx context.Context, log mlog.Log, tx *bstore.Tx, mb *Mailbox, l []Message) ([]Change, error) {

2116 if len(l) == 0 {

2117 return nil, nil

2118 }

2119 ids := make([]int64, len(l))

2120 anyids := make([]any, len(l))

2121 for i, m := range l {

2122 ids[i] = m.ID

2123 anyids[i] = m.ID

2124 }

2125

2126 // Remove any message recipients. Should not happen, but a user can move messages

2127 // from a Sent mailbox to the rejects mailbox...

2128 qdmr := bstore.QueryTx[Recipient](tx)

2129 qdmr.FilterEqual("MessageID", anyids...)

2130 if _, err := qdmr.Delete(); err != nil {

2131 return nil, fmt.Errorf("deleting from message recipient: %w", err)

2132 }

2133

2134 // Assign new modseq.

2135 modseq, err := a.NextModSeq(tx)

2136 if err != nil {

2137 return nil, fmt.Errorf("assign next modseq: %w", err)

2138 }

2139

2140 // Expunge the messages.

2141 qx := bstore.QueryTx[Message](tx)

2142 qx.FilterIDs(ids)

2143 var expunged []Message

2144 qx.Gather(&expunged)

2145 if _, err := qx.UpdateNonzero(Message{ModSeq: modseq, Expunged: true}); err != nil {

2146 return nil, fmt.Errorf("expunging messages: %w", err)

2147 }

2148

2149 var totalSize int64

2150 for _, m := range expunged {

2151 m.Expunged = false // Was set by update, but would cause wrong count.

2152 mb.MailboxCounts.Sub(m.MailboxCounts())

2153 totalSize += m.Size

2154 }

2155 if err := tx.Update(mb); err != nil {

2156 return nil, fmt.Errorf("updating mailbox counts: %w", err)

2157 }

2158 if err := a.AddMessageSize(log, tx, -totalSize); err != nil {

2159 return nil, fmt.Errorf("updating disk usage: %w", err)

2160 }

2161

2162 // Mark as neutral and train so junk filter gets untrained with these (junk) messages.

2163 for i := range expunged {

2164 expunged[i].Junk = false

2165 expunged[i].Notjunk = false

2166 }

2167 if err := a.RetrainMessages(ctx, log, tx, expunged, true); err != nil {

2168 return nil, fmt.Errorf("retraining expunged messages: %w", err)

2169 }

2170

2171 changes := make([]Change, len(l), len(l)+1)

2172 for i, m := range l {

2173 changes[i] = ChangeRemoveUIDs{mb.ID, []UID{m.UID}, modseq}

2174 }

2175 changes = append(changes, mb.ChangeCounts())

2176 return changes, nil

2177}

2178

2179// RejectsRemove removes a message from the rejects mailbox if present.

2180// Caller most hold account wlock.

2181// Changes are broadcasted.

2182func (a *Account) RejectsRemove(log mlog.Log, rejectsMailbox, messageID string) error {

2183 var changes []Change

2184

2185 var remove []Message

2186 defer func() {

2187 for _, m := range remove {

2188 p := a.MessagePath(m.ID)

2189 err := os.Remove(p)

2190 log.Check(err, "removing rejects message file", slog.String("path", p))

2191 }

2192 }()

2193

2194 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

2195 mb, err := a.MailboxFind(tx, rejectsMailbox)

2196 if err != nil {

2197 return fmt.Errorf("finding mailbox: %w", err)

2198 }

2199 if mb == nil {

2200 return nil

2201 }

2202

2203 q := bstore.QueryTx[Message](tx)

2204 q.FilterNonzero(Message{MailboxID: mb.ID, MessageID: messageID})

2205 q.FilterEqual("Expunged", false)

2206 remove, err = q.List()

2207 if err != nil {

2208 return fmt.Errorf("listing messages to remove: %w", err)

2209 }

2210

2211 changes, err = a.rejectsRemoveMessages(context.TODO(), log, tx, mb, remove)

2212 if err != nil {

2213 return fmt.Errorf("removing messages: %w", err)

2214 }

2215

2216 return nil

2217 })

2218 if err != nil {

2219 remove = nil // Don't remove files on failure.

2220 return err

2221 }

2222

2223 BroadcastChanges(a, changes)

2224

2225 return nil

2226}

2227

2228// AddMessageSize adjusts the DiskUsage.MessageSize by size.

2229func (a *Account) AddMessageSize(log mlog.Log, tx *bstore.Tx, size int64) error {

2230 du := DiskUsage{ID: 1}

2231 if err := tx.Get(&du); err != nil {

2232 return fmt.Errorf("get diskusage: %v", err)

2233 }

2234 du.MessageSize += size

2235 if du.MessageSize < 0 {

2236 log.Error("negative total message size", slog.Int64("delta", size), slog.Int64("newtotalsize", du.MessageSize))

2237 }

2238 if err := tx.Update(&du); err != nil {

2239 return fmt.Errorf("update total message size: %v", err)

2240 }

2241 return nil

2242}

2243

2244// QuotaMessageSize returns the effective maximum total message size for an

2245// account. Returns 0 if there is no maximum.

2246func (a *Account) QuotaMessageSize() int64 {

2247 conf, _ := a.Conf()

2248 size := conf.QuotaMessageSize

2249 if size == 0 {

2250 size = mox.Conf.Static.QuotaMessageSize

2251 }

2252 if size < 0 {

2253 size = 0

2254 }

2255 return size

2256}

2257

2258// CanAddMessageSize checks if a message of size bytes can be added, depending on

2259// total message size and configured quota for account.

2260func (a *Account) CanAddMessageSize(tx *bstore.Tx, size int64) (ok bool, maxSize int64, err error) {

2261 maxSize = a.QuotaMessageSize()

2262 if maxSize <= 0 {

2263 return true, 0, nil

2264 }

2265

2266 du := DiskUsage{ID: 1}

2267 if err := tx.Get(&du); err != nil {

2268 return false, maxSize, fmt.Errorf("get diskusage: %v", err)

2269 }

2270 return du.MessageSize+size <= maxSize, maxSize, nil

2271}

2272

2273// We keep a cache of recent successful authentications, so we don't have to bcrypt successful calls each time.

2274var authCache = struct {

2275 sync.Mutex

2276 success map[authKey]string

2277}{

2278 success: map[authKey]string{},

2279}

2280

2281type authKey struct {

2282 email, hash string

2283}

2284

2285// StartAuthCache starts a goroutine that regularly clears the auth cache.

2286func StartAuthCache() {

2287 go manageAuthCache()

2288}

2289

2290func manageAuthCache() {

2291 for {

2292 authCache.Lock()

2293 authCache.success = map[authKey]string{}

2294 authCache.Unlock()

2295 time.Sleep(15 * time.Minute)

2296 }

2297}

2298

2299// OpenEmailAuth opens an account given an email address and password.

2300//

2301// The email address may contain a catchall separator.

2302// For invalid credentials, a nil account is returned, but accName may be

2303// non-empty.

2304func OpenEmailAuth(log mlog.Log, email string, password string, checkLoginDisabled bool) (acc *Account, accName string, rerr error) {

2305 // We check for LoginDisabled after verifying the password. Otherwise users can get

2306 // messages about the account being disabled without knowing the password.

2307 acc, accName, _, rerr = OpenEmail(log, email, false)

2308 if rerr != nil {

2309 return

2310 }

2311

2312 defer func() {

2313 if rerr != nil {

2314 err := acc.Close()

2315 log.Check(err, "closing account after open auth failure")

2316 acc = nil

2317 }

2318 }()

2319

2320 password, err := precis.OpaqueString.String(password)

2321 if err != nil {

2322 return nil, accName, ErrUnknownCredentials

2323 }

2324

2325 pw, err := bstore.QueryDB[Password](context.TODO(), acc.DB).Get()

2326 if err != nil {

2327 if err == bstore.ErrAbsent {

2328 return acc, accName, ErrUnknownCredentials

2329 }

2330 return acc, accName, fmt.Errorf("looking up password: %v", err)

2331 }

2332 authCache.Lock()

2333 ok := len(password) >= 8 && authCache.success[authKey{email, pw.Hash}] == password

2334 authCache.Unlock()

2335 if !ok {

2336 if err := bcrypt.CompareHashAndPassword([]byte(pw.Hash), []byte(password)); err != nil {

2337 return acc, accName, ErrUnknownCredentials

2338 }

2339 }

2340 if checkLoginDisabled {

2341 conf, aok := acc.Conf()

2342 if !aok {

2343 return acc, accName, fmt.Errorf("cannot find config for account")

2344 } else if conf.LoginDisabled != "" {

2345 return acc, accName, fmt.Errorf("%w: %s", ErrLoginDisabled, conf.LoginDisabled)

2346 }

2347 }

2348 authCache.Lock()

2349 authCache.success[authKey{email, pw.Hash}] = password

2350 authCache.Unlock()

2351 return

2352}

2353

2354// OpenEmail opens an account given an email address.

2355//

2356// The email address may contain a catchall separator.

2357//

2358// Returns account on success, may return non-empty account name even on error.

2359func OpenEmail(log mlog.Log, email string, checkLoginDisabled bool) (*Account, string, config.Destination, error) {

2360 addr, err := smtp.ParseAddress(email)

2361 if err != nil {

2362 return nil, "", config.Destination{}, fmt.Errorf("%w: %v", ErrUnknownCredentials, err)

2363 }

2364 accountName, _, _, dest, err := mox.LookupAddress(addr.Localpart, addr.Domain, false, false, false)

2365 if err != nil && (errors.Is(err, mox.ErrAddressNotFound) || errors.Is(err, mox.ErrDomainNotFound)) {

2366 return nil, accountName, config.Destination{}, ErrUnknownCredentials

2367 } else if err != nil {

2368 return nil, accountName, config.Destination{}, fmt.Errorf("looking up address: %v", err)

2369 }

2370 acc, err := OpenAccount(log, accountName, checkLoginDisabled)

2371 if err != nil {

2372 return nil, accountName, config.Destination{}, err

2373 }

2374 return acc, accountName, dest, nil

2375}

2376

2377// 64 characters, must be power of 2 for MessagePath

2378const msgDirChars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ-_"

2379

2380// MessagePath returns the filename of the on-disk filename, relative to the

2381// containing directory such as <account>/msg or queue.

2382// Returns names like "AB/1".

2383func MessagePath(messageID int64) string {

2384 return strings.Join(messagePathElems(messageID), string(filepath.Separator))

2385}

2386

2387// messagePathElems returns the elems, for a single join without intermediate

2388// string allocations.

2389func messagePathElems(messageID int64) []string {

2390 v := messageID >> 13 // 8k files per directory.

2391 dir := ""

2392 for {

2393 dir += string(msgDirChars[int(v)&(len(msgDirChars)-1)])

2394 v >>= 6

2395 if v == 0 {

2396 break

2397 }

2398 }

2399 return []string{dir, strconv.FormatInt(messageID, 10)}

2400}

2401

2402// Set returns a copy of f, with each flag that is true in mask set to the

2403// value from flags.

2404func (f Flags) Set(mask, flags Flags) Flags {

2405 set := func(d *bool, m, v bool) {

2406 if m {

2407 *d = v

2408 }

2409 }

2410 r := f

2411 set(&r.Seen, mask.Seen, flags.Seen)

2412 set(&r.Answered, mask.Answered, flags.Answered)

2413 set(&r.Flagged, mask.Flagged, flags.Flagged)

2414 set(&r.Forwarded, mask.Forwarded, flags.Forwarded)

2415 set(&r.Junk, mask.Junk, flags.Junk)

2416 set(&r.Notjunk, mask.Notjunk, flags.Notjunk)

2417 set(&r.Deleted, mask.Deleted, flags.Deleted)

2418 set(&r.Draft, mask.Draft, flags.Draft)

2419 set(&r.Phishing, mask.Phishing, flags.Phishing)

2420 set(&r.MDNSent, mask.MDNSent, flags.MDNSent)

2421 return r

2422}

2423

2424// Changed returns a mask of flags that have been between f and other.

2425func (f Flags) Changed(other Flags) (mask Flags) {

2426 mask.Seen = f.Seen != other.Seen

2427 mask.Answered = f.Answered != other.Answered

2428 mask.Flagged = f.Flagged != other.Flagged

2429 mask.Forwarded = f.Forwarded != other.Forwarded

2430 mask.Junk = f.Junk != other.Junk

2431 mask.Notjunk = f.Notjunk != other.Notjunk

2432 mask.Deleted = f.Deleted != other.Deleted

2433 mask.Draft = f.Draft != other.Draft

2434 mask.Phishing = f.Phishing != other.Phishing

2435 mask.MDNSent = f.MDNSent != other.MDNSent

2436 return

2437}

2438

2439// Strings returns the flags that are set in their string form.

2440func (f Flags) Strings() []string {

2441 fields := []struct {

2442 word string

2443 have bool

2444 }{

2445 {`$forwarded`, f.Forwarded},

2446 {`$junk`, f.Junk},

2447 {`$mdnsent`, f.MDNSent},

2448 {`$notjunk`, f.Notjunk},

2449 {`$phishing`, f.Phishing},

2450 {`\answered`, f.Answered},

2451 {`\deleted`, f.Deleted},

2452 {`\draft`, f.Draft},

2453 {`\flagged`, f.Flagged},

2454 {`\seen`, f.Seen},

2455 }

2456 var l []string

2457 for _, fh := range fields {

2458 if fh.have {

2459 l = append(l, fh.word)

2460 }

2461 }

2462 return l

2463}

2464

2465var systemWellKnownFlags = map[string]bool{

2466 `\answered`: true,

2467 `\flagged`: true,

2468 `\deleted`: true,

2469 `\seen`: true,

2470 `\draft`: true,

2471 `$junk`: true,

2472 `$notjunk`: true,

2473 `$forwarded`: true,

2474 `$phishing`: true,

2475 `$mdnsent`: true,

2476}

2477

2478// ParseFlagsKeywords parses a list of textual flags into system/known flags, and

2479// other keywords. Keywords are lower-cased and sorted and check for valid syntax.

2480func ParseFlagsKeywords(l []string) (flags Flags, keywords []string, rerr error) {

2481 fields := map[string]*bool{

2482 `\answered`: &flags.Answered,

2483 `\flagged`: &flags.Flagged,

2484 `\deleted`: &flags.Deleted,

2485 `\seen`: &flags.Seen,

2486 `\draft`: &flags.Draft,

2487 `$junk`: &flags.Junk,

2488 `$notjunk`: &flags.Notjunk,

2489 `$forwarded`: &flags.Forwarded,

2490 `$phishing`: &flags.Phishing,

2491 `$mdnsent`: &flags.MDNSent,

2492 }

2493 seen := map[string]bool{}

2494 for _, f := range l {

2495 f = strings.ToLower(f)

2496 if field, ok := fields[f]; ok {

2497 *field = true

2498 } else if seen[f] {

2499 if mox.Pedantic {

2500 return Flags{}, nil, fmt.Errorf("duplicate keyword %s", f)

2501 }

2502 } else {

2503 if err := CheckKeyword(f); err != nil {

2504 return Flags{}, nil, fmt.Errorf("invalid keyword %s", f)

2505 }

2506 keywords = append(keywords, f)

2507 seen[f] = true

2508 }

2509 }

2510 sort.Strings(keywords)

2511 return flags, keywords, nil

2512}

2513

2514// RemoveKeywords removes keywords from l, returning whether any modifications were

2515// made, and a slice, a new slice in case of modifications. Keywords must have been

2516// validated earlier, e.g. through ParseFlagKeywords or CheckKeyword. Should only

2517// be used with valid keywords, not with system flags like \Seen.

2518func RemoveKeywords(l, remove []string) ([]string, bool) {

2519 var copied bool

2520 var changed bool

2521 for _, k := range remove {

2522 if i := slices.Index(l, k); i >= 0 {

2523 if !copied {

2524 l = append([]string{}, l...)

2525 copied = true

2526 }

2527 copy(l[i:], l[i+1:])

2528 l = l[:len(l)-1]

2529 changed = true

2530 }

2531 }

2532 return l, changed

2533}

2534

2535// MergeKeywords adds keywords from add into l, returning whether it added any

2536// keyword, and the slice with keywords, a new slice if modifications were made.

2537// Keywords are only added if they aren't already present. Should only be used with

2538// keywords, not with system flags like \Seen.

2539func MergeKeywords(l, add []string) ([]string, bool) {

2540 var copied bool

2541 var changed bool

2542 for _, k := range add {

2543 if !slices.Contains(l, k) {

2544 if !copied {

2545 l = append([]string{}, l...)

2546 copied = true

2547 }

2548 l = append(l, k)

2549 changed = true

2550 }

2551 }

2552 if changed {

2553 sort.Strings(l)

2554 }

2555 return l, changed

2556}

2557

2558// CheckKeyword returns an error if kw is not a valid keyword. Kw should

2559// already be in lower-case.

2560func CheckKeyword(kw string) error {

2561 if kw == "" {

2562 return fmt.Errorf("keyword cannot be empty")

2563 }

2564 if systemWellKnownFlags[kw] {

2565 return fmt.Errorf("cannot use well-known flag as keyword")

2566 }

2567 for _, c := range kw {

2568 // ../rfc/9051:6334

2569 if c <= ' ' || c > 0x7e || c >= 'A' && c <= 'Z' || strings.ContainsRune(`(){%*"\]`, c) {

2570 return errors.New(`not a valid keyword, must be lower-case ascii without spaces and without any of these characters: (){%*"\]`)

2571 }

2572 }

2573 return nil

2574}

2575

2576// SendLimitReached checks whether sending a message to recipients would reach

2577// the limit of outgoing messages for the account. If so, the message should

2578// not be sent. If the returned numbers are >= 0, the limit was reached and the

2579// values are the configured limits.

2580//

2581// To limit damage to the internet and our reputation in case of account

2582// compromise, we limit the max number of messages sent in a 24 hour window, both

2583// total number of messages and number of first-time recipients.

2584func (a *Account) SendLimitReached(tx *bstore.Tx, recipients []smtp.Path) (msglimit, rcptlimit int, rerr error) {

2585 conf, _ := a.Conf()

2586 msgmax := conf.MaxOutgoingMessagesPerDay

2587 if msgmax == 0 {

2588 // For human senders, 1000 recipients in a day is quite a lot.

2589 msgmax = 1000

2590 }

2591 rcptmax := conf.MaxFirstTimeRecipientsPerDay

2592 if rcptmax == 0 {

2593 // Human senders may address a new human-sized list of people once in a while. In

2594 // case of a compromise, a spammer will probably try to send to many new addresses.

2595 rcptmax = 200

2596 }

2597

2598 rcpts := map[string]time.Time{}

2599 n := 0

2600 err := bstore.QueryTx[Outgoing](tx).FilterGreater("Submitted", time.Now().Add(-24*time.Hour)).ForEach(func(o Outgoing) error {

2601 n++

2602 if rcpts[o.Recipient].IsZero() || o.Submitted.Before(rcpts[o.Recipient]) {

2603 rcpts[o.Recipient] = o.Submitted

2604 }

2605 return nil

2606 })

2607 if err != nil {

2608 return -1, -1, fmt.Errorf("querying message recipients in past 24h: %w", err)

2609 }

2610 if n+len(recipients) > msgmax {

2611 return msgmax, -1, nil

2612 }

2613

2614 // Only check if max first-time recipients is reached if there are enough messages

2615 // to trigger the limit.

2616 if n+len(recipients) < rcptmax {

2617 return -1, -1, nil

2618 }

2619

2620 isFirstTime := func(rcpt string, before time.Time) (bool, error) {

2621 exists, err := bstore.QueryTx[Outgoing](tx).FilterNonzero(Outgoing{Recipient: rcpt}).FilterLess("Submitted", before).Exists()

2622 return !exists, err

2623 }

2624

2625 firsttime := 0

2626 now := time.Now()

2627 for _, r := range recipients {

2628 if first, err := isFirstTime(r.XString(true), now); err != nil {

2629 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

2630 } else if first {

2631 firsttime++

2632 }

2633 }

2634 for r, t := range rcpts {

2635 if first, err := isFirstTime(r, t); err != nil {

2636 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

2637 } else if first {

2638 firsttime++

2639 }

2640 }

2641 if firsttime > rcptmax {

2642 return -1, rcptmax, nil

2643 }

2644 return -1, -1, nil

2645}

2646

2647// MailboxCreate creates a new mailbox, including any missing parent mailboxes,

2648// the total list of created mailboxes is returned in created. On success, if

2649// exists is false and rerr nil, the changes must be broadcasted by the caller.

2650//

2651// The mailbox is created with special-use flags, with those flags taken away from

2652// other mailboxes if they have them, reflected in the returned changes.

2653//

2654// Name must be in normalized form.

2655func (a *Account) MailboxCreate(tx *bstore.Tx, name string, specialUse SpecialUse) (changes []Change, created []string, exists bool, rerr error) {

2656 elems := strings.Split(name, "/")

2657 var p string

2658 for i, elem := range elems {

2659 if i > 0 {

2660 p += "/"

2661 }

2662 p += elem

2663 exists, err := a.MailboxExists(tx, p)

2664 if err != nil {

2665 return nil, nil, false, fmt.Errorf("checking if mailbox exists")

2666 }

2667 if exists {

2668 if i == len(elems)-1 {

2669 return nil, nil, true, fmt.Errorf("mailbox already exists")

2670 }

2671 continue

2672 }

2673 _, nchanges, err := a.MailboxEnsure(tx, p, true, specialUse)

2674 if err != nil {

2675 return nil, nil, false, fmt.Errorf("ensuring mailbox exists: %v", err)

2676 }

2677 changes = append(changes, nchanges...)

2678 created = append(created, p)

2679 }

2680 return changes, created, false, nil

2681}

2682

2683// MailboxRename renames mailbox mbsrc to dst, and any missing parents for the

2684// destination, and any children of mbsrc and the destination.

2685//

2686// Names must be normalized and cannot be Inbox.

2687func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc Mailbox, dst string) (changes []Change, isInbox, notExists, alreadyExists bool, rerr error) {

2688 if mbsrc.Name == "Inbox" || dst == "Inbox" {

2689 return nil, true, false, false, fmt.Errorf("inbox cannot be renamed")

2690 }

2691

2692 // We gather existing mailboxes that we need for deciding what to create/delete/update.

2693 q := bstore.QueryTx[Mailbox](tx)

2694 srcPrefix := mbsrc.Name + "/"

2695 dstRoot := strings.SplitN(dst, "/", 2)[0]

2696 dstRootPrefix := dstRoot + "/"

2697 q.FilterFn(func(mb Mailbox) bool {

2698 return mb.Name == mbsrc.Name || strings.HasPrefix(mb.Name, srcPrefix) || mb.Name == dstRoot || strings.HasPrefix(mb.Name, dstRootPrefix)

2699 })

2700 q.SortAsc("Name") // We'll rename the parents before children.

2701 l, err := q.List()

2702 if err != nil {

2703 return nil, false, false, false, fmt.Errorf("listing relevant mailboxes: %v", err)

2704 }

2705

2706 mailboxes := map[string]Mailbox{}

2707 for _, mb := range l {

2708 mailboxes[mb.Name] = mb

2709 }

2710

2711 if _, ok := mailboxes[mbsrc.Name]; !ok {

2712 return nil, false, true, false, fmt.Errorf("mailbox does not exist")

2713 }

2714

2715 uidval, err := a.NextUIDValidity(tx)

2716 if err != nil {

2717 return nil, false, false, false, fmt.Errorf("next uid validity: %v", err)

2718 }

2719

2720 // Ensure parent mailboxes for the destination paths exist.

2721 var parent string

2722 dstElems := strings.Split(dst, "/")

2723 for i, elem := range dstElems[:len(dstElems)-1] {

2724 if i > 0 {

2725 parent += "/"

2726 }

2727 parent += elem

2728

2729 mb, ok := mailboxes[parent]

2730 if ok {

2731 continue

2732 }

2733 omb := mb

2734 mb = Mailbox{

2735 ID: omb.ID,

2736 Name: parent,

2737 UIDValidity: uidval,

2738 UIDNext: 1,

2739 HaveCounts: true,

2740 }

2741 if err := tx.Insert(&mb); err != nil {

2742 return nil, false, false, false, fmt.Errorf("creating parent mailbox %q: %v", mb.Name, err)

2743 }

2744 if err := tx.Get(&Subscription{Name: parent}); err != nil {

2745 if err := tx.Insert(&Subscription{Name: parent}); err != nil {

2746 return nil, false, false, false, fmt.Errorf("creating subscription for %q: %v", parent, err)

2747 }

2748 }

2749 changes = append(changes, ChangeAddMailbox{Mailbox: mb, Flags: []string{`\Subscribed`}})

2750 }

2751

2752 // Process src mailboxes, renaming them to dst.

2753 for _, srcmb := range l {

2754 if srcmb.Name != mbsrc.Name && !strings.HasPrefix(srcmb.Name, srcPrefix) {

2755 continue

2756 }

2757 srcName := srcmb.Name

2758 dstName := dst + srcmb.Name[len(mbsrc.Name):]

2759 if _, ok := mailboxes[dstName]; ok {

2760 return nil, false, false, true, fmt.Errorf("destination mailbox %q already exists", dstName)

2761 }

2762

2763 srcmb.Name = dstName

2764 srcmb.UIDValidity = uidval

2765 if err := tx.Update(&srcmb); err != nil {

2766 return nil, false, false, false, fmt.Errorf("renaming mailbox: %v", err)

2767 }

2768

2769 var dstFlags []string

2770 if tx.Get(&Subscription{Name: dstName}) == nil {

2771 dstFlags = []string{`\Subscribed`}

2772 }

2773 changes = append(changes, ChangeRenameMailbox{MailboxID: srcmb.ID, OldName: srcName, NewName: dstName, Flags: dstFlags})

2774 }

2775

2776 // If we renamed e.g. a/b to a/b/c/d, and a/b/c to a/b/c/d/c, we'll have to recreate a/b and a/b/c.

2777 srcElems := strings.Split(mbsrc.Name, "/")

2778 xsrc := mbsrc.Name

2779 for i := 0; i < len(dstElems) && strings.HasPrefix(dst, xsrc+"/"); i++ {

2780 mb := Mailbox{

2781 UIDValidity: uidval,

2782 UIDNext: 1,

2783 Name: xsrc,

2784 HaveCounts: true,

2785 }

2786 if err := tx.Insert(&mb); err != nil {

2787 return nil, false, false, false, fmt.Errorf("creating mailbox at old path %q: %v", mb.Name, err)

2788 }

2789 xsrc += "/" + dstElems[len(srcElems)+i]

2790 }

2791 return changes, false, false, false, nil

2792}

2793

2794// MailboxDelete deletes a mailbox by ID, including its annotations. If it has

2795// children, the return value indicates that and an error is returned.

2796//

2797// Caller should broadcast the changes and remove files for the removed message IDs.

2798func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx, mailbox Mailbox) (changes []Change, removeMessageIDs []int64, hasChildren bool, rerr error) {

2799 // Look for existence of child mailboxes. There is a lot of text in the IMAP RFCs about

2800 // NoInferior and NoSelect. We just require only leaf mailboxes are deleted.

2801 qmb := bstore.QueryTx[Mailbox](tx)

2802 mbprefix := mailbox.Name + "/"

2803 qmb.FilterFn(func(mb Mailbox) bool {

2804 return strings.HasPrefix(mb.Name, mbprefix)

2805 })

2806 if childExists, err := qmb.Exists(); err != nil {

2807 return nil, nil, false, fmt.Errorf("checking if mailbox has child: %v", err)

2808 } else if childExists {

2809 return nil, nil, true, fmt.Errorf("mailbox has a child, only leaf mailboxes can be deleted")

2810 }

2811

2812 // todo jmap: instead of completely deleting a mailbox and its messages, we need to mark them all as expunged.

2813

2814 qm := bstore.QueryTx[Message](tx)

2815 qm.FilterNonzero(Message{MailboxID: mailbox.ID})

2816 remove, err := qm.List()

2817 if err != nil {

2818 return nil, nil, false, fmt.Errorf("listing messages to remove: %v", err)

2819 }

2820

2821 if len(remove) > 0 {

2822 removeIDs := make([]any, len(remove))

2823 for i, m := range remove {

2824 removeIDs[i] = m.ID

2825 }

2826 qmr := bstore.QueryTx[Recipient](tx)

2827 qmr.FilterEqual("MessageID", removeIDs...)

2828 if _, err = qmr.Delete(); err != nil {

2829 return nil, nil, false, fmt.Errorf("removing message recipients for messages: %v", err)

2830 }

2831

2832 qm = bstore.QueryTx[Message](tx)

2833 qm.FilterNonzero(Message{MailboxID: mailbox.ID})

2834 if _, err := qm.Delete(); err != nil {

2835 return nil, nil, false, fmt.Errorf("removing messages: %v", err)

2836 }

2837

2838 var totalSize int64

2839 for _, m := range remove {

2840 if !m.Expunged {

2841 removeMessageIDs = append(removeMessageIDs, m.ID)

2842 totalSize += m.Size

2843 }

2844 }

2845 if err := a.AddMessageSize(log, tx, -totalSize); err != nil {

2846 return nil, nil, false, fmt.Errorf("updating disk usage: %v", err)

2847 }

2848

2849 // Mark messages as not needing training. Then retrain them, so they are untrained if they were.

2850 n := 0

2851 o := 0

2852 for _, m := range remove {

2853 if !m.Expunged {

2854 remove[o] = m

2855 remove[o].Junk = false

2856 remove[o].Notjunk = false

2857 n++

2858 }

2859 }

2860 remove = remove[:n]

2861 if err := a.RetrainMessages(ctx, log, tx, remove, true); err != nil {

2862 return nil, nil, false, fmt.Errorf("untraining deleted messages: %v", err)

2863 }

2864 }

2865

2866 // Remove metadata annotations. ../rfc/5464:373

2867 if _, err := bstore.QueryTx[Annotation](tx).FilterNonzero(Annotation{MailboxID: mailbox.ID}).Delete(); err != nil {

2868 return nil, nil, false, fmt.Errorf("removing annotations for mailbox: %v", err)

2869 }

2870 // Not sending changes about annotations on this mailbox, since the entire mailbox

2871 // is being removed.

2872

2873 if err := tx.Delete(&Mailbox{ID: mailbox.ID}); err != nil {

2874 return nil, nil, false, fmt.Errorf("removing mailbox: %v", err)

2875 }

2876 return []Change{ChangeRemoveMailbox{MailboxID: mailbox.ID, Name: mailbox.Name}}, removeMessageIDs, false, nil

2877}

2878

2879// CheckMailboxName checks if name is valid, returning an INBOX-normalized name.

2880// I.e. it changes various casings of INBOX and INBOX/* to Inbox and Inbox/*.

2881// Name is invalid if it contains leading/trailing/double slashes, or when it isn't

2882// unicode-normalized, or when empty or has special characters.

2883//

2884// If name is the inbox, and allowInbox is false, this is indicated with the isInbox return parameter.

2885// For that case, and for other invalid names, an error is returned.

2886func CheckMailboxName(name string, allowInbox bool) (normalizedName string, isInbox bool, rerr error) {

2887 first := strings.SplitN(name, "/", 2)[0]

2888 if strings.EqualFold(first, "inbox") {

2889 if len(name) == len("inbox") && !allowInbox {

2890 return "", true, fmt.Errorf("special mailbox name Inbox not allowed")

2891 }

2892 name = "Inbox" + name[len("Inbox"):]

2893 }

2894

2895 if norm.NFC.String(name) != name {

2896 return "", false, errors.New("non-unicode-normalized mailbox names not allowed")

2897 }

2898

2899 if name == "" {

2900 return "", false, errors.New("empty mailbox name")

2901 }

2902 if strings.HasPrefix(name, "/") || strings.HasSuffix(name, "/") || strings.Contains(name, "//") {

2903 return "", false, errors.New("bad slashes in mailbox name")

2904 }

2905

2906 // "%" and "*" are difficult to use with the IMAP LIST command, but we allow mostly

2907 // allow them. ../rfc/3501:1002 ../rfc/9051:983

2908 if strings.HasPrefix(name, "#") {

2909 return "", false, errors.New("mailbox name cannot start with hash due to conflict with imap namespaces")

2910 }

2911

2912 // "#" and "&" are special in IMAP mailbox names. "#" for namespaces, "&" for

2913 // IMAP-UTF-7 encoding. We do allow them. ../rfc/3501:1018 ../rfc/9051:991

2914

2915 for _, c := range name {

2916 // ../rfc/3501:999 ../rfc/6855:192 ../rfc/9051:979

2917 if c <= 0x1f || c >= 0x7f && c <= 0x9f || c == 0x2028 || c == 0x2029 {

2918 return "", false, errors.New("control characters not allowed in mailbox name")

2919 }

2920 }

2921 return name, false, nil

2922}

2923