1/*

2Package store implements storage for accounts, their mailboxes, IMAP

3subscriptions and messages, and broadcasts updates (e.g. mail delivery) to

4interested sessions (e.g. IMAP connections).

5

6Layout of storage for accounts:

7

8 <DataDir>/accounts/<name>/index.db

9 <DataDir>/accounts/<name>/msg/[a-zA-Z0-9_-]+/<id>

10

11Index.db holds tables for user information, mailboxes, and messages. Messages

12are stored in the msg/ subdirectory, each in their own file. The on-disk message

13does not contain headers generated during an incoming SMTP transaction, such as

14Received and Authentication-Results headers. Those are in the database to

15prevent having to rewrite incoming messages (e.g. Authentication-Result for DKIM

16signatures can only be determined after having read the message). Messages must

17be read through MsgReader, which transparently adds the prefix from the

18database.

19*/

20package store

21

22// todo: make up a function naming scheme that indicates whether caller should broadcast changes.

23

24import (

25 "context"

26 "crypto/md5"

27 cryptorand "crypto/rand"

28 "crypto/sha1"

29 "crypto/sha256"

30 "encoding"

31 "encoding/json"

32 "errors"

33 "fmt"

34 "hash"

35 "io"

36 "log/slog"

37 "os"

38 "path/filepath"

39 "runtime/debug"

40 "slices"

41 "sort"

42 "strconv"

43 "strings"

44 "sync"

45 "time"

46

47 "golang.org/x/crypto/bcrypt"

48 "golang.org/x/text/secure/precis"

49 "golang.org/x/text/unicode/norm"

50

51 "github.com/mjl-/bstore"

52

53 "github.com/mjl-/mox/config"

54 "github.com/mjl-/mox/dns"

55 "github.com/mjl-/mox/message"

56 "github.com/mjl-/mox/metrics"

57 "github.com/mjl-/mox/mlog"

58 "github.com/mjl-/mox/mox-"

59 "github.com/mjl-/mox/moxio"

60 "github.com/mjl-/mox/publicsuffix"

61 "github.com/mjl-/mox/scram"

62 "github.com/mjl-/mox/smtp"

63)

64

65// If true, each time an account is closed its database file is checked for

66// consistency. If an inconsistency is found, panic is called. Set by default

67// because of all the packages with tests, the mox main function sets it to

68// false again.

69var CheckConsistencyOnClose = true

70

71var (

72 ErrUnknownMailbox = errors.New("no such mailbox")

73 ErrUnknownCredentials = errors.New("credentials not found")

74 ErrAccountUnknown = errors.New("no such account")

75 ErrOverQuota = errors.New("account over quota")

76)

77

78var DefaultInitialMailboxes = config.InitialMailboxes{

79 SpecialUse: config.SpecialUseMailboxes{

80 Sent: "Sent",

81 Archive: "Archive",

82 Trash: "Trash",

83 Draft: "Drafts",

84 Junk: "Junk",

85 },

86}

87

88type SCRAM struct {

89 Salt []byte

90 Iterations int

91 SaltedPassword []byte

92}

93

94// CRAMMD5 holds HMAC ipad and opad hashes that are initialized with the first

95// block with (a derivation of) the key/password, so we don't store the password in plain

96// text.

97type CRAMMD5 struct {

98 Ipad hash.Hash

99 Opad hash.Hash

100}

101

102// BinaryMarshal is used by bstore to store the ipad/opad hash states.

103func (c CRAMMD5) MarshalBinary() ([]byte, error) {

104 if c.Ipad == nil || c.Opad == nil {

105 return nil, nil

106 }

107

108 ipad, err := c.Ipad.(encoding.BinaryMarshaler).MarshalBinary()

109 if err != nil {

110 return nil, fmt.Errorf("marshal ipad: %v", err)

111 }

112 opad, err := c.Opad.(encoding.BinaryMarshaler).MarshalBinary()

113 if err != nil {

114 return nil, fmt.Errorf("marshal opad: %v", err)

115 }

116 buf := make([]byte, 2+len(ipad)+len(opad))

117 ipadlen := uint16(len(ipad))

118 buf[0] = byte(ipadlen >> 8)

119 buf[1] = byte(ipadlen >> 0)

120 copy(buf[2:], ipad)

121 copy(buf[2+len(ipad):], opad)

122 return buf, nil

123}

124

125// BinaryUnmarshal is used by bstore to restore the ipad/opad hash states.

126func (c *CRAMMD5) UnmarshalBinary(buf []byte) error {

127 if len(buf) == 0 {

128 *c = CRAMMD5{}

129 return nil

130 }

131 if len(buf) < 2 {

132 return fmt.Errorf("short buffer")

133 }

134 ipadlen := int(uint16(buf[0])<<8 | uint16(buf[1])<<0)

135 if len(buf) < 2+ipadlen {

136 return fmt.Errorf("buffer too short for ipadlen")

137 }

138 ipad := md5.New()

139 opad := md5.New()

140 if err := ipad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2 : 2+ipadlen]); err != nil {

141 return fmt.Errorf("unmarshal ipad: %v", err)

142 }

143 if err := opad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2+ipadlen:]); err != nil {

144 return fmt.Errorf("unmarshal opad: %v", err)

145 }

146 *c = CRAMMD5{ipad, opad}

147 return nil

148}

149

150// Password holds credentials in various forms, for logging in with SMTP/IMAP.

151type Password struct {

152 Hash string // bcrypt hash for IMAP LOGIN, SASL PLAIN and HTTP basic authentication.

153 CRAMMD5 CRAMMD5 // For SASL CRAM-MD5.

154 SCRAMSHA1 SCRAM // For SASL SCRAM-SHA-1.

155 SCRAMSHA256 SCRAM // For SASL SCRAM-SHA-256.

156}

157

158// Subjectpass holds the secret key used to sign subjectpass tokens.

159type Subjectpass struct {

160 Email string // Our destination address (canonical, with catchall localpart stripped).

161 Key string

162}

163

164// NextUIDValidity is a singleton record in the database with the next UIDValidity

165// to use for the next mailbox.

166type NextUIDValidity struct {

167 ID int // Just a single record with ID 1.

168 Next uint32

169}

170

171// SyncState track ModSeqs.

172type SyncState struct {

173 ID int // Just a single record with ID 1.

174

175 // Last used, next assigned will be one higher. The first value we hand out is 2.

176 // That's because 0 (the default value for old existing messages, from before the

177 // Message.ModSeq field) is special in IMAP, so we return it as 1.

178 LastModSeq ModSeq `bstore:"nonzero"`

179

180 // Highest ModSeq of expunged record that we deleted. When a clients synchronizes

181 // and requests changes based on a modseq before this one, we don't have the

182 // history to provide information about deletions. We normally keep these expunged

183 // records around, but we may periodically truly delete them to reclaim storage

184 // space. Initially set to -1 because we don't want to match with any ModSeq in the

185 // database, which can be zero values.

186 HighestDeletedModSeq ModSeq

187}

188

189// Mailbox is collection of messages, e.g. Inbox or Sent.

190type Mailbox struct {

191 ID int64

192

193 // "Inbox" is the name for the special IMAP "INBOX". Slash separated

194 // for hierarchy.

195 Name string `bstore:"nonzero,unique"`

196

197 // If UIDs are invalidated, e.g. when renaming a mailbox to a previously existing

198 // name, UIDValidity must be changed. Used by IMAP for synchronization.

199 UIDValidity uint32

200

201 // UID likely to be assigned to next message. Used by IMAP to detect messages

202 // delivered to a mailbox.

203 UIDNext UID

204

205 SpecialUse

206

207 // Keywords as used in messages. Storing a non-system keyword for a message

208 // automatically adds it to this list. Used in the IMAP FLAGS response. Only

209 // "atoms" are allowed (IMAP syntax), keywords are case-insensitive, only stored in

210 // lower case (for JMAP), sorted.

211 Keywords []string

212

213 HaveCounts bool // Whether MailboxCounts have been initialized.

214 MailboxCounts // Statistics about messages, kept up to date whenever a change happens.

215}

216

217// MailboxCounts tracks statistics about messages for a mailbox.

218type MailboxCounts struct {

219 Total int64 // Total number of messages, excluding \Deleted. For JMAP.

220 Deleted int64 // Number of messages with \Deleted flag. Used for IMAP message count that includes messages with \Deleted.

221 Unread int64 // Messages without \Seen, excluding those with \Deleted, for JMAP.

222 Unseen int64 // Messages without \Seen, including those with \Deleted, for IMAP.

223 Size int64 // Number of bytes for all messages.

224}

225

226func (mc MailboxCounts) String() string {

227 return fmt.Sprintf("%d total, %d deleted, %d unread, %d unseen, size %d bytes", mc.Total, mc.Deleted, mc.Unread, mc.Unseen, mc.Size)

228}

229

230// Add increases mailbox counts mc with those of delta.

231func (mc *MailboxCounts) Add(delta MailboxCounts) {

232 mc.Total += delta.Total

233 mc.Deleted += delta.Deleted

234 mc.Unread += delta.Unread

235 mc.Unseen += delta.Unseen

236 mc.Size += delta.Size

237}

238

239// Add decreases mailbox counts mc with those of delta.

240func (mc *MailboxCounts) Sub(delta MailboxCounts) {

241 mc.Total -= delta.Total

242 mc.Deleted -= delta.Deleted

243 mc.Unread -= delta.Unread

244 mc.Unseen -= delta.Unseen

245 mc.Size -= delta.Size

246}

247

248// SpecialUse identifies a specific role for a mailbox, used by clients to

249// understand where messages should go.

250type SpecialUse struct {

251 Archive bool

252 Draft bool

253 Junk bool

254 Sent bool

255 Trash bool

256}

257

258// CalculateCounts calculates the full current counts for messages in the mailbox.

259func (mb *Mailbox) CalculateCounts(tx *bstore.Tx) (mc MailboxCounts, err error) {

260 q := bstore.QueryTx[Message](tx)

261 q.FilterNonzero(Message{MailboxID: mb.ID})

262 q.FilterEqual("Expunged", false)

263 err = q.ForEach(func(m Message) error {

264 mc.Add(m.MailboxCounts())

265 return nil

266 })

267 return

268}

269

270// ChangeSpecialUse returns a change for special-use flags, for broadcasting to

271// other connections.

272func (mb Mailbox) ChangeSpecialUse() ChangeMailboxSpecialUse {

273 return ChangeMailboxSpecialUse{mb.ID, mb.Name, mb.SpecialUse}

274}

275

276// ChangeKeywords returns a change with new keywords for a mailbox (e.g. after

277// setting a new keyword on a message in the mailbox), for broadcasting to other

278// connections.

279func (mb Mailbox) ChangeKeywords() ChangeMailboxKeywords {

280 return ChangeMailboxKeywords{mb.ID, mb.Name, mb.Keywords}

281}

282

283// KeywordsChanged returns whether the keywords in a mailbox have changed.

284func (mb Mailbox) KeywordsChanged(origmb Mailbox) bool {

285 if len(mb.Keywords) != len(origmb.Keywords) {

286 return true

287 }

288 // Keywords are stored sorted.

289 for i, kw := range mb.Keywords {

290 if origmb.Keywords[i] != kw {

291 return true

292 }

293 }

294 return false

295}

296

297// CountsChange returns a change with mailbox counts.

298func (mb Mailbox) ChangeCounts() ChangeMailboxCounts {

299 return ChangeMailboxCounts{mb.ID, mb.Name, mb.MailboxCounts}

300}

301

302// Subscriptions are separate from existence of mailboxes.

303type Subscription struct {

304 Name string

305}

306

307// Flags for a mail message.

308type Flags struct {

309 Seen bool

310 Answered bool

311 Flagged bool

312 Forwarded bool

313 Junk bool

314 Notjunk bool

315 Deleted bool

316 Draft bool

317 Phishing bool

318 MDNSent bool

319}

320

321// FlagsAll is all flags set, for use as mask.

322var FlagsAll = Flags{true, true, true, true, true, true, true, true, true, true}

323

324// Validation of "message From" domain.

325type Validation uint8

326

327const (

328 ValidationUnknown Validation = 0

329 ValidationStrict Validation = 1 // Like DMARC, with strict policies.

330 ValidationDMARC Validation = 2 // Actual DMARC policy.

331 ValidationRelaxed Validation = 3 // Like DMARC, with relaxed policies.

332 ValidationPass Validation = 4 // For SPF.

333 ValidationNeutral Validation = 5 // For SPF.

334 ValidationTemperror Validation = 6

335 ValidationPermerror Validation = 7

336 ValidationFail Validation = 8

337 ValidationSoftfail Validation = 9 // For SPF.

338 ValidationNone Validation = 10 // E.g. No records.

339)

340

341// Message stored in database and per-message file on disk.

342//

343// Contents are always the combined data from MsgPrefix and the on-disk file named

344// based on ID.

345//

346// Messages always have a header section, even if empty. Incoming messages without

347// header section must get an empty header section added before inserting.

348type Message struct {

349 // ID, unchanged over lifetime, determines path to on-disk msg file.

350 // Set during deliver.

351 ID int64

352

353 UID UID `bstore:"nonzero"` // UID, for IMAP. Set during deliver.

354 MailboxID int64 `bstore:"nonzero,unique MailboxID+UID,index MailboxID+Received,index MailboxID+ModSeq,ref Mailbox"`

355

356 // Modification sequence, for faster syncing with IMAP QRESYNC and JMAP.

357 // ModSeq is the last modification. CreateSeq is the Seq the message was inserted,

358 // always <= ModSeq. If Expunged is set, the message has been removed and should not

359 // be returned to the user. In this case, ModSeq is the Seq where the message is

360 // removed, and will never be changed again.

361 // We have an index on both ModSeq (for JMAP that synchronizes per account) and

362 // MailboxID+ModSeq (for IMAP that synchronizes per mailbox).

363 // The index on CreateSeq helps efficiently finding created messages for JMAP.

364 // The value of ModSeq is special for IMAP. Messages that existed before ModSeq was

365 // added have 0 as value. But modseq 0 in IMAP is special, so we return it as 1. If

366 // we get modseq 1 from a client, the IMAP server will translate it to 0. When we

367 // return modseq to clients, we turn 0 into 1.

368 ModSeq ModSeq `bstore:"index"`

369 CreateSeq ModSeq `bstore:"index"`

370 Expunged bool

371

372 // If set, this message was delivered to a Rejects mailbox. When it is moved to a

373 // different mailbox, its MailboxOrigID is set to the destination mailbox and this

374 // flag cleared.

375 IsReject bool

376

377 // If set, this is a forwarded message (through a ruleset with IsForward). This

378 // causes fields used during junk analysis to be moved to their Orig variants, and

379 // masked IP fields cleared, so they aren't used in junk classifications for

380 // incoming messages. This ensures the forwarded messages don't cause negative

381 // reputation for the forwarding mail server, which may also be sending regular

382 // messages.

383 IsForward bool

384

385 // MailboxOrigID is the mailbox the message was originally delivered to. Typically

386 // Inbox or Rejects, but can also be a mailbox configured in a Ruleset, or

387 // Postmaster, TLS/DMARC reporting addresses. MailboxOrigID is not changed when the

388 // message is moved to another mailbox, e.g. Archive/Trash/Junk. Used for

389 // per-mailbox reputation.

390 //

391 // MailboxDestinedID is normally 0, but when a message is delivered to the Rejects

392 // mailbox, it is set to the intended mailbox according to delivery rules,

393 // typically that of Inbox. When such a message is moved out of Rejects, the

394 // MailboxOrigID is corrected by setting it to MailboxDestinedID. This ensures the

395 // message is used for reputation calculation for future deliveries to that

396 // mailbox.

397 //

398 // These are not bstore references to prevent having to update all messages in a

399 // mailbox when the original mailbox is removed. Use of these fields requires

400 // checking if the mailbox still exists.

401 MailboxOrigID int64

402 MailboxDestinedID int64

403

404 Received time.Time `bstore:"default now,index"`

405

406 // Full IP address of remote SMTP server. Empty if not delivered over SMTP. The

407 // masked IPs are used to classify incoming messages. They are left empty for

408 // messages matching a ruleset for forwarded messages.

409 RemoteIP string

410 RemoteIPMasked1 string `bstore:"index RemoteIPMasked1+Received"` // For IPv4 /32, for IPv6 /64, for reputation.

411 RemoteIPMasked2 string `bstore:"index RemoteIPMasked2+Received"` // For IPv4 /26, for IPv6 /48.

412 RemoteIPMasked3 string `bstore:"index RemoteIPMasked3+Received"` // For IPv4 /21, for IPv6 /32.

413

414 // Only set if present and not an IP address. Unicode string. Empty for forwarded

415 // messages.

416 EHLODomain string `bstore:"index EHLODomain+Received"`

417 MailFrom string // With localpart and domain. Can be empty.

418 MailFromLocalpart smtp.Localpart // SMTP "MAIL FROM", can be empty.

419 // Only set if it is a domain, not an IP. Unicode string. Empty for forwarded

420 // messages, but see OrigMailFromDomain.

421 MailFromDomain string `bstore:"index MailFromDomain+Received"`

422 RcptToLocalpart smtp.Localpart // SMTP "RCPT TO", can be empty.

423 RcptToDomain string // Unicode string.

424

425 // Parsed "From" message header, used for reputation along with domain validation.

426 MsgFromLocalpart smtp.Localpart

427 MsgFromDomain string `bstore:"index MsgFromDomain+Received"` // Unicode string.

428 MsgFromOrgDomain string `bstore:"index MsgFromOrgDomain+Received"` // Unicode string.

429

430 // Simplified statements of the Validation fields below, used for incoming messages

431 // to check reputation.

432 EHLOValidated bool

433 MailFromValidated bool

434 MsgFromValidated bool

435

436 EHLOValidation Validation // Validation can also take reverse IP lookup into account, not only SPF.

437 MailFromValidation Validation // Can have SPF-specific validations like ValidationSoftfail.

438 MsgFromValidation Validation // Desirable validations: Strict, DMARC, Relaxed. Will not be just Pass.

439

440 // Domains with verified DKIM signatures. Unicode string. For forwarded messages, a

441 // DKIM domain that matched a ruleset's verified domain is left out, but included

442 // in OrigDKIMDomains.

443 DKIMDomains []string `bstore:"index DKIMDomains+Received"`

444

445 // For forwarded messages,

446 OrigEHLODomain string

447 OrigDKIMDomains []string

448

449 // Canonicalized Message-Id, always lower-case and normalized quoting, without

450 // <>'s. Empty if missing. Used for matching message threads, and to prevent

451 // duplicate reject delivery.

452 MessageID string `bstore:"index"`

453 // lower-case: ../rfc/5256:495

454

455 // For matching threads in case there is no References/In-Reply-To header. It is

456 // lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed.

457 SubjectBase string `bstore:"index"`

458 // ../rfc/5256:90

459

460 // Hash of message. For rejects delivery in case there is no Message-ID, only set

461 // when delivered as reject.

462 MessageHash []byte

463

464 // ID of message starting this thread.

465 ThreadID int64 `bstore:"index"`

466 // IDs of parent messages, from closest parent to the root message. Parent messages

467 // may be in a different mailbox, or may no longer exist. ThreadParentIDs must

468 // never contain the message id itself (a cycle), and parent messages must

469 // reference the same ancestors.

470 ThreadParentIDs []int64

471 // ThreadMissingLink is true if there is no match with a direct parent. E.g. first

472 // ID in ThreadParentIDs is not the direct ancestor (an intermediate message may

473 // have been deleted), or subject-based matching was done.

474 ThreadMissingLink bool

475 // If set, newly delivered child messages are automatically marked as read. This

476 // field is copied to new child messages. Changes are propagated to the webmail

477 // client.

478 ThreadMuted bool

479 // If set, this (sub)thread is collapsed in the webmail client, for threading mode

480 // "on" (mode "unread" ignores it). This field is copied to new child message.

481 // Changes are propagated to the webmail client.

482 ThreadCollapsed bool

483

484 // If received message was known to match a mailing list rule (with modified junk

485 // filtering).

486 IsMailingList bool

487

488 // If this message is a DSN. For DSNs, we don't look at the subject when matching

489 // threads.

490 DSN bool

491

492 ReceivedTLSVersion uint16 // 0 if unknown, 1 if plaintext/no TLS, otherwise TLS cipher suite.

493 ReceivedTLSCipherSuite uint16

494 ReceivedRequireTLS bool // Whether RequireTLS was known to be used for incoming delivery.

495

496 Flags

497 // For keywords other than system flags or the basic well-known $-flags. Only in

498 // "atom" syntax (IMAP), they are case-insensitive, always stored in lower-case

499 // (for JMAP), sorted.

500 Keywords []string `bstore:"index"`

501 Size int64

502 TrainedJunk *bool // If nil, no training done yet. Otherwise, true is trained as junk, false trained as nonjunk.

503 MsgPrefix []byte // Typically holds received headers and/or header separator.

504

505 // ParsedBuf message structure. Currently saved as JSON of message.Part because bstore

506 // cannot yet store recursive types. Created when first needed, and saved in the

507 // database.

508 // todo: once replaced with non-json storage, remove date fixup in ../message/part.go.

509 ParsedBuf []byte

510}

511

512// MailboxCounts returns the delta to counts this message means for its

513// mailbox.

514func (m Message) MailboxCounts() (mc MailboxCounts) {

515 if m.Expunged {

516 return

517 }

518 if m.Deleted {

519 mc.Deleted++

520 } else {

521 mc.Total++

522 }

523 if !m.Seen {

524 mc.Unseen++

525 if !m.Deleted {

526 mc.Unread++

527 }

528 }

529 mc.Size += m.Size

530 return

531}

532

533func (m Message) ChangeAddUID() ChangeAddUID {

534 return ChangeAddUID{m.MailboxID, m.UID, m.ModSeq, m.Flags, m.Keywords}

535}

536

537func (m Message) ChangeFlags(orig Flags) ChangeFlags {

538 mask := m.Flags.Changed(orig)

539 return ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, ModSeq: m.ModSeq, Mask: mask, Flags: m.Flags, Keywords: m.Keywords}

540}

541

542func (m Message) ChangeThread() ChangeThread {

543 return ChangeThread{[]int64{m.ID}, m.ThreadMuted, m.ThreadCollapsed}

544}

545

546// ModSeq represents a modseq as stored in the database. ModSeq 0 in the

547// database is sent to the client as 1, because modseq 0 is special in IMAP.

548// ModSeq coming from the client are of type int64.

549type ModSeq int64

550

551func (ms ModSeq) Client() int64 {

552 if ms == 0 {

553 return 1

554 }

555 return int64(ms)

556}

557

558// ModSeqFromClient converts a modseq from a client to a modseq for internal

559// use, e.g. in a database query.

560// ModSeq 1 is turned into 0 (the Go zero value for ModSeq).

561func ModSeqFromClient(modseq int64) ModSeq {

562 if modseq == 1 {

563 return 0

564 }

565 return ModSeq(modseq)

566}

567

568// PrepareExpunge clears fields that are no longer needed after an expunge, so

569// almost all fields. Does not change ModSeq, but does set Expunged.

570func (m *Message) PrepareExpunge() {

571 *m = Message{

572 ID: m.ID,

573 UID: m.UID,

574 MailboxID: m.MailboxID,

575 CreateSeq: m.CreateSeq,

576 ModSeq: m.ModSeq,

577 Expunged: true,

578 ThreadID: m.ThreadID,

579 }

580}

581

582// PrepareThreading sets MessageID, SubjectBase and DSN (used in threading) based

583// on the part.

584func (m *Message) PrepareThreading(log mlog.Log, part *message.Part) {

585 m.DSN = part.IsDSN()

586

587 if part.Envelope == nil {

588 return

589 }

590 messageID, raw, err := message.MessageIDCanonical(part.Envelope.MessageID)

591 if err != nil {

592 log.Debugx("parsing message-id, ignoring", err, slog.String("messageid", part.Envelope.MessageID))

593 } else if raw {

594 log.Debug("could not parse message-id as address, continuing with raw value", slog.String("messageid", part.Envelope.MessageID))

595 }

596 m.MessageID = messageID

597 m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false)

598}

599

600// LoadPart returns a message.Part by reading from m.ParsedBuf.

601func (m Message) LoadPart(r io.ReaderAt) (message.Part, error) {

602 if m.ParsedBuf == nil {

603 return message.Part{}, fmt.Errorf("message not parsed")

604 }

605 var p message.Part

606 err := json.Unmarshal(m.ParsedBuf, &p)

607 if err != nil {

608 return p, fmt.Errorf("unmarshal message part")

609 }

610 p.SetReaderAt(r)

611 return p, nil

612}

613

614// NeedsTraining returns whether message needs a training update, based on

615// TrainedJunk (current training status) and new Junk/Notjunk flags.

616func (m Message) NeedsTraining() bool {

617 untrain := m.TrainedJunk != nil

618 untrainJunk := untrain && *m.TrainedJunk

619 train := m.Junk || m.Notjunk && !(m.Junk && m.Notjunk)

620 trainJunk := m.Junk

621 return untrain != train || untrain && train && untrainJunk != trainJunk

622}

623

624// JunkFlagsForMailbox sets Junk and Notjunk flags based on mailbox name if configured. Often

625// used when delivering/moving/copying messages to a mailbox. Mail clients are not

626// very helpful with setting junk/notjunk flags. But clients can move/copy messages

627// to other mailboxes. So we set flags when clients move a message.

628func (m *Message) JunkFlagsForMailbox(mb Mailbox, conf config.Account) {

629 if mb.Junk {

630 m.Junk = true

631 m.Notjunk = false

632 return

633 }

634

635 if !conf.AutomaticJunkFlags.Enabled {

636 return

637 }

638

639 lmailbox := strings.ToLower(mb.Name)

640

641 if conf.JunkMailbox != nil && conf.JunkMailbox.MatchString(lmailbox) {

642 m.Junk = true

643 m.Notjunk = false

644 } else if conf.NeutralMailbox != nil && conf.NeutralMailbox.MatchString(lmailbox) {

645 m.Junk = false

646 m.Notjunk = false

647 } else if conf.NotJunkMailbox != nil && conf.NotJunkMailbox.MatchString(lmailbox) {

648 m.Junk = false

649 m.Notjunk = true

650 } else if conf.JunkMailbox == nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox != nil {

651 m.Junk = true

652 m.Notjunk = false

653 } else if conf.JunkMailbox != nil && conf.NeutralMailbox == nil && conf.NotJunkMailbox != nil {

654 m.Junk = false

655 m.Notjunk = false

656 } else if conf.JunkMailbox != nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox == nil {

657 m.Junk = false

658 m.Notjunk = true

659 }

660}

661

662// Recipient represents the recipient of a message. It is tracked to allow

663// first-time incoming replies from users this account has sent messages to. When a

664// mailbox is added to the Sent mailbox the message is parsed and recipients are

665// inserted as recipient. Recipients are never removed other than for removing the

666// message. On move/copy of a message, recipients aren't modified either. For IMAP,

667// this assumes a client simply appends messages to the Sent mailbox (as opposed to

668// copying messages from some place).

669type Recipient struct {

670 ID int64

671 MessageID int64 `bstore:"nonzero,ref Message"` // Ref gives it its own index, useful for fast removal as well.

672 Localpart string `bstore:"nonzero"` // Encoded localpart.

673 Domain string `bstore:"nonzero,index Domain+Localpart"` // Unicode string.

674 OrgDomain string `bstore:"nonzero,index"` // Unicode string.

675 Sent time.Time `bstore:"nonzero"`

676}

677

678// Outgoing is a message submitted for delivery from the queue. Used to enforce

679// maximum outgoing messages.

680type Outgoing struct {

681 ID int64

682 Recipient string `bstore:"nonzero,index"` // Canonical international address with utf8 domain.

683 Submitted time.Time `bstore:"nonzero,default now"`

684}

685

686// RecipientDomainTLS stores TLS capabilities of a recipient domain as encountered

687// during most recent connection (delivery attempt).

688type RecipientDomainTLS struct {

689 Domain string // Unicode.

690 Updated time.Time `bstore:"default now"`

691 STARTTLS bool // Supports STARTTLS.

692 RequireTLS bool // Supports RequireTLS SMTP extension.

693}

694

695// DiskUsage tracks quota use.

696type DiskUsage struct {

697 ID int64 // Always one record with ID 1.

698 MessageSize int64 // Sum of all messages, for quota accounting.

699}

700

701// SessionToken and CSRFToken are types to prevent mixing them up.

702// Base64 raw url encoded.

703type SessionToken string

704type CSRFToken string

705

706// LoginSession represents a login session. We keep a limited number of sessions

707// for a user, removing the oldest session when a new one is created.

708type LoginSession struct {

709 ID int64

710 Created time.Time `bstore:"nonzero,default now"` // Of original login.

711 Expires time.Time `bstore:"nonzero"` // Extended each time it is used.

712 SessionTokenBinary [16]byte `bstore:"nonzero"` // Stored in cookie, like "webmailsession" or "webaccountsession".

713 CSRFTokenBinary [16]byte // For API requests, in "x-mox-csrf" header.

714 AccountName string `bstore:"nonzero"`

715 LoginAddress string `bstore:"nonzero"`

716

717 // Set when loading from database.

718 sessionToken SessionToken

719 csrfToken CSRFToken

720}

721

722// Types stored in DB.

723var DBTypes = []any{NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{}, SyncState{}, Upgrade{}, RecipientDomainTLS{}, DiskUsage{}, LoginSession{}}

724

725// Account holds the information about a user, includings mailboxes, messages, imap subscriptions.

726type Account struct {

727 Name string // Name, according to configuration.

728 Dir string // Directory where account files, including the database, bloom filter, and mail messages, are stored for this account.

729 DBPath string // Path to database with mailboxes, messages, etc.

730 DB *bstore.DB // Open database connection.

731

732 // Channel that is closed if/when account has/gets "threads" accounting (see

733 // Upgrade.Threads).

734 threadsCompleted chan struct{}

735 // If threads upgrade completed with error, this is set. Used for warning during

736 // delivery, or aborting when importing.

737 threadsErr error

738

739 // Write lock must be held for account/mailbox modifications including message delivery.

740 // Read lock for reading mailboxes/messages.

741 // When making changes to mailboxes/messages, changes must be broadcasted before

742 // releasing the lock to ensure proper UID ordering.

743 sync.RWMutex

744

745 nused int // Reference count, while >0, this account is alive and shared.

746}

747

748type Upgrade struct {

749 ID byte

750 Threads byte // 0: None, 1: Adding MessageID's completed, 2: Adding ThreadID's completed.

751}

752

753// InitialUIDValidity returns a UIDValidity used for initializing an account.

754// It can be replaced during tests with a predictable value.

755var InitialUIDValidity = func() uint32 {

756 return uint32(time.Now().Unix() >> 1) // A 2-second resolution will get us far enough beyond 2038.

757}

758

759var openAccounts = struct {

760 names map[string]*Account

761 sync.Mutex

762}{

763 names: map[string]*Account{},

764}

765

766func closeAccount(acc *Account) (rerr error) {

767 openAccounts.Lock()

768 acc.nused--

769 defer openAccounts.Unlock()

770 if acc.nused == 0 {

771 // threadsCompleted must be closed now because it increased nused.

772 rerr = acc.DB.Close()

773 acc.DB = nil

774 delete(openAccounts.names, acc.Name)

775 }

776 return

777}

778

779// OpenAccount opens an account by name.

780//

781// No additional data path prefix or ".db" suffix should be added to the name.

782// A single shared account exists per name.

783func OpenAccount(log mlog.Log, name string) (*Account, error) {

784 openAccounts.Lock()

785 defer openAccounts.Unlock()

786 if acc, ok := openAccounts.names[name]; ok {

787 acc.nused++

788 return acc, nil

789 }

790

791 if _, ok := mox.Conf.Account(name); !ok {

792 return nil, ErrAccountUnknown

793 }

794

795 acc, err := openAccount(log, name)

796 if err != nil {

797 return nil, err

798 }

799 openAccounts.names[name] = acc

800 return acc, nil

801}

802

803// openAccount opens an existing account, or creates it if it is missing.

804func openAccount(log mlog.Log, name string) (a *Account, rerr error) {

805 dir := filepath.Join(mox.DataDirPath("accounts"), name)

806 return OpenAccountDB(log, dir, name)

807}

808

809// OpenAccountDB opens an account database file and returns an initialized account

810// or error. Only exported for use by subcommands that verify the database file.

811// Almost all account opens must go through OpenAccount/OpenEmail/OpenEmailAuth.

812func OpenAccountDB(log mlog.Log, accountDir, accountName string) (a *Account, rerr error) {

813 dbpath := filepath.Join(accountDir, "index.db")

814

815 // Create account if it doesn't exist yet.

816 isNew := false

817 if _, err := os.Stat(dbpath); err != nil && os.IsNotExist(err) {

818 isNew = true

819 os.MkdirAll(accountDir, 0770)

820 }

821

822 db, err := bstore.Open(context.TODO(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)

823 if err != nil {

824 return nil, err

825 }

826

827 defer func() {

828 if rerr != nil {

829 db.Close()

830 if isNew {

831 os.Remove(dbpath)

832 }

833 }

834 }()

835

836 acc := &Account{

837 Name: accountName,

838 Dir: accountDir,

839 DBPath: dbpath,

840 DB: db,

841 nused: 1,

842 threadsCompleted: make(chan struct{}),

843 }

844

845 if isNew {

846 if err := initAccount(db); err != nil {

847 return nil, fmt.Errorf("initializing account: %v", err)

848 }

849 close(acc.threadsCompleted)

850 return acc, nil

851 }

852

853 // Ensure mailbox counts and total message size are set.

854 var mentioned bool

855 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

856 err := bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error {

857 if !mentioned {

858 mentioned = true

859 log.Info("first calculation of mailbox counts for account", slog.String("account", accountName))

860 }

861 mc, err := mb.CalculateCounts(tx)

862 if err != nil {

863 return err

864 }

865 mb.HaveCounts = true

866 mb.MailboxCounts = mc

867 return tx.Update(&mb)

868 })

869 if err != nil {

870 return err

871 }

872

873 du := DiskUsage{ID: 1}

874 err = tx.Get(&du)

875 if err == nil || !errors.Is(err, bstore.ErrAbsent) {

876 return err

877 }

878 // No DiskUsage record yet, calculate total size and insert.

879 err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {

880 du.MessageSize += mb.Size

881 return nil

882 })

883 if err != nil {

884 return err

885 }

886 return tx.Insert(&du)

887 })

888 if err != nil {

889 return nil, fmt.Errorf("calculating counts for mailbox: %v", err)

890 }

891

892 // Start adding threading if needed.

893 up := Upgrade{ID: 1}

894 err = db.Write(context.TODO(), func(tx *bstore.Tx) error {

895 err := tx.Get(&up)

896 if err == bstore.ErrAbsent {

897 if err := tx.Insert(&up); err != nil {

898 return fmt.Errorf("inserting initial upgrade record: %v", err)

899 }

900 err = nil

901 }

902 return err

903 })

904 if err != nil {

905 return nil, fmt.Errorf("checking message threading: %v", err)

906 }

907 if up.Threads == 2 {

908 close(acc.threadsCompleted)

909 return acc, nil

910 }

911

912 // Increase account use before holding on to account in background.

913 // Caller holds the lock. The goroutine below decreases nused by calling

914 // closeAccount.

915 acc.nused++

916

917 // Ensure all messages have a MessageID and SubjectBase, which are needed when

918 // matching threads.

919 // Then assign messages to threads, in the same way we do during imports.

920 log.Info("upgrading account for threading, in background", slog.String("account", acc.Name))

921 go func() {

922 defer func() {

923 err := closeAccount(acc)

924 log.Check(err, "closing use of account after upgrading account storage for threads", slog.String("account", a.Name))

925 }()

926

927 defer func() {

928 x := recover() // Should not happen, but don't take program down if it does.

929 if x != nil {

930 log.Error("upgradeThreads panic", slog.Any("err", x))

931 debug.PrintStack()

932 metrics.PanicInc(metrics.Upgradethreads)

933 acc.threadsErr = fmt.Errorf("panic during upgradeThreads: %v", x)

934 }

935

936 // Mark that upgrade has finished, possibly error is indicated in threadsErr.

937 close(acc.threadsCompleted)

938 }()

939

940 err := upgradeThreads(mox.Shutdown, log, acc, &up)

941 if err != nil {

942 a.threadsErr = err

943 log.Errorx("upgrading account for threading, aborted", err, slog.String("account", a.Name))

944 } else {

945 log.Info("upgrading account for threading, completed", slog.String("account", a.Name))

946 }

947 }()

948 return acc, nil

949}

950

951// ThreadingWait blocks until the one-time account threading upgrade for the

952// account has completed, and returns an error if not successful.

953//

954// To be used before starting an import of messages.

955func (a *Account) ThreadingWait(log mlog.Log) error {

956 select {

957 case <-a.threadsCompleted:

958 return a.threadsErr

959 default:

960 }

961 log.Debug("waiting for account upgrade to complete")

962

963 <-a.threadsCompleted

964 return a.threadsErr

965}

966

967func initAccount(db *bstore.DB) error {

968 return db.Write(context.TODO(), func(tx *bstore.Tx) error {

969 uidvalidity := InitialUIDValidity()

970

971 if err := tx.Insert(&Upgrade{ID: 1, Threads: 2}); err != nil {

972 return err

973 }

974 if err := tx.Insert(&DiskUsage{ID: 1}); err != nil {

975 return err

976 }

977

978 if len(mox.Conf.Static.DefaultMailboxes) > 0 {

979 // Deprecated in favor of InitialMailboxes.

980 defaultMailboxes := mox.Conf.Static.DefaultMailboxes

981 mailboxes := []string{"Inbox"}

982 for _, name := range defaultMailboxes {

983 if strings.EqualFold(name, "Inbox") {

984 continue

985 }

986 mailboxes = append(mailboxes, name)

987 }

988 for _, name := range mailboxes {

989 mb := Mailbox{Name: name, UIDValidity: uidvalidity, UIDNext: 1, HaveCounts: true}

990 if strings.HasPrefix(name, "Archive") {

991 mb.Archive = true

992 } else if strings.HasPrefix(name, "Drafts") {

993 mb.Draft = true

994 } else if strings.HasPrefix(name, "Junk") {

995 mb.Junk = true

996 } else if strings.HasPrefix(name, "Sent") {

997 mb.Sent = true

998 } else if strings.HasPrefix(name, "Trash") {

999 mb.Trash = true

1000 }

1001 if err := tx.Insert(&mb); err != nil {

1002 return fmt.Errorf("creating mailbox: %w", err)

1003 }

1004 if err := tx.Insert(&Subscription{name}); err != nil {

1005 return fmt.Errorf("adding subscription: %w", err)

1006 }

1007 }

1008 } else {

1009 mailboxes := mox.Conf.Static.InitialMailboxes

1010 var zerouse config.SpecialUseMailboxes

1011 if mailboxes.SpecialUse == zerouse && len(mailboxes.Regular) == 0 {

1012 mailboxes = DefaultInitialMailboxes

1013 }

1014

1015 add := func(name string, use SpecialUse) error {

1016 mb := Mailbox{Name: name, UIDValidity: uidvalidity, UIDNext: 1, SpecialUse: use, HaveCounts: true}

1017 if err := tx.Insert(&mb); err != nil {

1018 return fmt.Errorf("creating mailbox: %w", err)

1019 }

1020 if err := tx.Insert(&Subscription{name}); err != nil {

1021 return fmt.Errorf("adding subscription: %w", err)

1022 }

1023 return nil

1024 }

1025 addSpecialOpt := func(nameOpt string, use SpecialUse) error {

1026 if nameOpt == "" {

1027 return nil

1028 }

1029 return add(nameOpt, use)

1030 }

1031 l := []struct {

1032 nameOpt string

1033 use SpecialUse

1034 }{

1035 {"Inbox", SpecialUse{}},

1036 {mailboxes.SpecialUse.Archive, SpecialUse{Archive: true}},

1037 {mailboxes.SpecialUse.Draft, SpecialUse{Draft: true}},

1038 {mailboxes.SpecialUse.Junk, SpecialUse{Junk: true}},

1039 {mailboxes.SpecialUse.Sent, SpecialUse{Sent: true}},

1040 {mailboxes.SpecialUse.Trash, SpecialUse{Trash: true}},

1041 }

1042 for _, e := range l {

1043 if err := addSpecialOpt(e.nameOpt, e.use); err != nil {

1044 return err

1045 }

1046 }

1047 for _, name := range mailboxes.Regular {

1048 if err := add(name, SpecialUse{}); err != nil {

1049 return err

1050 }

1051 }

1052 }

1053

1054 uidvalidity++

1055 if err := tx.Insert(&NextUIDValidity{1, uidvalidity}); err != nil {

1056 return fmt.Errorf("inserting nextuidvalidity: %w", err)

1057 }

1058 return nil

1059 })

1060}

1061

1062// Close reduces the reference count, and closes the database connection when

1063// it was the last user.

1064func (a *Account) Close() error {

1065 if CheckConsistencyOnClose {

1066 xerr := a.CheckConsistency()

1067 err := closeAccount(a)

1068 if xerr != nil {

1069 panic(xerr)

1070 }

1071 return err

1072 }

1073 return closeAccount(a)

1074}

1075

1076// CheckConsistency checks the consistency of the database and returns a non-nil

1077// error for these cases:

1078//

1079// - Missing on-disk file for message.

1080// - Mismatch between message size and length of MsgPrefix and on-disk file.

1081// - Missing HaveCounts.

1082// - Incorrect mailbox counts.

1083// - Incorrect total message size.

1084// - Message with UID >= mailbox uid next.

1085// - Mailbox uidvalidity >= account uid validity.

1086// - ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq.

1087// - All messages have a nonzero ThreadID, and no cycles in ThreadParentID, and parent messages the same ThreadParentIDs tail.

1088func (a *Account) CheckConsistency() error {

1089 var uidErrors []string // With a limit, could be many.

1090 var modseqErrors []string // With limit.

1091 var fileErrors []string // With limit.

1092 var threadidErrors []string // With limit.

1093 var threadParentErrors []string // With limit.

1094 var threadAncestorErrors []string // With limit.

1095 var errors []string

1096

1097 err := a.DB.Read(context.Background(), func(tx *bstore.Tx) error {

1098 nuv := NextUIDValidity{ID: 1}

1099 err := tx.Get(&nuv)

1100 if err != nil {

1101 return fmt.Errorf("fetching next uid validity: %v", err)

1102 }

1103

1104 mailboxes := map[int64]Mailbox{}

1105 err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {

1106 mailboxes[mb.ID] = mb

1107

1108 if mb.UIDValidity >= nuv.Next {

1109 errmsg := fmt.Sprintf("mailbox %q (id %d) has uidvalidity %d >= account next uidvalidity %d", mb.Name, mb.ID, mb.UIDValidity, nuv.Next)

1110 errors = append(errors, errmsg)

1111 }

1112 return nil

1113 })

1114 if err != nil {

1115 return fmt.Errorf("listing mailboxes: %v", err)

1116 }

1117

1118 counts := map[int64]MailboxCounts{}

1119 err = bstore.QueryTx[Message](tx).ForEach(func(m Message) error {

1120 mc := counts[m.MailboxID]

1121 mc.Add(m.MailboxCounts())

1122 counts[m.MailboxID] = mc

1123

1124 mb := mailboxes[m.MailboxID]

1125

1126 if (m.ModSeq == 0 || m.CreateSeq == 0 || m.CreateSeq > m.ModSeq) && len(modseqErrors) < 20 {

1127 modseqerr := fmt.Sprintf("message %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and createseq <= modseq", m.ID, mb.Name, mb.ID, m.ModSeq, m.CreateSeq)

1128 modseqErrors = append(modseqErrors, modseqerr)

1129 }

1130 if m.UID >= mb.UIDNext && len(uidErrors) < 20 {

1131 uiderr := fmt.Sprintf("message %d in mailbox %q (id %d) has uid %d >= mailbox uidnext %d", m.ID, mb.Name, mb.ID, m.UID, mb.UIDNext)

1132 uidErrors = append(uidErrors, uiderr)

1133 }

1134 if m.Expunged {

1135 return nil

1136 }

1137 p := a.MessagePath(m.ID)

1138 st, err := os.Stat(p)

1139 if err != nil {

1140 existserr := fmt.Sprintf("message %d in mailbox %q (id %d) on-disk file %s: %v", m.ID, mb.Name, mb.ID, p, err)

1141 fileErrors = append(fileErrors, existserr)

1142 } else if len(fileErrors) < 20 && m.Size != int64(len(m.MsgPrefix))+st.Size() {

1143 sizeerr := fmt.Sprintf("message %d in mailbox %q (id %d) has size %d != len msgprefix %d + on-disk file size %d = %d", m.ID, mb.Name, mb.ID, m.Size, len(m.MsgPrefix), st.Size(), int64(len(m.MsgPrefix))+st.Size())

1144 fileErrors = append(fileErrors, sizeerr)

1145 }

1146

1147 if m.ThreadID <= 0 && len(threadidErrors) < 20 {

1148 err := fmt.Sprintf("message %d in mailbox %q (id %d) has threadid 0", m.ID, mb.Name, mb.ID)

1149 threadidErrors = append(threadidErrors, err)

1150 }

1151 if slices.Contains(m.ThreadParentIDs, m.ID) && len(threadParentErrors) < 20 {

1152 err := fmt.Sprintf("message %d in mailbox %q (id %d) references itself in threadparentids", m.ID, mb.Name, mb.ID)

1153 threadParentErrors = append(threadParentErrors, err)

1154 }

1155 for i, pid := range m.ThreadParentIDs {

1156 am := Message{ID: pid}

1157 if err := tx.Get(&am); err == bstore.ErrAbsent {

1158 continue

1159 } else if err != nil {

1160 return fmt.Errorf("get ancestor message: %v", err)

1161 } else if !slices.Equal(m.ThreadParentIDs[i+1:], am.ThreadParentIDs) && len(threadAncestorErrors) < 20 {

1162 err := fmt.Sprintf("message %d, thread %d has ancestor ids %v, and ancestor at index %d with id %d should have the same tail but has %v\n", m.ID, m.ThreadID, m.ThreadParentIDs, i, am.ID, am.ThreadParentIDs)

1163 threadAncestorErrors = append(threadAncestorErrors, err)

1164 } else {

1165 break

1166 }

1167 }

1168 return nil

1169 })

1170 if err != nil {

1171 return fmt.Errorf("reading messages: %v", err)

1172 }

1173

1174 var totalSize int64

1175 for _, mb := range mailboxes {

1176 totalSize += mb.Size

1177 if !mb.HaveCounts {

1178 errmsg := fmt.Sprintf("mailbox %q (id %d) does not have counts, should be %#v", mb.Name, mb.ID, counts[mb.ID])

1179 errors = append(errors, errmsg)

1180 } else if mb.MailboxCounts != counts[mb.ID] {

1181 mbcounterr := fmt.Sprintf("mailbox %q (id %d) has wrong counts %s, should be %s", mb.Name, mb.ID, mb.MailboxCounts, counts[mb.ID])

1182 errors = append(errors, mbcounterr)

1183 }

1184 }

1185

1186 du := DiskUsage{ID: 1}

1187 if err := tx.Get(&du); err != nil {

1188 return fmt.Errorf("get diskusage")

1189 }

1190 if du.MessageSize != totalSize {

1191 errmsg := fmt.Sprintf("total message size in database is %d, sum of mailbox message sizes is %d", du.MessageSize, totalSize)

1192 errors = append(errors, errmsg)

1193 }

1194

1195 return nil

1196 })

1197 if err != nil {

1198 return err

1199 }

1200 errors = append(errors, uidErrors...)

1201 errors = append(errors, modseqErrors...)

1202 errors = append(errors, fileErrors...)

1203 errors = append(errors, threadidErrors...)

1204 errors = append(errors, threadParentErrors...)

1205 errors = append(errors, threadAncestorErrors...)

1206 if len(errors) > 0 {

1207 return fmt.Errorf("%s", strings.Join(errors, "; "))

1208 }

1209 return nil

1210}

1211

1212// Conf returns the configuration for this account if it still exists. During

1213// an SMTP session, a configuration update may drop an account.

1214func (a *Account) Conf() (config.Account, bool) {

1215 return mox.Conf.Account(a.Name)

1216}

1217

1218// NextUIDValidity returns the next new/unique uidvalidity to use for this account.

1219func (a *Account) NextUIDValidity(tx *bstore.Tx) (uint32, error) {

1220 nuv := NextUIDValidity{ID: 1}

1221 if err := tx.Get(&nuv); err != nil {

1222 return 0, err

1223 }

1224 v := nuv.Next

1225 nuv.Next++

1226 if err := tx.Update(&nuv); err != nil {

1227 return 0, err

1228 }

1229 return v, nil

1230}

1231

1232// NextModSeq returns the next modification sequence, which is global per account,

1233// over all types.

1234func (a *Account) NextModSeq(tx *bstore.Tx) (ModSeq, error) {

1235 v := SyncState{ID: 1}

1236 if err := tx.Get(&v); err == bstore.ErrAbsent {

1237 // We start assigning from modseq 2. Modseq 0 is not usable, so returned as 1, so

1238 // already used.

1239 // HighestDeletedModSeq is -1 so comparison against the default ModSeq zero value

1240 // makes sense.

1241 v = SyncState{1, 2, -1}

1242 return v.LastModSeq, tx.Insert(&v)

1243 } else if err != nil {

1244 return 0, err

1245 }

1246 v.LastModSeq++

1247 return v.LastModSeq, tx.Update(&v)

1248}

1249

1250func (a *Account) HighestDeletedModSeq(tx *bstore.Tx) (ModSeq, error) {

1251 v := SyncState{ID: 1}

1252 err := tx.Get(&v)

1253 if err == bstore.ErrAbsent {

1254 return 0, nil

1255 }

1256 return v.HighestDeletedModSeq, err

1257}

1258

1259// WithWLock runs fn with account writelock held. Necessary for account/mailbox modification. For message delivery, a read lock is required.

1260func (a *Account) WithWLock(fn func()) {

1261 a.Lock()

1262 defer a.Unlock()

1263 fn()

1264}

1265

1266// WithRLock runs fn with account read lock held. Needed for message delivery.

1267func (a *Account) WithRLock(fn func()) {

1268 a.RLock()

1269 defer a.RUnlock()

1270 fn()

1271}

1272

1273// DeliverMessage delivers a mail message to the account.

1274//

1275// The message, with msg.MsgPrefix and msgFile combined, must have a header

1276// section. The caller is responsible for adding a header separator to

1277// msg.MsgPrefix if missing from an incoming message.

1278//

1279// If the destination mailbox has the Sent special-use flag, the message is parsed

1280// for its recipients (to/cc/bcc). Their domains are added to Recipients for use in

1281// dmarc reputation.

1282//

1283// If sync is true, the message file and its directory are synced. Should be true

1284// for regular mail delivery, but can be false when importing many messages.

1285//

1286// If updateDiskUsage is true, the account total message size (for quota) is

1287// updated. Callers must check if a message can be added within quota before

1288// calling DeliverMessage.

1289//

1290// If CreateSeq/ModSeq is not set, it is assigned automatically.

1291//

1292// Must be called with account rlock or wlock.

1293//

1294// Caller must broadcast new message.

1295//

1296// Caller must update mailbox counts.

1297func (a *Account) DeliverMessage(log mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, sync, notrain, nothreads, updateDiskUsage bool) error {

1298 if m.Expunged {

1299 return fmt.Errorf("cannot deliver expunged message")

1300 }

1301

1302 mb := Mailbox{ID: m.MailboxID}

1303 if err := tx.Get(&mb); err != nil {

1304 return fmt.Errorf("get mailbox: %w", err)

1305 }

1306 m.UID = mb.UIDNext

1307 mb.UIDNext++

1308 if err := tx.Update(&mb); err != nil {

1309 return fmt.Errorf("updating mailbox nextuid: %w", err)

1310 }

1311

1312 if updateDiskUsage {

1313 du := DiskUsage{ID: 1}

1314 if err := tx.Get(&du); err != nil {

1315 return fmt.Errorf("get disk usage: %v", err)

1316 }

1317 du.MessageSize += m.Size

1318 if err := tx.Update(&du); err != nil {

1319 return fmt.Errorf("update disk usage: %v", err)

1320 }

1321 }

1322

1323 conf, _ := a.Conf()

1324 m.JunkFlagsForMailbox(mb, conf)

1325

1326 mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile.

1327 var part *message.Part

1328 if m.ParsedBuf == nil {

1329 p, err := message.EnsurePart(log.Logger, false, mr, m.Size)

1330 if err != nil {

1331 log.Infox("parsing delivered message", err, slog.String("parse", ""), slog.Int64("message", m.ID))

1332 // We continue, p is still valid.

1333 }

1334 part = &p

1335 buf, err := json.Marshal(part)

1336 if err != nil {

1337 return fmt.Errorf("marshal parsed message: %w", err)

1338 }

1339 m.ParsedBuf = buf

1340 } else {

1341 var p message.Part

1342 if err := json.Unmarshal(m.ParsedBuf, &p); err != nil {

1343 log.Errorx("unmarshal parsed message, continuing", err, slog.String("parse", ""))

1344 } else {

1345 part = &p

1346 }

1347 }

1348

1349 // If we are delivering to the originally intended mailbox, no need to store the mailbox ID again.

1350 if m.MailboxDestinedID != 0 && m.MailboxDestinedID == m.MailboxOrigID {

1351 m.MailboxDestinedID = 0

1352 }

1353 if m.CreateSeq == 0 || m.ModSeq == 0 {

1354 modseq, err := a.NextModSeq(tx)

1355 if err != nil {

1356 return fmt.Errorf("assigning next modseq: %w", err)

1357 }

1358 m.CreateSeq = modseq

1359 m.ModSeq = modseq

1360 }

1361

1362 if part != nil && m.MessageID == "" && m.SubjectBase == "" {

1363 m.PrepareThreading(log, part)

1364 }

1365

1366 // Assign to thread (if upgrade has completed).

1367 noThreadID := nothreads

1368 if m.ThreadID == 0 && !nothreads && part != nil {

1369 select {

1370 case <-a.threadsCompleted:

1371 if a.threadsErr != nil {

1372 log.Info("not assigning threads for new delivery, upgrading to threads failed")

1373 noThreadID = true

1374 } else {

1375 if err := assignThread(log, tx, m, part); err != nil {

1376 return fmt.Errorf("assigning thread: %w", err)

1377 }

1378 }

1379 default:

1380 // note: since we have a write transaction to get here, we can't wait for the

1381 // thread upgrade to finish.

1382 // If we don't assign a threadid the upgrade process will do it.

1383 log.Info("not assigning threads for new delivery, upgrading to threads in progress which will assign this message")

1384 noThreadID = true

1385 }

1386 }

1387

1388 if err := tx.Insert(m); err != nil {

1389 return fmt.Errorf("inserting message: %w", err)

1390 }

1391 if !noThreadID && m.ThreadID == 0 {

1392 m.ThreadID = m.ID

1393 if err := tx.Update(m); err != nil {

1394 return fmt.Errorf("updating message for its own thread id: %w", err)

1395 }

1396 }

1397

1398 // todo: perhaps we should match the recipients based on smtp submission and a matching message-id? we now miss the addresses in bcc's. for webmail, we could insert the recipients directly.

1399 if mb.Sent && part != nil && part.Envelope != nil {

1400 e := part.Envelope

1401 sent := e.Date

1402 if sent.IsZero() {

1403 sent = m.Received

1404 }

1405 if sent.IsZero() {

1406 sent = time.Now()

1407 }

1408 addrs := append(append(e.To, e.CC...), e.BCC...)

1409 for _, addr := range addrs {

1410 if addr.User == "" {

1411 // Would trigger error because Recipient.Localpart must be nonzero. todo: we could allow empty localpart in db, and filter by not using FilterNonzero.

1412 log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", slog.Any("address", addr))

1413 continue

1414 }

1415 d, err := dns.ParseDomain(addr.Host)

1416 if err != nil {

1417 log.Debugx("parsing domain in to/cc/bcc address", err, slog.Any("address", addr))

1418 continue

1419 }

1420 lp, err := smtp.ParseLocalpart(addr.User)

1421 if err != nil {

1422 log.Debugx("parsing localpart in to/cc/bcc address", err, slog.Any("address", addr))

1423 continue

1424 }

1425 mr := Recipient{

1426 MessageID: m.ID,

1427 Localpart: lp.String(),

1428 Domain: d.Name(),

1429 OrgDomain: publicsuffix.Lookup(context.TODO(), log.Logger, d).Name(),

1430 Sent: sent,

1431 }

1432 if err := tx.Insert(&mr); err != nil {

1433 return fmt.Errorf("inserting sent message recipients: %w", err)

1434 }

1435 }

1436 }

1437

1438 msgPath := a.MessagePath(m.ID)

1439 msgDir := filepath.Dir(msgPath)

1440 os.MkdirAll(msgDir, 0770)

1441

1442 // Sync file data to disk.

1443 if sync {

1444 if err := msgFile.Sync(); err != nil {

1445 return fmt.Errorf("fsync message file: %w", err)

1446 }

1447 }

1448

1449 if err := moxio.LinkOrCopy(log, msgPath, msgFile.Name(), &moxio.AtReader{R: msgFile}, true); err != nil {

1450 return fmt.Errorf("linking/copying message to new file: %w", err)

1451 }

1452

1453 if sync {

1454 if err := moxio.SyncDir(log, msgDir); err != nil {

1455 xerr := os.Remove(msgPath)

1456 log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath))

1457 return fmt.Errorf("sync directory: %w", err)

1458 }

1459 }

1460

1461 if !notrain && m.NeedsTraining() {

1462 l := []Message{*m}

1463 if err := a.RetrainMessages(context.TODO(), log, tx, l, false); err != nil {

1464 xerr := os.Remove(msgPath)

1465 log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath))

1466 return fmt.Errorf("training junkfilter: %w", err)

1467 }

1468 *m = l[0]

1469 }

1470

1471 return nil

1472}

1473

1474// SetPassword saves a new password for this account. This password is used for

1475// IMAP, SMTP (submission) sessions and the HTTP account web page.

1476func (a *Account) SetPassword(log mlog.Log, password string) error {

1477 password, err := precis.OpaqueString.String(password)

1478 if err != nil {

1479 return fmt.Errorf(`password not allowed by "precis"`)

1480 }

1481

1482 if len(password) < 8 {

1483 // We actually check for bytes...

1484 return fmt.Errorf("password must be at least 8 characters long")

1485 }

1486

1487 hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)

1488 if err != nil {

1489 return fmt.Errorf("generating password hash: %w", err)

1490 }

1491

1492 err = a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1493 if _, err := bstore.QueryTx[Password](tx).Delete(); err != nil {

1494 return fmt.Errorf("deleting existing password: %v", err)

1495 }

1496 var pw Password

1497 pw.Hash = string(hash)

1498

1499 // CRAM-MD5 calculates an HMAC-MD5, with the password as key, over a per-attempt

1500 // unique text that includes a timestamp. HMAC performs two hashes. Both times, the

1501 // first block is based on the key/password. We hash those first blocks now, and

1502 // store the hash state in the database. When we actually authenticate, we'll

1503 // complete the HMAC by hashing only the text. We cannot store crypto/hmac's hash,

1504 // because it does not expose its internal state and isn't a BinaryMarshaler.

1505 // ../rfc/2104:121

1506 pw.CRAMMD5.Ipad = md5.New()

1507 pw.CRAMMD5.Opad = md5.New()

1508 key := []byte(password)

1509 if len(key) > 64 {

1510 t := md5.Sum(key)

1511 key = t[:]

1512 }

1513 ipad := make([]byte, md5.BlockSize)

1514 opad := make([]byte, md5.BlockSize)

1515 copy(ipad, key)

1516 copy(opad, key)

1517 for i := range ipad {

1518 ipad[i] ^= 0x36

1519 opad[i] ^= 0x5c

1520 }

1521 pw.CRAMMD5.Ipad.Write(ipad)

1522 pw.CRAMMD5.Opad.Write(opad)

1523

1524 pw.SCRAMSHA1.Salt = scram.MakeRandom()

1525 pw.SCRAMSHA1.Iterations = 2 * 4096

1526 pw.SCRAMSHA1.SaltedPassword = scram.SaltPassword(sha1.New, password, pw.SCRAMSHA1.Salt, pw.SCRAMSHA1.Iterations)

1527

1528 pw.SCRAMSHA256.Salt = scram.MakeRandom()

1529 pw.SCRAMSHA256.Iterations = 4096

1530 pw.SCRAMSHA256.SaltedPassword = scram.SaltPassword(sha256.New, password, pw.SCRAMSHA256.Salt, pw.SCRAMSHA256.Iterations)

1531

1532 if err := tx.Insert(&pw); err != nil {

1533 return fmt.Errorf("inserting new password: %v", err)

1534 }

1535

1536 return sessionRemoveAll(context.TODO(), log, tx, a.Name)

1537 })

1538 if err == nil {

1539 log.Info("new password set for account", slog.String("account", a.Name))

1540 }

1541 return err

1542}

1543

1544// Subjectpass returns the signing key for use with subjectpass for the given

1545// email address with canonical localpart.

1546func (a *Account) Subjectpass(email string) (key string, err error) {

1547 return key, a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1548 v := Subjectpass{Email: email}

1549 err := tx.Get(&v)

1550 if err == nil {

1551 key = v.Key

1552 return nil

1553 }

1554 if !errors.Is(err, bstore.ErrAbsent) {

1555 return fmt.Errorf("get subjectpass key from accounts database: %w", err)

1556 }

1557 key = ""

1558 const chars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"

1559 buf := make([]byte, 16)

1560 if _, err := cryptorand.Read(buf); err != nil {

1561 return err

1562 }

1563 for _, b := range buf {

1564 key += string(chars[int(b)%len(chars)])

1565 }

1566 v.Key = key

1567 return tx.Insert(&v)

1568 })

1569}

1570

1571// Ensure mailbox is present in database, adding records for the mailbox and its

1572// parents if they aren't present.

1573//

1574// If subscribe is true, any mailboxes that were created will also be subscribed to.

1575// Caller must hold account wlock.

1576// Caller must propagate changes if any.

1577func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool) (mb Mailbox, changes []Change, rerr error) {

1578 if norm.NFC.String(name) != name {

1579 return Mailbox{}, nil, fmt.Errorf("mailbox name not normalized")

1580 }

1581

1582 // Quick sanity check.

1583 if strings.EqualFold(name, "inbox") && name != "Inbox" {

1584 return Mailbox{}, nil, fmt.Errorf("bad casing for inbox")

1585 }

1586

1587 elems := strings.Split(name, "/")

1588 q := bstore.QueryTx[Mailbox](tx)

1589 q.FilterFn(func(mb Mailbox) bool {

1590 return mb.Name == elems[0] || strings.HasPrefix(mb.Name, elems[0]+"/")

1591 })

1592 l, err := q.List()

1593 if err != nil {

1594 return Mailbox{}, nil, fmt.Errorf("list mailboxes: %v", err)

1595 }

1596

1597 mailboxes := map[string]Mailbox{}

1598 for _, xmb := range l {

1599 mailboxes[xmb.Name] = xmb

1600 }

1601

1602 p := ""

1603 for _, elem := range elems {

1604 if p != "" {

1605 p += "/"

1606 }

1607 p += elem

1608 var ok bool

1609 mb, ok = mailboxes[p]

1610 if ok {

1611 continue

1612 }

1613 uidval, err := a.NextUIDValidity(tx)

1614 if err != nil {

1615 return Mailbox{}, nil, fmt.Errorf("next uid validity: %v", err)

1616 }

1617 mb = Mailbox{

1618 Name: p,

1619 UIDValidity: uidval,

1620 UIDNext: 1,

1621 HaveCounts: true,

1622 }

1623 err = tx.Insert(&mb)

1624 if err != nil {

1625 return Mailbox{}, nil, fmt.Errorf("creating new mailbox: %v", err)

1626 }

1627

1628 var flags []string

1629 if subscribe {

1630 if tx.Get(&Subscription{p}) != nil {

1631 err := tx.Insert(&Subscription{p})

1632 if err != nil {

1633 return Mailbox{}, nil, fmt.Errorf("subscribing to mailbox: %v", err)

1634 }

1635 }

1636 flags = []string{`\Subscribed`}

1637 }

1638 changes = append(changes, ChangeAddMailbox{mb, flags})

1639 }

1640 return mb, changes, nil

1641}

1642

1643// MailboxExists checks if mailbox exists.

1644// Caller must hold account rlock.

1645func (a *Account) MailboxExists(tx *bstore.Tx, name string) (bool, error) {

1646 q := bstore.QueryTx[Mailbox](tx)

1647 q.FilterEqual("Name", name)

1648 return q.Exists()

1649}

1650

1651// MailboxFind finds a mailbox by name, returning a nil mailbox and nil error if mailbox does not exist.

1652func (a *Account) MailboxFind(tx *bstore.Tx, name string) (*Mailbox, error) {

1653 q := bstore.QueryTx[Mailbox](tx)

1654 q.FilterEqual("Name", name)

1655 mb, err := q.Get()

1656 if err == bstore.ErrAbsent {

1657 return nil, nil

1658 }

1659 if err != nil {

1660 return nil, fmt.Errorf("looking up mailbox: %w", err)

1661 }

1662 return &mb, nil

1663}

1664

1665// SubscriptionEnsure ensures a subscription for name exists. The mailbox does not

1666// have to exist. Any parents are not automatically subscribed.

1667// Changes are returned and must be broadcasted by the caller.

1668func (a *Account) SubscriptionEnsure(tx *bstore.Tx, name string) ([]Change, error) {

1669 if err := tx.Get(&Subscription{name}); err == nil {

1670 return nil, nil

1671 }

1672

1673 if err := tx.Insert(&Subscription{name}); err != nil {

1674 return nil, fmt.Errorf("inserting subscription: %w", err)

1675 }

1676

1677 q := bstore.QueryTx[Mailbox](tx)

1678 q.FilterEqual("Name", name)

1679 _, err := q.Get()

1680 if err == nil {

1681 return []Change{ChangeAddSubscription{name, nil}}, nil

1682 } else if err != bstore.ErrAbsent {

1683 return nil, fmt.Errorf("looking up mailbox for subscription: %w", err)

1684 }

1685 return []Change{ChangeAddSubscription{name, []string{`\NonExistent`}}}, nil

1686}

1687

1688// MessageRuleset returns the first ruleset (if any) that message the message

1689// represented by msgPrefix and msgFile, with smtp and validation fields from m.

1690func MessageRuleset(log mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset {

1691 if len(dest.Rulesets) == 0 {

1692 return nil

1693 }

1694

1695 mr := FileMsgReader(msgPrefix, msgFile) // We don't close, it would close the msgFile.

1696 p, err := message.Parse(log.Logger, false, mr)

1697 if err != nil {

1698 log.Errorx("parsing message for evaluating rulesets, continuing with headers", err, slog.String("parse", ""))

1699 // note: part is still set.

1700 }

1701 // todo optimize: only parse header if needed for rulesets. and probably reuse an earlier parsing.

1702 header, err := p.Header()

1703 if err != nil {

1704 log.Errorx("parsing message headers for evaluating rulesets, delivering to default mailbox", err, slog.String("parse", ""))

1705 // todo: reject message?

1706 return nil

1707 }

1708

1709ruleset:

1710 for _, rs := range dest.Rulesets {

1711 if rs.SMTPMailFromRegexpCompiled != nil {

1712 if !rs.SMTPMailFromRegexpCompiled.MatchString(m.MailFrom) {

1713 continue ruleset

1714 }

1715 }

1716

1717 if !rs.VerifiedDNSDomain.IsZero() {

1718 d := rs.VerifiedDNSDomain.Name()

1719 suffix := "." + d

1720 matchDomain := func(s string) bool {

1721 return s == d || strings.HasSuffix(s, suffix)

1722 }

1723 var ok bool

1724 if m.EHLOValidated && matchDomain(m.EHLODomain) {

1725 ok = true

1726 }

1727 if m.MailFromValidated && matchDomain(m.MailFromDomain) {

1728 ok = true

1729 }

1730 for _, d := range m.DKIMDomains {

1731 if matchDomain(d) {

1732 ok = true

1733 break

1734 }

1735 }

1736 if !ok {

1737 continue ruleset

1738 }

1739 }

1740

1741 header:

1742 for _, t := range rs.HeadersRegexpCompiled {

1743 for k, vl := range header {

1744 k = strings.ToLower(k)

1745 if !t[0].MatchString(k) {

1746 continue

1747 }

1748 for _, v := range vl {

1749 v = strings.ToLower(strings.TrimSpace(v))

1750 if t[1].MatchString(v) {

1751 continue header

1752 }

1753 }

1754 }

1755 continue ruleset

1756 }

1757 return &rs

1758 }

1759 return nil

1760}

1761

1762// MessagePath returns the file system path of a message.

1763func (a *Account) MessagePath(messageID int64) string {

1764 return strings.Join(append([]string{a.Dir, "msg"}, messagePathElems(messageID)...), string(filepath.Separator))

1765}

1766

1767// MessageReader opens a message for reading, transparently combining the

1768// message prefix with the original incoming message.

1769func (a *Account) MessageReader(m Message) *MsgReader {

1770 return &MsgReader{prefix: m.MsgPrefix, path: a.MessagePath(m.ID), size: m.Size}

1771}

1772

1773// DeliverDestination delivers an email to dest, based on the configured rulesets.

1774//

1775// Returns ErrOverQuota when account would be over quota after adding message.

1776//

1777// Caller must hold account wlock (mailbox may be created).

1778// Message delivery, possible mailbox creation, and updated mailbox counts are

1779// broadcasted.

1780func (a *Account) DeliverDestination(log mlog.Log, dest config.Destination, m *Message, msgFile *os.File) error {

1781 var mailbox string

1782 rs := MessageRuleset(log, dest, m, m.MsgPrefix, msgFile)

1783 if rs != nil {

1784 mailbox = rs.Mailbox

1785 } else if dest.Mailbox == "" {

1786 mailbox = "Inbox"

1787 } else {

1788 mailbox = dest.Mailbox

1789 }

1790 return a.DeliverMailbox(log, mailbox, m, msgFile)

1791}

1792

1793// DeliverMailbox delivers an email to the specified mailbox.

1794//

1795// Returns ErrOverQuota when account would be over quota after adding message.

1796//

1797// Caller must hold account wlock (mailbox may be created).

1798// Message delivery, possible mailbox creation, and updated mailbox counts are

1799// broadcasted.

1800func (a *Account) DeliverMailbox(log mlog.Log, mailbox string, m *Message, msgFile *os.File) error {

1801 var changes []Change

1802 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1803 if ok, _, err := a.CanAddMessageSize(tx, m.Size); err != nil {

1804 return err

1805 } else if !ok {

1806 return ErrOverQuota

1807 }

1808

1809 mb, chl, err := a.MailboxEnsure(tx, mailbox, true)

1810 if err != nil {

1811 return fmt.Errorf("ensuring mailbox: %w", err)

1812 }

1813 m.MailboxID = mb.ID

1814 m.MailboxOrigID = mb.ID

1815

1816 // Update count early, DeliverMessage will update mb too and we don't want to fetch

1817 // it again before updating.

1818 mb.MailboxCounts.Add(m.MailboxCounts())

1819 if err := tx.Update(&mb); err != nil {

1820 return fmt.Errorf("updating mailbox for delivery: %w", err)

1821 }

1822

1823 if err := a.DeliverMessage(log, tx, m, msgFile, true, false, false, true); err != nil {

1824 return err

1825 }

1826

1827 changes = append(changes, chl...)

1828 changes = append(changes, m.ChangeAddUID(), mb.ChangeCounts())

1829 return nil

1830 })

1831 // todo: if rename succeeded but transaction failed, we should remove the file.

1832 if err != nil {

1833 return err

1834 }

1835

1836 BroadcastChanges(a, changes)

1837 return nil

1838}

1839

1840// TidyRejectsMailbox removes old reject emails, and returns whether there is space for a new delivery.

1841//

1842// Caller most hold account wlock.

1843// Changes are broadcasted.

1844func (a *Account) TidyRejectsMailbox(log mlog.Log, rejectsMailbox string) (hasSpace bool, rerr error) {

1845 var changes []Change

1846

1847 var remove []Message

1848 defer func() {

1849 for _, m := range remove {

1850 p := a.MessagePath(m.ID)

1851 err := os.Remove(p)

1852 log.Check(err, "removing rejects message file", slog.String("path", p))

1853 }

1854 }()

1855

1856 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1857 mb, err := a.MailboxFind(tx, rejectsMailbox)

1858 if err != nil {

1859 return fmt.Errorf("finding mailbox: %w", err)

1860 }

1861 if mb == nil {

1862 // No messages have been delivered yet.

1863 hasSpace = true

1864 return nil

1865 }

1866

1867 // Gather old messages to remove.

1868 old := time.Now().Add(-14 * 24 * time.Hour)

1869 qdel := bstore.QueryTx[Message](tx)

1870 qdel.FilterNonzero(Message{MailboxID: mb.ID})

1871 qdel.FilterEqual("Expunged", false)

1872 qdel.FilterLess("Received", old)

1873 remove, err = qdel.List()

1874 if err != nil {

1875 return fmt.Errorf("listing old messages: %w", err)

1876 }

1877

1878 changes, err = a.rejectsRemoveMessages(context.TODO(), log, tx, mb, remove)

1879 if err != nil {

1880 return fmt.Errorf("removing messages: %w", err)

1881 }

1882

1883 // We allow up to n messages.

1884 qcount := bstore.QueryTx[Message](tx)

1885 qcount.FilterNonzero(Message{MailboxID: mb.ID})

1886 qcount.FilterEqual("Expunged", false)

1887 qcount.Limit(1000)

1888 n, err := qcount.Count()

1889 if err != nil {

1890 return fmt.Errorf("counting rejects: %w", err)

1891 }

1892 hasSpace = n < 1000

1893

1894 return nil

1895 })

1896 if err != nil {

1897 remove = nil // Don't remove files on failure.

1898 return false, err

1899 }

1900

1901 BroadcastChanges(a, changes)

1902

1903 return hasSpace, nil

1904}

1905

1906func (a *Account) rejectsRemoveMessages(ctx context.Context, log mlog.Log, tx *bstore.Tx, mb *Mailbox, l []Message) ([]Change, error) {

1907 if len(l) == 0 {

1908 return nil, nil

1909 }

1910 ids := make([]int64, len(l))

1911 anyids := make([]any, len(l))

1912 for i, m := range l {

1913 ids[i] = m.ID

1914 anyids[i] = m.ID

1915 }

1916

1917 // Remove any message recipients. Should not happen, but a user can move messages

1918 // from a Sent mailbox to the rejects mailbox...

1919 qdmr := bstore.QueryTx[Recipient](tx)

1920 qdmr.FilterEqual("MessageID", anyids...)

1921 if _, err := qdmr.Delete(); err != nil {

1922 return nil, fmt.Errorf("deleting from message recipient: %w", err)

1923 }

1924

1925 // Assign new modseq.

1926 modseq, err := a.NextModSeq(tx)

1927 if err != nil {

1928 return nil, fmt.Errorf("assign next modseq: %w", err)

1929 }

1930

1931 // Expunge the messages.

1932 qx := bstore.QueryTx[Message](tx)

1933 qx.FilterIDs(ids)

1934 var expunged []Message

1935 qx.Gather(&expunged)

1936 if _, err := qx.UpdateNonzero(Message{ModSeq: modseq, Expunged: true}); err != nil {

1937 return nil, fmt.Errorf("expunging messages: %w", err)

1938 }

1939

1940 var totalSize int64

1941 for _, m := range expunged {

1942 m.Expunged = false // Was set by update, but would cause wrong count.

1943 mb.MailboxCounts.Sub(m.MailboxCounts())

1944 totalSize += m.Size

1945 }

1946 if err := tx.Update(mb); err != nil {

1947 return nil, fmt.Errorf("updating mailbox counts: %w", err)

1948 }

1949 if err := a.AddMessageSize(log, tx, -totalSize); err != nil {

1950 return nil, fmt.Errorf("updating disk usage: %w", err)

1951 }

1952

1953 // Mark as neutral and train so junk filter gets untrained with these (junk) messages.

1954 for i := range expunged {

1955 expunged[i].Junk = false

1956 expunged[i].Notjunk = false

1957 }

1958 if err := a.RetrainMessages(ctx, log, tx, expunged, true); err != nil {

1959 return nil, fmt.Errorf("retraining expunged messages: %w", err)

1960 }

1961

1962 changes := make([]Change, len(l), len(l)+1)

1963 for i, m := range l {

1964 changes[i] = ChangeRemoveUIDs{mb.ID, []UID{m.UID}, modseq}

1965 }

1966 changes = append(changes, mb.ChangeCounts())

1967 return changes, nil

1968}

1969

1970// RejectsRemove removes a message from the rejects mailbox if present.

1971// Caller most hold account wlock.

1972// Changes are broadcasted.

1973func (a *Account) RejectsRemove(log mlog.Log, rejectsMailbox, messageID string) error {

1974 var changes []Change

1975

1976 var remove []Message

1977 defer func() {

1978 for _, m := range remove {

1979 p := a.MessagePath(m.ID)

1980 err := os.Remove(p)

1981 log.Check(err, "removing rejects message file", slog.String("path", p))

1982 }

1983 }()

1984

1985 err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {

1986 mb, err := a.MailboxFind(tx, rejectsMailbox)

1987 if err != nil {

1988 return fmt.Errorf("finding mailbox: %w", err)

1989 }

1990 if mb == nil {

1991 return nil

1992 }

1993

1994 q := bstore.QueryTx[Message](tx)

1995 q.FilterNonzero(Message{MailboxID: mb.ID, MessageID: messageID})

1996 q.FilterEqual("Expunged", false)

1997 remove, err = q.List()

1998 if err != nil {

1999 return fmt.Errorf("listing messages to remove: %w", err)

2000 }

2001

2002 changes, err = a.rejectsRemoveMessages(context.TODO(), log, tx, mb, remove)

2003 if err != nil {

2004 return fmt.Errorf("removing messages: %w", err)

2005 }

2006

2007 return nil

2008 })

2009 if err != nil {

2010 remove = nil // Don't remove files on failure.

2011 return err

2012 }

2013

2014 BroadcastChanges(a, changes)

2015

2016 return nil

2017}

2018

2019// AddMessageSize adjusts the DiskUsage.MessageSize by size.

2020func (a *Account) AddMessageSize(log mlog.Log, tx *bstore.Tx, size int64) error {

2021 du := DiskUsage{ID: 1}

2022 if err := tx.Get(&du); err != nil {

2023 return fmt.Errorf("get diskusage: %v", err)

2024 }

2025 du.MessageSize += size

2026 if du.MessageSize < 0 {

2027 log.Error("negative total message size", slog.Int64("delta", size), slog.Int64("newtotalsize", du.MessageSize))

2028 }

2029 if err := tx.Update(&du); err != nil {

2030 return fmt.Errorf("update total message size: %v", err)

2031 }

2032 return nil

2033}

2034

2035// QuotaMessageSize returns the effective maximum total message size for an

2036// account. Returns 0 if there is no maximum.

2037func (a *Account) QuotaMessageSize() int64 {

2038 conf, _ := a.Conf()

2039 size := conf.QuotaMessageSize

2040 if size == 0 {

2041 size = mox.Conf.Static.QuotaMessageSize

2042 }

2043 if size < 0 {

2044 size = 0

2045 }

2046 return size

2047}

2048

2049// CanAddMessageSize checks if a message of size bytes can be added, depending on

2050// total message size and configured quota for account.

2051func (a *Account) CanAddMessageSize(tx *bstore.Tx, size int64) (ok bool, maxSize int64, err error) {

2052 maxSize = a.QuotaMessageSize()

2053 if maxSize <= 0 {

2054 return true, 0, nil

2055 }

2056

2057 du := DiskUsage{ID: 1}

2058 if err := tx.Get(&du); err != nil {

2059 return false, maxSize, fmt.Errorf("get diskusage: %v", err)

2060 }

2061 return du.MessageSize+size <= maxSize, maxSize, nil

2062}

2063

2064// We keep a cache of recent successful authentications, so we don't have to bcrypt successful calls each time.

2065var authCache = struct {

2066 sync.Mutex

2067 success map[authKey]string

2068}{

2069 success: map[authKey]string{},

2070}

2071

2072type authKey struct {

2073 email, hash string

2074}

2075

2076// StartAuthCache starts a goroutine that regularly clears the auth cache.

2077func StartAuthCache() {

2078 go manageAuthCache()

2079}

2080

2081func manageAuthCache() {

2082 for {

2083 authCache.Lock()

2084 authCache.success = map[authKey]string{}

2085 authCache.Unlock()

2086 time.Sleep(15 * time.Minute)

2087 }

2088}

2089

2090// OpenEmailAuth opens an account given an email address and password.

2091//

2092// The email address may contain a catchall separator.

2093func OpenEmailAuth(log mlog.Log, email string, password string) (acc *Account, rerr error) {

2094 password, err := precis.OpaqueString.String(password)

2095 if err != nil {

2096 return nil, ErrUnknownCredentials

2097 }

2098

2099 acc, _, rerr = OpenEmail(log, email)

2100 if rerr != nil {

2101 return

2102 }

2103

2104 defer func() {

2105 if rerr != nil && acc != nil {

2106 err := acc.Close()

2107 log.Check(err, "closing account after open auth failure")

2108 acc = nil

2109 }

2110 }()

2111

2112 pw, err := bstore.QueryDB[Password](context.TODO(), acc.DB).Get()

2113 if err != nil {

2114 if err == bstore.ErrAbsent {

2115 return acc, ErrUnknownCredentials

2116 }

2117 return acc, fmt.Errorf("looking up password: %v", err)

2118 }

2119 authCache.Lock()

2120 ok := len(password) >= 8 && authCache.success[authKey{email, pw.Hash}] == password

2121 authCache.Unlock()

2122 if ok {

2123 return

2124 }

2125 if err := bcrypt.CompareHashAndPassword([]byte(pw.Hash), []byte(password)); err != nil {

2126 rerr = ErrUnknownCredentials

2127 } else {

2128 authCache.Lock()

2129 authCache.success[authKey{email, pw.Hash}] = password

2130 authCache.Unlock()

2131 }

2132 return

2133}

2134

2135// OpenEmail opens an account given an email address.

2136//

2137// The email address may contain a catchall separator.

2138func OpenEmail(log mlog.Log, email string) (*Account, config.Destination, error) {

2139 addr, err := smtp.ParseAddress(email)

2140 if err != nil {

2141 return nil, config.Destination{}, fmt.Errorf("%w: %v", ErrUnknownCredentials, err)

2142 }

2143 accountName, _, dest, err := mox.FindAccount(addr.Localpart, addr.Domain, false)

2144 if err != nil && (errors.Is(err, mox.ErrAccountNotFound) || errors.Is(err, mox.ErrDomainNotFound)) {

2145 return nil, config.Destination{}, ErrUnknownCredentials

2146 } else if err != nil {

2147 return nil, config.Destination{}, fmt.Errorf("looking up address: %v", err)

2148 }

2149 acc, err := OpenAccount(log, accountName)

2150 if err != nil {

2151 return nil, config.Destination{}, err

2152 }

2153 return acc, dest, nil

2154}

2155

2156// 64 characters, must be power of 2 for MessagePath

2157const msgDirChars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ-_"

2158

2159// MessagePath returns the filename of the on-disk filename, relative to the

2160// containing directory such as <account>/msg or queue.

2161// Returns names like "AB/1".

2162func MessagePath(messageID int64) string {

2163 return strings.Join(messagePathElems(messageID), string(filepath.Separator))

2164}

2165

2166// messagePathElems returns the elems, for a single join without intermediate

2167// string allocations.

2168func messagePathElems(messageID int64) []string {

2169 v := messageID >> 13 // 8k files per directory.

2170 dir := ""

2171 for {

2172 dir += string(msgDirChars[int(v)&(len(msgDirChars)-1)])

2173 v >>= 6

2174 if v == 0 {

2175 break

2176 }

2177 }

2178 return []string{dir, strconv.FormatInt(messageID, 10)}

2179}

2180

2181// Set returns a copy of f, with each flag that is true in mask set to the

2182// value from flags.

2183func (f Flags) Set(mask, flags Flags) Flags {

2184 set := func(d *bool, m, v bool) {

2185 if m {

2186 *d = v

2187 }

2188 }

2189 r := f

2190 set(&r.Seen, mask.Seen, flags.Seen)

2191 set(&r.Answered, mask.Answered, flags.Answered)

2192 set(&r.Flagged, mask.Flagged, flags.Flagged)

2193 set(&r.Forwarded, mask.Forwarded, flags.Forwarded)

2194 set(&r.Junk, mask.Junk, flags.Junk)

2195 set(&r.Notjunk, mask.Notjunk, flags.Notjunk)

2196 set(&r.Deleted, mask.Deleted, flags.Deleted)

2197 set(&r.Draft, mask.Draft, flags.Draft)

2198 set(&r.Phishing, mask.Phishing, flags.Phishing)

2199 set(&r.MDNSent, mask.MDNSent, flags.MDNSent)

2200 return r

2201}

2202

2203// Changed returns a mask of flags that have been between f and other.

2204func (f Flags) Changed(other Flags) (mask Flags) {

2205 mask.Seen = f.Seen != other.Seen

2206 mask.Answered = f.Answered != other.Answered

2207 mask.Flagged = f.Flagged != other.Flagged

2208 mask.Forwarded = f.Forwarded != other.Forwarded

2209 mask.Junk = f.Junk != other.Junk

2210 mask.Notjunk = f.Notjunk != other.Notjunk

2211 mask.Deleted = f.Deleted != other.Deleted

2212 mask.Draft = f.Draft != other.Draft

2213 mask.Phishing = f.Phishing != other.Phishing

2214 mask.MDNSent = f.MDNSent != other.MDNSent

2215 return

2216}

2217

2218var systemWellKnownFlags = map[string]bool{

2219 `\answered`: true,

2220 `\flagged`: true,

2221 `\deleted`: true,

2222 `\seen`: true,

2223 `\draft`: true,

2224 `$junk`: true,

2225 `$notjunk`: true,

2226 `$forwarded`: true,

2227 `$phishing`: true,

2228 `$mdnsent`: true,

2229}

2230

2231// ParseFlagsKeywords parses a list of textual flags into system/known flags, and

2232// other keywords. Keywords are lower-cased and sorted and check for valid syntax.

2233func ParseFlagsKeywords(l []string) (flags Flags, keywords []string, rerr error) {

2234 fields := map[string]*bool{

2235 `\answered`: &flags.Answered,

2236 `\flagged`: &flags.Flagged,

2237 `\deleted`: &flags.Deleted,

2238 `\seen`: &flags.Seen,

2239 `\draft`: &flags.Draft,

2240 `$junk`: &flags.Junk,

2241 `$notjunk`: &flags.Notjunk,

2242 `$forwarded`: &flags.Forwarded,

2243 `$phishing`: &flags.Phishing,

2244 `$mdnsent`: &flags.MDNSent,

2245 }

2246 seen := map[string]bool{}

2247 for _, f := range l {

2248 f = strings.ToLower(f)

2249 if field, ok := fields[f]; ok {

2250 *field = true

2251 } else if seen[f] {

2252 if mox.Pedantic {

2253 return Flags{}, nil, fmt.Errorf("duplicate keyword %s", f)

2254 }

2255 } else {

2256 if err := CheckKeyword(f); err != nil {

2257 return Flags{}, nil, fmt.Errorf("invalid keyword %s", f)

2258 }

2259 keywords = append(keywords, f)

2260 seen[f] = true

2261 }

2262 }

2263 sort.Strings(keywords)

2264 return flags, keywords, nil

2265}

2266

2267// RemoveKeywords removes keywords from l, returning whether any modifications were

2268// made, and a slice, a new slice in case of modifications. Keywords must have been

2269// validated earlier, e.g. through ParseFlagKeywords or CheckKeyword. Should only

2270// be used with valid keywords, not with system flags like \Seen.

2271func RemoveKeywords(l, remove []string) ([]string, bool) {

2272 var copied bool

2273 var changed bool

2274 for _, k := range remove {

2275 if i := slices.Index(l, k); i >= 0 {

2276 if !copied {

2277 l = append([]string{}, l...)

2278 copied = true

2279 }

2280 copy(l[i:], l[i+1:])

2281 l = l[:len(l)-1]

2282 changed = true

2283 }

2284 }

2285 return l, changed

2286}

2287

2288// MergeKeywords adds keywords from add into l, returning whether it added any

2289// keyword, and the slice with keywords, a new slice if modifications were made.

2290// Keywords are only added if they aren't already present. Should only be used with

2291// keywords, not with system flags like \Seen.

2292func MergeKeywords(l, add []string) ([]string, bool) {

2293 var copied bool

2294 var changed bool

2295 for _, k := range add {

2296 if !slices.Contains(l, k) {

2297 if !copied {

2298 l = append([]string{}, l...)

2299 copied = true

2300 }

2301 l = append(l, k)

2302 changed = true

2303 }

2304 }

2305 if changed {

2306 sort.Strings(l)

2307 }

2308 return l, changed

2309}

2310

2311// CheckKeyword returns an error if kw is not a valid keyword. Kw should

2312// already be in lower-case.

2313func CheckKeyword(kw string) error {

2314 if kw == "" {

2315 return fmt.Errorf("keyword cannot be empty")

2316 }

2317 if systemWellKnownFlags[kw] {

2318 return fmt.Errorf("cannot use well-known flag as keyword")

2319 }

2320 for _, c := range kw {

2321 // ../rfc/9051:6334

2322 if c <= ' ' || c > 0x7e || c >= 'A' && c <= 'Z' || strings.ContainsRune(`(){%*"\]`, c) {

2323 return errors.New(`not a valid keyword, must be lower-case ascii without spaces and without any of these characters: (){%*"\]`)

2324 }

2325 }

2326 return nil

2327}

2328

2329// SendLimitReached checks whether sending a message to recipients would reach

2330// the limit of outgoing messages for the account. If so, the message should

2331// not be sent. If the returned numbers are >= 0, the limit was reached and the

2332// values are the configured limits.

2333//

2334// To limit damage to the internet and our reputation in case of account

2335// compromise, we limit the max number of messages sent in a 24 hour window, both

2336// total number of messages and number of first-time recipients.

2337func (a *Account) SendLimitReached(tx *bstore.Tx, recipients []smtp.Path) (msglimit, rcptlimit int, rerr error) {

2338 conf, _ := a.Conf()

2339 msgmax := conf.MaxOutgoingMessagesPerDay

2340 if msgmax == 0 {

2341 // For human senders, 1000 recipients in a day is quite a lot.

2342 msgmax = 1000

2343 }

2344 rcptmax := conf.MaxFirstTimeRecipientsPerDay

2345 if rcptmax == 0 {

2346 // Human senders may address a new human-sized list of people once in a while. In

2347 // case of a compromise, a spammer will probably try to send to many new addresses.

2348 rcptmax = 200

2349 }

2350

2351 rcpts := map[string]time.Time{}

2352 n := 0

2353 err := bstore.QueryTx[Outgoing](tx).FilterGreater("Submitted", time.Now().Add(-24*time.Hour)).ForEach(func(o Outgoing) error {

2354 n++

2355 if rcpts[o.Recipient].IsZero() || o.Submitted.Before(rcpts[o.Recipient]) {

2356 rcpts[o.Recipient] = o.Submitted

2357 }

2358 return nil

2359 })

2360 if err != nil {

2361 return -1, -1, fmt.Errorf("querying message recipients in past 24h: %w", err)

2362 }

2363 if n+len(recipients) > msgmax {

2364 return msgmax, -1, nil

2365 }

2366

2367 // Only check if max first-time recipients is reached if there are enough messages

2368 // to trigger the limit.

2369 if n+len(recipients) < rcptmax {

2370 return -1, -1, nil

2371 }

2372

2373 isFirstTime := func(rcpt string, before time.Time) (bool, error) {

2374 exists, err := bstore.QueryTx[Outgoing](tx).FilterNonzero(Outgoing{Recipient: rcpt}).FilterLess("Submitted", before).Exists()

2375 return !exists, err

2376 }

2377

2378 firsttime := 0

2379 now := time.Now()

2380 for _, r := range recipients {

2381 if first, err := isFirstTime(r.XString(true), now); err != nil {

2382 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

2383 } else if first {

2384 firsttime++

2385 }

2386 }

2387 for r, t := range rcpts {

2388 if first, err := isFirstTime(r, t); err != nil {

2389 return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)

2390 } else if first {

2391 firsttime++

2392 }

2393 }

2394 if firsttime > rcptmax {

2395 return -1, rcptmax, nil

2396 }

2397 return -1, -1, nil

2398}

2399

2400// MailboxCreate creates a new mailbox, including any missing parent mailboxes,

2401// the total list of created mailboxes is returned in created. On success, if

2402// exists is false and rerr nil, the changes must be broadcasted by the caller.

2403//

2404// Name must be in normalized form.

2405func (a *Account) MailboxCreate(tx *bstore.Tx, name string) (changes []Change, created []string, exists bool, rerr error) {

2406 elems := strings.Split(name, "/")

2407 var p string

2408 for i, elem := range elems {

2409 if i > 0 {

2410 p += "/"

2411 }

2412 p += elem

2413 exists, err := a.MailboxExists(tx, p)

2414 if err != nil {

2415 return nil, nil, false, fmt.Errorf("checking if mailbox exists")

2416 }

2417 if exists {

2418 if i == len(elems)-1 {

2419 return nil, nil, true, fmt.Errorf("mailbox already exists")

2420 }

2421 continue

2422 }

2423 _, nchanges, err := a.MailboxEnsure(tx, p, true)

2424 if err != nil {

2425 return nil, nil, false, fmt.Errorf("ensuring mailbox exists")

2426 }

2427 changes = append(changes, nchanges...)

2428 created = append(created, p)

2429 }

2430 return changes, created, false, nil

2431}

2432

2433// MailboxRename renames mailbox mbsrc to dst, and any missing parents for the

2434// destination, and any children of mbsrc and the destination.

2435//

2436// Names must be normalized and cannot be Inbox.

2437func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc Mailbox, dst string) (changes []Change, isInbox, notExists, alreadyExists bool, rerr error) {

2438 if mbsrc.Name == "Inbox" || dst == "Inbox" {

2439 return nil, true, false, false, fmt.Errorf("inbox cannot be renamed")

2440 }

2441

2442 // We gather existing mailboxes that we need for deciding what to create/delete/update.

2443 q := bstore.QueryTx[Mailbox](tx)

2444 srcPrefix := mbsrc.Name + "/"

2445 dstRoot := strings.SplitN(dst, "/", 2)[0]

2446 dstRootPrefix := dstRoot + "/"

2447 q.FilterFn(func(mb Mailbox) bool {

2448 return mb.Name == mbsrc.Name || strings.HasPrefix(mb.Name, srcPrefix) || mb.Name == dstRoot || strings.HasPrefix(mb.Name, dstRootPrefix)

2449 })

2450 q.SortAsc("Name") // We'll rename the parents before children.

2451 l, err := q.List()

2452 if err != nil {

2453 return nil, false, false, false, fmt.Errorf("listing relevant mailboxes: %v", err)

2454 }

2455

2456 mailboxes := map[string]Mailbox{}

2457 for _, mb := range l {

2458 mailboxes[mb.Name] = mb

2459 }

2460

2461 if _, ok := mailboxes[mbsrc.Name]; !ok {

2462 return nil, false, true, false, fmt.Errorf("mailbox does not exist")

2463 }

2464

2465 uidval, err := a.NextUIDValidity(tx)

2466 if err != nil {

2467 return nil, false, false, false, fmt.Errorf("next uid validity: %v", err)

2468 }

2469

2470 // Ensure parent mailboxes for the destination paths exist.

2471 var parent string

2472 dstElems := strings.Split(dst, "/")

2473 for i, elem := range dstElems[:len(dstElems)-1] {

2474 if i > 0 {

2475 parent += "/"

2476 }

2477 parent += elem

2478

2479 mb, ok := mailboxes[parent]

2480 if ok {

2481 continue

2482 }

2483 omb := mb

2484 mb = Mailbox{

2485 ID: omb.ID,

2486 Name: parent,

2487 UIDValidity: uidval,

2488 UIDNext: 1,

2489 HaveCounts: true,

2490 }

2491 if err := tx.Insert(&mb); err != nil {

2492 return nil, false, false, false, fmt.Errorf("creating parent mailbox %q: %v", mb.Name, err)

2493 }

2494 if err := tx.Get(&Subscription{Name: parent}); err != nil {

2495 if err := tx.Insert(&Subscription{Name: parent}); err != nil {

2496 return nil, false, false, false, fmt.Errorf("creating subscription for %q: %v", parent, err)

2497 }

2498 }

2499 changes = append(changes, ChangeAddMailbox{Mailbox: mb, Flags: []string{`\Subscribed`}})

2500 }

2501

2502 // Process src mailboxes, renaming them to dst.

2503 for _, srcmb := range l {

2504 if srcmb.Name != mbsrc.Name && !strings.HasPrefix(srcmb.Name, srcPrefix) {

2505 continue

2506 }

2507 srcName := srcmb.Name

2508 dstName := dst + srcmb.Name[len(mbsrc.Name):]

2509 if _, ok := mailboxes[dstName]; ok {

2510 return nil, false, false, true, fmt.Errorf("destination mailbox %q already exists", dstName)

2511 }

2512

2513 srcmb.Name = dstName

2514 srcmb.UIDValidity = uidval

2515 if err := tx.Update(&srcmb); err != nil {

2516 return nil, false, false, false, fmt.Errorf("renaming mailbox: %v", err)

2517 }

2518

2519 var dstFlags []string

2520 if tx.Get(&Subscription{Name: dstName}) == nil {

2521 dstFlags = []string{`\Subscribed`}

2522 }

2523 changes = append(changes, ChangeRenameMailbox{MailboxID: srcmb.ID, OldName: srcName, NewName: dstName, Flags: dstFlags})

2524 }

2525

2526 // If we renamed e.g. a/b to a/b/c/d, and a/b/c to a/b/c/d/c, we'll have to recreate a/b and a/b/c.

2527 srcElems := strings.Split(mbsrc.Name, "/")

2528 xsrc := mbsrc.Name

2529 for i := 0; i < len(dstElems) && strings.HasPrefix(dst, xsrc+"/"); i++ {

2530 mb := Mailbox{

2531 UIDValidity: uidval,

2532 UIDNext: 1,

2533 Name: xsrc,

2534 HaveCounts: true,

2535 }

2536 if err := tx.Insert(&mb); err != nil {

2537 return nil, false, false, false, fmt.Errorf("creating mailbox at old path %q: %v", mb.Name, err)

2538 }

2539 xsrc += "/" + dstElems[len(srcElems)+i]

2540 }

2541 return changes, false, false, false, nil

2542}

2543

2544// MailboxDelete deletes a mailbox by ID. If it has children, the return value

2545// indicates that and an error is returned.

2546//

2547// Caller should broadcast the changes and remove files for the removed message IDs.

2548func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx, mailbox Mailbox) (changes []Change, removeMessageIDs []int64, hasChildren bool, rerr error) {

2549 // Look for existence of child mailboxes. There is a lot of text in the IMAP RFCs about

2550 // NoInferior and NoSelect. We just require only leaf mailboxes are deleted.

2551 qmb := bstore.QueryTx[Mailbox](tx)

2552 mbprefix := mailbox.Name + "/"

2553 qmb.FilterFn(func(mb Mailbox) bool {

2554 return strings.HasPrefix(mb.Name, mbprefix)

2555 })

2556 if childExists, err := qmb.Exists(); err != nil {

2557 return nil, nil, false, fmt.Errorf("checking if mailbox has child: %v", err)

2558 } else if childExists {

2559 return nil, nil, true, fmt.Errorf("mailbox has a child, only leaf mailboxes can be deleted")

2560 }

2561

2562 // todo jmap: instead of completely deleting a mailbox and its messages, we need to mark them all as expunged.

2563

2564 qm := bstore.QueryTx[Message](tx)

2565 qm.FilterNonzero(Message{MailboxID: mailbox.ID})

2566 remove, err := qm.List()

2567 if err != nil {

2568 return nil, nil, false, fmt.Errorf("listing messages to remove: %v", err)

2569 }

2570

2571 if len(remove) > 0 {

2572 removeIDs := make([]any, len(remove))

2573 for i, m := range remove {

2574 removeIDs[i] = m.ID

2575 }

2576 qmr := bstore.QueryTx[Recipient](tx)

2577 qmr.FilterEqual("MessageID", removeIDs...)

2578 if _, err = qmr.Delete(); err != nil {

2579 return nil, nil, false, fmt.Errorf("removing message recipients for messages: %v", err)

2580 }

2581

2582 qm = bstore.QueryTx[Message](tx)

2583 qm.FilterNonzero(Message{MailboxID: mailbox.ID})

2584 if _, err := qm.Delete(); err != nil {

2585 return nil, nil, false, fmt.Errorf("removing messages: %v", err)

2586 }

2587

2588 var totalSize int64

2589 for _, m := range remove {

2590 if !m.Expunged {

2591 removeMessageIDs = append(removeMessageIDs, m.ID)

2592 totalSize += m.Size

2593 }

2594 }

2595 if err := a.AddMessageSize(log, tx, -totalSize); err != nil {

2596 return nil, nil, false, fmt.Errorf("updating disk usage: %v", err)

2597 }

2598

2599 // Mark messages as not needing training. Then retrain them, so they are untrained if they were.

2600 n := 0

2601 o := 0

2602 for _, m := range remove {

2603 if !m.Expunged {

2604 remove[o] = m

2605 remove[o].Junk = false

2606 remove[o].Notjunk = false

2607 n++

2608 }

2609 }

2610 remove = remove[:n]

2611 if err := a.RetrainMessages(ctx, log, tx, remove, true); err != nil {

2612 return nil, nil, false, fmt.Errorf("untraining deleted messages: %v", err)

2613 }

2614 }

2615

2616 if err := tx.Delete(&Mailbox{ID: mailbox.ID}); err != nil {

2617 return nil, nil, false, fmt.Errorf("removing mailbox: %v", err)

2618 }

2619 return []Change{ChangeRemoveMailbox{MailboxID: mailbox.ID, Name: mailbox.Name}}, removeMessageIDs, false, nil

2620}

2621

2622// CheckMailboxName checks if name is valid, returning an INBOX-normalized name.

2623// I.e. it changes various casings of INBOX and INBOX/* to Inbox and Inbox/*.

2624// Name is invalid if it contains leading/trailing/double slashes, or when it isn't

2625// unicode-normalized, or when empty or has special characters.

2626//

2627// If name is the inbox, and allowInbox is false, this is indicated with the isInbox return parameter.

2628// For that case, and for other invalid names, an error is returned.

2629func CheckMailboxName(name string, allowInbox bool) (normalizedName string, isInbox bool, rerr error) {

2630 first := strings.SplitN(name, "/", 2)[0]

2631 if strings.EqualFold(first, "inbox") {

2632 if len(name) == len("inbox") && !allowInbox {

2633 return "", true, fmt.Errorf("special mailbox name Inbox not allowed")

2634 }

2635 name = "Inbox" + name[len("Inbox"):]

2636 }

2637

2638 if norm.NFC.String(name) != name {

2639 return "", false, errors.New("non-unicode-normalized mailbox names not allowed")

2640 }

2641

2642 if name == "" {

2643 return "", false, errors.New("empty mailbox name")

2644 }

2645 if strings.HasPrefix(name, "/") || strings.HasSuffix(name, "/") || strings.Contains(name, "//") {

2646 return "", false, errors.New("bad slashes in mailbox name")

2647 }

2648

2649 // "%" and "*" are difficult to use with the IMAP LIST command, but we allow mostly

2650 // allow them. ../rfc/3501:1002 ../rfc/9051:983

2651 if strings.HasPrefix(name, "#") {

2652 return "", false, errors.New("mailbox name cannot start with hash due to conflict with imap namespaces")

2653 }

2654

2655 // "#" and "&" are special in IMAP mailbox names. "#" for namespaces, "&" for

2656 // IMAP-UTF-7 encoding. We do allow them. ../rfc/3501:1018 ../rfc/9051:991

2657

2658 for _, c := range name {

2659 // ../rfc/3501:999 ../rfc/6855:192 ../rfc/9051:979

2660 if c <= 0x1f || c >= 0x7f && c <= 0x9f || c == 0x2028 || c == 0x2029 {

2661 return "", false, errors.New("control characters not allowed in mailbox name")

2662 }

2663 }

2664 return name, false, nil

2665}

2666