1package webmail

2

3import (

4 "context"

5 cryptorand "crypto/rand"

6 "encoding/base64"

7 "encoding/json"

8 "errors"

9 "fmt"

10 "io"

11 "mime"

12 "mime/multipart"

13 "net"

14 "net/http"

15 "net/mail"

16 "net/textproto"

17 "os"

18 "runtime/debug"

19 "sort"

20 "strings"

21 "sync"

22 "time"

23

24 _ "embed"

25

26 "golang.org/x/exp/maps"

27 "golang.org/x/exp/slog"

28

29 "github.com/mjl-/bstore"

30 "github.com/mjl-/sherpa"

31 "github.com/mjl-/sherpadoc"

32 "github.com/mjl-/sherpaprom"

33

34 "github.com/mjl-/mox/dkim"

35 "github.com/mjl-/mox/dns"

36 "github.com/mjl-/mox/message"

37 "github.com/mjl-/mox/metrics"

38 "github.com/mjl-/mox/mox-"

39 "github.com/mjl-/mox/moxio"

40 "github.com/mjl-/mox/moxvar"

41 "github.com/mjl-/mox/mtasts"

42 "github.com/mjl-/mox/mtastsdb"

43 "github.com/mjl-/mox/queue"

44 "github.com/mjl-/mox/smtp"

45 "github.com/mjl-/mox/smtpclient"

46 "github.com/mjl-/mox/store"

47 "github.com/mjl-/mox/webauth"

48)

49

50//go:embed api.json

51var webmailapiJSON []byte

52

53type Webmail struct {

54 maxMessageSize int64 // From listener.

55 cookiePath string // From listener.

56 isForwarded bool // From listener, whether we look at X-Forwarded-* headers.

57}

58

59func mustParseAPI(api string, buf []byte) (doc sherpadoc.Section) {

60 err := json.Unmarshal(buf, &doc)

61 if err != nil {

62 pkglog.Fatalx("parsing webmail api docs", err, slog.String("api", api))

63 }

64 return doc

65}

66

67var webmailDoc = mustParseAPI("webmail", webmailapiJSON)

68

69var sherpaHandlerOpts *sherpa.HandlerOpts

70

71func makeSherpaHandler(maxMessageSize int64, cookiePath string, isForwarded bool) (http.Handler, error) {

72 return sherpa.NewHandler("/api/", moxvar.Version, Webmail{maxMessageSize, cookiePath, isForwarded}, &webmailDoc, sherpaHandlerOpts)

73}

74

75func init() {

76 collector, err := sherpaprom.NewCollector("moxwebmail", nil)

77 if err != nil {

78 pkglog.Fatalx("creating sherpa prometheus collector", err)

79 }

80

81 sherpaHandlerOpts = &sherpa.HandlerOpts{Collector: collector, AdjustFunctionNames: "none", NoCORS: true}

82 // Just to validate.

83 _, err = makeSherpaHandler(0, "", false)

84 if err != nil {

85 pkglog.Fatalx("sherpa handler", err)

86 }

87}

88

89// LoginPrep returns a login token, and also sets it as cookie. Both must be

90// present in the call to Login.

91func (w Webmail) LoginPrep(ctx context.Context) string {

92 log := pkglog.WithContext(ctx)

93 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

94

95 var data [8]byte

96 _, err := cryptorand.Read(data[:])

97 xcheckf(ctx, err, "generate token")

98 loginToken := base64.RawURLEncoding.EncodeToString(data[:])

99

100 webauth.LoginPrep(ctx, log, "webmail", w.cookiePath, w.isForwarded, reqInfo.Response, reqInfo.Request, loginToken)

101

102 return loginToken

103}

104

105// Login returns a session token for the credentials, or fails with error code

106// "user:badLogin". Call LoginPrep to get a loginToken.

107func (w Webmail) Login(ctx context.Context, loginToken, username, password string) store.CSRFToken {

108 log := pkglog.WithContext(ctx)

109 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

110

111 csrfToken, err := webauth.Login(ctx, log, webauth.Accounts, "webmail", w.cookiePath, w.isForwarded, reqInfo.Response, reqInfo.Request, loginToken, username, password)

112 if _, ok := err.(*sherpa.Error); ok {

113 panic(err)

114 }

115 xcheckf(ctx, err, "login")

116 return csrfToken

117}

118

119// Logout invalidates the session token.

120func (w Webmail) Logout(ctx context.Context) {

121 log := pkglog.WithContext(ctx)

122 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

123

124 err := webauth.Logout(ctx, log, webauth.Accounts, "webmail", w.cookiePath, w.isForwarded, reqInfo.Response, reqInfo.Request, reqInfo.AccountName, reqInfo.SessionToken)

125 xcheckf(ctx, err, "logout")

126}

127

128// Token returns a token to use for an SSE connection. A token can only be used for

129// a single SSE connection. Tokens are stored in memory for a maximum of 1 minute,

130// with at most 10 unused tokens (the most recently created) per account.

131func (Webmail) Token(ctx context.Context) string {

132 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

133 return sseTokens.xgenerate(ctx, reqInfo.AccountName, reqInfo.LoginAddress, reqInfo.SessionToken)

134}

135

136// Requests sends a new request for an open SSE connection. Any currently active

137// request for the connection will be canceled, but this is done asynchrously, so

138// the SSE connection may still send results for the previous request. Callers

139// should take care to ignore such results. If req.Cancel is set, no new request is

140// started.

141func (Webmail) Request(ctx context.Context, req Request) {

142 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

143

144 if !req.Cancel && req.Page.Count <= 0 {

145 xcheckuserf(ctx, errors.New("Page.Count must be >= 1"), "checking request")

146 }

147

148 sse, ok := sseGet(req.SSEID, reqInfo.AccountName)

149 if !ok {

150 xcheckuserf(ctx, errors.New("unknown sseid"), "looking up connection")

151 }

152 sse.Request <- req

153}

154

155// ParsedMessage returns enough to render the textual body of a message. It is

156// assumed the client already has other fields through MessageItem.

157func (Webmail) ParsedMessage(ctx context.Context, msgID int64) (pm ParsedMessage) {

158 log := pkglog.WithContext(ctx)

159 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

160 acc, err := store.OpenAccount(log, reqInfo.AccountName)

161 xcheckf(ctx, err, "open account")

162 defer func() {

163 err := acc.Close()

164 log.Check(err, "closing account")

165 }()

166

167 var m store.Message

168 xdbread(ctx, acc, func(tx *bstore.Tx) {

169 m = xmessageID(ctx, tx, msgID)

170 })

171

172 state := msgState{acc: acc}

173 defer state.clear()

174 pm, err = parsedMessage(log, m, &state, true, false)

175 xcheckf(ctx, err, "parsing message")

176 return

177}

178

179// Attachment is a MIME part is an existing message that is not intended as

180// viewable text or HTML part.

181type Attachment struct {

182 Path []int // Indices into top-level message.Part.Parts.

183

184 // File name based on "name" attribute of "Content-Type", or the "filename"

185 // attribute of "Content-Disposition".

186 Filename string

187

188 Part message.Part

189}

190

191// SubmitMessage is an email message to be sent to one or more recipients.

192// Addresses are formatted as just email address, or with a name like "name

193// <user@host>".

194type SubmitMessage struct {

195 From string

196 To []string

197 Cc []string

198 Bcc []string

199 Subject string

200 TextBody string

201 Attachments []File

202 ForwardAttachments ForwardAttachments

203 IsForward bool

204 ResponseMessageID int64 // If set, this was a reply or forward, based on IsForward.

205 ReplyTo string // If non-empty, Reply-To header to add to message.

206 UserAgent string // User-Agent header added if not empty.

207 RequireTLS *bool // For "Require TLS" extension during delivery.

208}

209

210// ForwardAttachments references attachments by a list of message.Part paths.

211type ForwardAttachments struct {

212 MessageID int64 // Only relevant if MessageID is not 0.

213 Paths [][]int // List of attachments, each path is a list of indices into the top-level message.Part.Parts.

214}

215

216// File is a new attachment (not from an existing message that is being

217// forwarded) to send with a SubmitMessage.

218type File struct {

219 Filename string

220 DataURI string // Full data of the attachment, with base64 encoding and including content-type.

221}

222

223// parseAddress expects either a plain email address like "user@domain", or a

224// single address as used in a message header, like "name <user@domain>".

225func parseAddress(msghdr string) (message.NameAddress, error) {

226 a, err := mail.ParseAddress(msghdr)

227 if err != nil {

228 return message.NameAddress{}, nil

229 }

230

231 // todo: parse more fully according to ../rfc/5322:959

232 path, err := smtp.ParseAddress(a.Address)

233 if err != nil {

234 return message.NameAddress{}, err

235 }

236 return message.NameAddress{DisplayName: a.Name, Address: path}, nil

237}

238

239func xmailboxID(ctx context.Context, tx *bstore.Tx, mailboxID int64) store.Mailbox {

240 if mailboxID == 0 {

241 xcheckuserf(ctx, errors.New("invalid zero mailbox ID"), "getting mailbox")

242 }

243 mb := store.Mailbox{ID: mailboxID}

244 err := tx.Get(&mb)

245 if err == bstore.ErrAbsent {

246 xcheckuserf(ctx, err, "getting mailbox")

247 }

248 xcheckf(ctx, err, "getting mailbox")

249 return mb

250}

251

252// xmessageID returns a non-expunged message or panics with a sherpa error.

253func xmessageID(ctx context.Context, tx *bstore.Tx, messageID int64) store.Message {

254 if messageID == 0 {

255 xcheckuserf(ctx, errors.New("invalid zero message id"), "getting message")

256 }

257 m := store.Message{ID: messageID}

258 err := tx.Get(&m)

259 if err == bstore.ErrAbsent {

260 xcheckuserf(ctx, errors.New("message does not exist"), "getting message")

261 } else if err == nil && m.Expunged {

262 xcheckuserf(ctx, errors.New("message was removed"), "getting message")

263 }

264 xcheckf(ctx, err, "getting message")

265 return m

266}

267

268// MessageSubmit sends a message by submitting it the outgoing email queue. The

269// message is sent to all addresses listed in the To, Cc and Bcc addresses, without

270// Bcc message header.

271//

272// If a Sent mailbox is configured, messages are added to it after submitting

273// to the delivery queue.

274func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {

275 // Similar between ../smtpserver/server.go:/submit\( and ../webmail/webmail.go:/MessageSubmit\(

276

277 // todo: consider making this an HTTP POST, so we can upload as regular form, which is probably more efficient for encoding for the client and we can stream the data in.

278

279 // Prevent any accidental control characters, or attempts at getting bare \r or \n

280 // into messages.

281 for _, l := range [][]string{m.To, m.Cc, m.Bcc, {m.From, m.Subject, m.ReplyTo, m.UserAgent}} {

282 for _, s := range l {

283 for _, c := range s {

284 if c < 0x20 {

285 xcheckuserf(ctx, errors.New("control characters not allowed"), "checking header values")

286 }

287 }

288 }

289 }

290

291 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

292 log := pkglog.WithContext(ctx).With(slog.String("account", reqInfo.AccountName))

293 acc, err := store.OpenAccount(log, reqInfo.AccountName)

294 xcheckf(ctx, err, "open account")

295 defer func() {

296 err := acc.Close()

297 log.Check(err, "closing account")

298 }()

299

300 log.Debug("message submit")

301

302 fromAddr, err := parseAddress(m.From)

303 xcheckuserf(ctx, err, "parsing From address")

304

305 var replyTo *message.NameAddress

306 if m.ReplyTo != "" {

307 a, err := parseAddress(m.ReplyTo)

308 xcheckuserf(ctx, err, "parsing Reply-To address")

309 replyTo = &a

310 }

311

312 var recipients []smtp.Address

313

314 var toAddrs []message.NameAddress

315 for _, s := range m.To {

316 addr, err := parseAddress(s)

317 xcheckuserf(ctx, err, "parsing To address")

318 toAddrs = append(toAddrs, addr)

319 recipients = append(recipients, addr.Address)

320 }

321

322 var ccAddrs []message.NameAddress

323 for _, s := range m.Cc {

324 addr, err := parseAddress(s)

325 xcheckuserf(ctx, err, "parsing Cc address")

326 ccAddrs = append(ccAddrs, addr)

327 recipients = append(recipients, addr.Address)

328 }

329

330 for _, s := range m.Bcc {

331 addr, err := parseAddress(s)

332 xcheckuserf(ctx, err, "parsing Bcc address")

333 recipients = append(recipients, addr.Address)

334 }

335

336 // Check if from address is allowed for account.

337 fromAccName, _, _, err := mox.FindAccount(fromAddr.Address.Localpart, fromAddr.Address.Domain, false)

338 if err == nil && fromAccName != reqInfo.AccountName {

339 err = mox.ErrAccountNotFound

340 }

341 if err != nil && (errors.Is(err, mox.ErrAccountNotFound) || errors.Is(err, mox.ErrDomainNotFound)) {

342 metricSubmission.WithLabelValues("badfrom").Inc()

343 xcheckuserf(ctx, errors.New("address not found"), "looking from address for account")

344 }

345 xcheckf(ctx, err, "checking if from address is allowed")

346

347 if len(recipients) == 0 {

348 xcheckuserf(ctx, fmt.Errorf("no recipients"), "composing message")

349 }

350

351 // Check outgoing message rate limit.

352 xdbread(ctx, acc, func(tx *bstore.Tx) {

353 rcpts := make([]smtp.Path, len(recipients))

354 for i, r := range recipients {

355 rcpts[i] = smtp.Path{Localpart: r.Localpart, IPDomain: dns.IPDomain{Domain: r.Domain}}

356 }

357 msglimit, rcptlimit, err := acc.SendLimitReached(tx, rcpts)

358 if msglimit >= 0 {

359 metricSubmission.WithLabelValues("messagelimiterror").Inc()

360 xcheckuserf(ctx, errors.New("send message limit reached"), "checking outgoing rate limit")

361 } else if rcptlimit >= 0 {

362 metricSubmission.WithLabelValues("recipientlimiterror").Inc()

363 xcheckuserf(ctx, errors.New("send message limit reached"), "checking outgoing rate limit")

364 }

365 xcheckf(ctx, err, "checking send limit")

366 })

367

368 has8bit := false // We update this later on.

369

370 // We only use smtputf8 if we have to, with a utf-8 localpart. For IDNA, we use ASCII domains.

371 smtputf8 := false

372 for _, a := range recipients {

373 if a.Localpart.IsInternational() {

374 smtputf8 = true

375 break

376 }

377 }

378 if !smtputf8 && fromAddr.Address.Localpart.IsInternational() {

379 // todo: may want to warn user that they should consider sending with a ascii-only localpart, in case receiver doesn't support smtputf8.

380 smtputf8 = true

381 }

382

383 // Create file to compose message into.

384 dataFile, err := store.CreateMessageTemp(log, "webmail-submit")

385 xcheckf(ctx, err, "creating temporary file for message")

386 defer store.CloseRemoveTempFile(log, dataFile, "message to submit")

387

388 // If writing to the message file fails, we abort immediately.

389 xc := message.NewComposer(dataFile, w.maxMessageSize)

390 defer func() {

391 x := recover()

392 if x == nil {

393 return

394 }

395 if err, ok := x.(error); ok && errors.Is(err, message.ErrMessageSize) {

396 xcheckuserf(ctx, err, "making message")

397 } else if ok && errors.Is(err, message.ErrCompose) {

398 xcheckf(ctx, err, "making message")

399 }

400 panic(x)

401 }()

402

403 // todo spec: can we add an Authentication-Results header that indicates this is an authenticated message? the "auth" method is for SMTP AUTH, which this isn't. ../rfc/8601 https://www.iana.org/assignments/email-auth/email-auth.xhtml

404

405 // Each queued message gets a Received header.

406 // We don't have access to the local IP for adding.

407 // We cannot use VIA, because there is no registered method. We would like to use

408 // it to add the ascii domain name in case of smtputf8 and IDNA host name.

409 recvFrom := message.HeaderCommentDomain(mox.Conf.Static.HostnameDomain, smtputf8)

410 recvBy := mox.Conf.Static.HostnameDomain.XName(smtputf8)

411 recvID := mox.ReceivedID(mox.CidFromCtx(ctx))

412 recvHdrFor := func(rcptTo string) string {

413 recvHdr := &message.HeaderWriter{}

414 // For additional Received-header clauses, see:

415 // https://www.iana.org/assignments/mail-parameters/mail-parameters.xhtml#table-mail-parameters-8

416 // Note: we don't have "via" or "with", there is no registered for webmail.

417 recvHdr.Add(" ", "Received:", "from", recvFrom, "by", recvBy, "id", recvID) // ../rfc/5321:3158

418 if reqInfo.Request.TLS != nil {

419 recvHdr.Add(" ", mox.TLSReceivedComment(log, *reqInfo.Request.TLS)...)

420 }

421 recvHdr.Add(" ", "for", "<"+rcptTo+">;", time.Now().Format(message.RFC5322Z))

422 return recvHdr.String()

423 }

424

425 // Outer message headers.

426 xc.HeaderAddrs("From", []message.NameAddress{fromAddr})

427 if replyTo != nil {

428 xc.HeaderAddrs("Reply-To", []message.NameAddress{*replyTo})

429 }

430 xc.HeaderAddrs("To", toAddrs)

431 xc.HeaderAddrs("Cc", ccAddrs)

432 if m.Subject != "" {

433 xc.Subject(m.Subject)

434 }

435

436 messageID := fmt.Sprintf("<%s>", mox.MessageIDGen(smtputf8))

437 xc.Header("Message-Id", messageID)

438 xc.Header("Date", time.Now().Format(message.RFC5322Z))

439 // Add In-Reply-To and References headers.

440 if m.ResponseMessageID > 0 {

441 xdbread(ctx, acc, func(tx *bstore.Tx) {

442 rm := xmessageID(ctx, tx, m.ResponseMessageID)

443 msgr := acc.MessageReader(rm)

444 defer func() {

445 err := msgr.Close()

446 log.Check(err, "closing message reader")

447 }()

448 rp, err := rm.LoadPart(msgr)

449 xcheckf(ctx, err, "load parsed message")

450 h, err := rp.Header()

451 xcheckf(ctx, err, "parsing header")

452

453 if rp.Envelope == nil {

454 return

455 }

456 xc.Header("In-Reply-To", rp.Envelope.MessageID)

457 ref := h.Get("References")

458 if ref == "" {

459 ref = h.Get("In-Reply-To")

460 }

461 if ref != "" {

462 xc.Header("References", ref+"\r\n\t"+rp.Envelope.MessageID)

463 } else {

464 xc.Header("References", rp.Envelope.MessageID)

465 }

466 })

467 }

468 if m.UserAgent != "" {

469 xc.Header("User-Agent", m.UserAgent)

470 }

471 if m.RequireTLS != nil && !*m.RequireTLS {

472 xc.Header("TLS-Required", "No")

473 }

474 xc.Header("MIME-Version", "1.0")

475

476 if len(m.Attachments) > 0 || len(m.ForwardAttachments.Paths) > 0 {

477 mp := multipart.NewWriter(xc)

478 xc.Header("Content-Type", fmt.Sprintf(`multipart/mixed; boundary="%s"`, mp.Boundary()))

479 xc.Line()

480

481 textBody, ct, cte := xc.TextPart(m.TextBody)

482 textHdr := textproto.MIMEHeader{}

483 textHdr.Set("Content-Type", ct)

484 textHdr.Set("Content-Transfer-Encoding", cte)

485

486 textp, err := mp.CreatePart(textHdr)

487 xcheckf(ctx, err, "adding text part to message")

488 _, err = textp.Write(textBody)

489 xcheckf(ctx, err, "writing text part")

490

491 xaddPart := func(ct, filename string) io.Writer {

492 ahdr := textproto.MIMEHeader{}

493 cd := mime.FormatMediaType("attachment", map[string]string{"filename": filename})

494

495 ahdr.Set("Content-Type", ct)

496 ahdr.Set("Content-Transfer-Encoding", "base64")

497 ahdr.Set("Content-Disposition", cd)

498 ap, err := mp.CreatePart(ahdr)

499 xcheckf(ctx, err, "adding attachment part to message")

500 return ap

501 }

502

503 xaddAttachmentBase64 := func(ct, filename string, base64Data []byte) {

504 ap := xaddPart(ct, filename)

505

506 for len(base64Data) > 0 {

507 line := base64Data

508 n := len(line)

509 if n > 78 {

510 n = 78

511 }

512 line, base64Data = base64Data[:n], base64Data[n:]

513 _, err := ap.Write(line)

514 xcheckf(ctx, err, "writing attachment")

515 _, err = ap.Write([]byte("\r\n"))

516 xcheckf(ctx, err, "writing attachment")

517 }

518 }

519

520 xaddAttachment := func(ct, filename string, r io.Reader) {

521 ap := xaddPart(ct, filename)

522 wc := moxio.Base64Writer(ap)

523 _, err := io.Copy(wc, r)

524 xcheckf(ctx, err, "adding attachment")

525 err = wc.Close()

526 xcheckf(ctx, err, "flushing attachment")

527 }

528

529 for _, a := range m.Attachments {

530 s := a.DataURI

531 if !strings.HasPrefix(s, "data:") {

532 xcheckuserf(ctx, errors.New("missing data: in datauri"), "parsing attachment")

533 }

534 s = s[len("data:"):]

535 t := strings.SplitN(s, ",", 2)

536 if len(t) != 2 {

537 xcheckuserf(ctx, errors.New("missing comma in datauri"), "parsing attachment")

538 }

539 if !strings.HasSuffix(t[0], "base64") {

540 xcheckuserf(ctx, errors.New("missing base64 in datauri"), "parsing attachment")

541 }

542 ct := strings.TrimSuffix(t[0], "base64")

543 ct = strings.TrimSuffix(ct, ";")

544 if ct == "" {

545 ct = "application/octet-stream"

546 }

547 filename := a.Filename

548 if filename == "" {

549 filename = "unnamed.bin"

550 }

551 params := map[string]string{"name": filename}

552 ct = mime.FormatMediaType(ct, params)

553

554 // Ensure base64 is valid, then we'll write the original string.

555 _, err := io.Copy(io.Discard, base64.NewDecoder(base64.StdEncoding, strings.NewReader(t[1])))

556 xcheckuserf(ctx, err, "parsing attachment as base64")

557

558 xaddAttachmentBase64(ct, filename, []byte(t[1]))

559 }

560

561 if len(m.ForwardAttachments.Paths) > 0 {

562 acc.WithRLock(func() {

563 xdbread(ctx, acc, func(tx *bstore.Tx) {

564 fm := xmessageID(ctx, tx, m.ForwardAttachments.MessageID)

565 msgr := acc.MessageReader(fm)

566 defer func() {

567 err := msgr.Close()

568 log.Check(err, "closing message reader")

569 }()

570

571 fp, err := fm.LoadPart(msgr)

572 xcheckf(ctx, err, "load parsed message")

573

574 for _, path := range m.ForwardAttachments.Paths {

575 ap := fp

576 for _, xp := range path {

577 if xp < 0 || xp >= len(ap.Parts) {

578 xcheckuserf(ctx, errors.New("unknown part"), "looking up attachment")

579 }

580 ap = ap.Parts[xp]

581 }

582

583 filename := tryDecodeParam(log, ap.ContentTypeParams["name"])

584 if filename == "" {

585 filename = "unnamed.bin"

586 }

587 params := map[string]string{"name": filename}

588 if pcharset := ap.ContentTypeParams["charset"]; pcharset != "" {

589 params["charset"] = pcharset

590 }

591 ct := strings.ToLower(ap.MediaType + "/" + ap.MediaSubType)

592 ct = mime.FormatMediaType(ct, params)

593 xaddAttachment(ct, filename, ap.Reader())

594 }

595 })

596 })

597 }

598

599 err = mp.Close()

600 xcheckf(ctx, err, "writing mime multipart")

601 } else {

602 textBody, ct, cte := xc.TextPart(m.TextBody)

603 xc.Header("Content-Type", ct)

604 xc.Header("Content-Transfer-Encoding", cte)

605 xc.Line()

606 xc.Write([]byte(textBody))

607 }

608

609 xc.Flush()

610

611 // Add DKIM-Signature headers.

612 var msgPrefix string

613 fd := fromAddr.Address.Domain

614 confDom, _ := mox.Conf.Domain(fd)

615 selectors := mox.DKIMSelectors(confDom.DKIM)

616 if len(selectors) > 0 {

617 dkimHeaders, err := dkim.Sign(ctx, log.Logger, fromAddr.Address.Localpart, fd, selectors, smtputf8, dataFile)

618 if err != nil {

619 metricServerErrors.WithLabelValues("dkimsign").Inc()

620 }

621 xcheckf(ctx, err, "sign dkim")

622

623 msgPrefix = dkimHeaders

624 }

625

626 fromPath := smtp.Path{

627 Localpart: fromAddr.Address.Localpart,

628 IPDomain: dns.IPDomain{Domain: fromAddr.Address.Domain},

629 }

630 for _, rcpt := range recipients {

631 rcptMsgPrefix := recvHdrFor(rcpt.Pack(smtputf8)) + msgPrefix

632 msgSize := int64(len(rcptMsgPrefix)) + xc.Size

633 toPath := smtp.Path{

634 Localpart: rcpt.Localpart,

635 IPDomain: dns.IPDomain{Domain: rcpt.Domain},

636 }

637 qm := queue.MakeMsg(reqInfo.AccountName, fromPath, toPath, has8bit, smtputf8, msgSize, messageID, []byte(rcptMsgPrefix), m.RequireTLS)

638 err := queue.Add(ctx, log, &qm, dataFile)

639 if err != nil {

640 metricSubmission.WithLabelValues("queueerror").Inc()

641 }

642 xcheckf(ctx, err, "adding message to the delivery queue")

643 metricSubmission.WithLabelValues("ok").Inc()

644 }

645

646 var modseq store.ModSeq // Only set if needed.

647

648 // Append message to Sent mailbox and mark original messages as answered/forwarded.

649 acc.WithRLock(func() {

650 var changes []store.Change

651

652 metricked := false

653 defer func() {

654 if x := recover(); x != nil {

655 if !metricked {

656 metricServerErrors.WithLabelValues("submit").Inc()

657 }

658 panic(x)

659 }

660 }()

661 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

662 if m.ResponseMessageID > 0 {

663 rm := xmessageID(ctx, tx, m.ResponseMessageID)

664 oflags := rm.Flags

665 if m.IsForward {

666 rm.Forwarded = true

667 } else {

668 rm.Answered = true

669 }

670 if !rm.Junk && !rm.Notjunk {

671 rm.Notjunk = true

672 }

673 if rm.Flags != oflags {

674 modseq, err = acc.NextModSeq(tx)

675 xcheckf(ctx, err, "next modseq")

676 rm.ModSeq = modseq

677 err := tx.Update(&rm)

678 xcheckf(ctx, err, "updating flags of replied/forwarded message")

679 changes = append(changes, rm.ChangeFlags(oflags))

680

681 err = acc.RetrainMessages(ctx, log, tx, []store.Message{rm}, false)

682 xcheckf(ctx, err, "retraining messages after reply/forward")

683 }

684 }

685

686 sentmb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Sent", true).Get()

687 if err == bstore.ErrAbsent {

688 // There is no mailbox designated as Sent mailbox, so we're done.

689 return

690 }

691 xcheckf(ctx, err, "message submitted to queue, adding to Sent mailbox")

692

693 if modseq == 0 {

694 modseq, err = acc.NextModSeq(tx)

695 xcheckf(ctx, err, "next modseq")

696 }

697

698 sentm := store.Message{

699 CreateSeq: modseq,

700 ModSeq: modseq,

701 MailboxID: sentmb.ID,

702 MailboxOrigID: sentmb.ID,

703 Flags: store.Flags{Notjunk: true, Seen: true},

704 Size: int64(len(msgPrefix)) + xc.Size,

705 MsgPrefix: []byte(msgPrefix),

706 }

707

708 if ok, maxSize, err := acc.CanAddMessageSize(tx, sentm.Size); err != nil {

709 xcheckf(ctx, err, "checking quota")

710 } else if !ok {

711 xcheckuserf(ctx, fmt.Errorf("account over maximum total message size %d", maxSize), "checking quota")

712 }

713

714 // Update mailbox before delivery, which changes uidnext.

715 sentmb.Add(sentm.MailboxCounts())

716 err = tx.Update(&sentmb)

717 xcheckf(ctx, err, "updating sent mailbox for counts")

718

719 err = acc.DeliverMessage(log, tx, &sentm, dataFile, true, false, false, true)

720 if err != nil {

721 metricSubmission.WithLabelValues("storesenterror").Inc()

722 metricked = true

723 }

724 xcheckf(ctx, err, "message submitted to queue, appending message to Sent mailbox")

725

726 changes = append(changes, sentm.ChangeAddUID(), sentmb.ChangeCounts())

727 })

728

729 store.BroadcastChanges(acc, changes)

730 })

731}

732

733// MessageMove moves messages to another mailbox. If the message is already in

734// the mailbox an error is returned.

735func (Webmail) MessageMove(ctx context.Context, messageIDs []int64, mailboxID int64) {

736 log := pkglog.WithContext(ctx)

737 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

738 acc, err := store.OpenAccount(log, reqInfo.AccountName)

739 xcheckf(ctx, err, "open account")

740 defer func() {

741 err := acc.Close()

742 log.Check(err, "closing account")

743 }()

744

745 acc.WithRLock(func() {

746 retrain := make([]store.Message, 0, len(messageIDs))

747 removeChanges := map[int64]store.ChangeRemoveUIDs{}

748 // n adds, 1 remove, 2 mailboxcounts, optimistic and at least for a single message.

749 changes := make([]store.Change, 0, len(messageIDs)+3)

750

751 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

752 var mbSrc store.Mailbox

753 var modseq store.ModSeq

754

755 mbDst := xmailboxID(ctx, tx, mailboxID)

756

757 if len(messageIDs) == 0 {

758 return

759 }

760

761 keywords := map[string]struct{}{}

762

763 for _, mid := range messageIDs {

764 m := xmessageID(ctx, tx, mid)

765

766 // We may have loaded this mailbox in the previous iteration of this loop.

767 if m.MailboxID != mbSrc.ID {

768 if mbSrc.ID != 0 {

769 err = tx.Update(&mbSrc)

770 xcheckf(ctx, err, "updating source mailbox counts")

771 changes = append(changes, mbSrc.ChangeCounts())

772 }

773 mbSrc = xmailboxID(ctx, tx, m.MailboxID)

774 }

775

776 if mbSrc.ID == mailboxID {

777 // Client should filter out messages that are already in mailbox.

778 xcheckuserf(ctx, errors.New("already in destination mailbox"), "moving message")

779 }

780

781 if modseq == 0 {

782 modseq, err = acc.NextModSeq(tx)

783 xcheckf(ctx, err, "assigning next modseq")

784 }

785

786 ch := removeChanges[m.MailboxID]

787 ch.UIDs = append(ch.UIDs, m.UID)

788 ch.ModSeq = modseq

789 ch.MailboxID = m.MailboxID

790 removeChanges[m.MailboxID] = ch

791

792 // Copy of message record that we'll insert when UID is freed up.

793 om := m

794 om.PrepareExpunge()

795 om.ID = 0 // Assign new ID.

796 om.ModSeq = modseq

797

798 mbSrc.Sub(m.MailboxCounts())

799

800 if mbDst.Trash {

801 m.Seen = true

802 }

803 conf, _ := acc.Conf()

804 m.MailboxID = mbDst.ID

805 if m.IsReject && m.MailboxDestinedID != 0 {

806 // Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message

807 // is used for reputation calculation during future deliveries.

808 m.MailboxOrigID = m.MailboxDestinedID

809 m.IsReject = false

810 m.Seen = false

811 }

812 m.UID = mbDst.UIDNext

813 m.ModSeq = modseq

814 mbDst.UIDNext++

815 m.JunkFlagsForMailbox(mbDst, conf)

816 err = tx.Update(&m)

817 xcheckf(ctx, err, "updating moved message in database")

818

819 // Now that UID is unused, we can insert the old record again.

820 err = tx.Insert(&om)

821 xcheckf(ctx, err, "inserting record for expunge after moving message")

822

823 mbDst.Add(m.MailboxCounts())

824

825 changes = append(changes, m.ChangeAddUID())

826 retrain = append(retrain, m)

827

828 for _, kw := range m.Keywords {

829 keywords[kw] = struct{}{}

830 }

831 }

832

833 err = tx.Update(&mbSrc)

834 xcheckf(ctx, err, "updating source mailbox counts")

835

836 changes = append(changes, mbSrc.ChangeCounts(), mbDst.ChangeCounts())

837

838 // Ensure destination mailbox has keywords of the moved messages.

839 var mbKwChanged bool

840 mbDst.Keywords, mbKwChanged = store.MergeKeywords(mbDst.Keywords, maps.Keys(keywords))

841 if mbKwChanged {

842 changes = append(changes, mbDst.ChangeKeywords())

843 }

844

845 err = tx.Update(&mbDst)

846 xcheckf(ctx, err, "updating mailbox with uidnext")

847

848 err = acc.RetrainMessages(ctx, log, tx, retrain, false)

849 xcheckf(ctx, err, "retraining messages after move")

850 })

851

852 // Ensure UIDs of the removed message are in increasing order. It is quite common

853 // for all messages to be from a single source mailbox, meaning this is just one

854 // change, for which we preallocated space.

855 for _, ch := range removeChanges {

856 sort.Slice(ch.UIDs, func(i, j int) bool {

857 return ch.UIDs[i] < ch.UIDs[j]

858 })

859 changes = append(changes, ch)

860 }

861 store.BroadcastChanges(acc, changes)

862 })

863}

864

865// MessageDelete permanently deletes messages, without moving them to the Trash mailbox.

866func (Webmail) MessageDelete(ctx context.Context, messageIDs []int64) {

867 log := pkglog.WithContext(ctx)

868 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

869 acc, err := store.OpenAccount(log, reqInfo.AccountName)

870 xcheckf(ctx, err, "open account")

871 defer func() {

872 err := acc.Close()

873 log.Check(err, "closing account")

874 }()

875

876 if len(messageIDs) == 0 {

877 return

878 }

879

880 acc.WithWLock(func() {

881 removeChanges := map[int64]store.ChangeRemoveUIDs{}

882 changes := make([]store.Change, 0, len(messageIDs)+1) // n remove, 1 mailbox counts

883

884 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

885 var modseq store.ModSeq

886 var mb store.Mailbox

887 remove := make([]store.Message, 0, len(messageIDs))

888

889 var totalSize int64

890 for _, mid := range messageIDs {

891 m := xmessageID(ctx, tx, mid)

892 totalSize += m.Size

893

894 if m.MailboxID != mb.ID {

895 if mb.ID != 0 {

896 err := tx.Update(&mb)

897 xcheckf(ctx, err, "updating mailbox counts")

898 changes = append(changes, mb.ChangeCounts())

899 }

900 mb = xmailboxID(ctx, tx, m.MailboxID)

901 }

902

903 qmr := bstore.QueryTx[store.Recipient](tx)

904 qmr.FilterEqual("MessageID", m.ID)

905 _, err = qmr.Delete()

906 xcheckf(ctx, err, "removing message recipients")

907

908 mb.Sub(m.MailboxCounts())

909

910 if modseq == 0 {

911 modseq, err = acc.NextModSeq(tx)

912 xcheckf(ctx, err, "assigning next modseq")

913 }

914 m.Expunged = true

915 m.ModSeq = modseq

916 err = tx.Update(&m)

917 xcheckf(ctx, err, "marking message as expunged")

918

919 ch := removeChanges[m.MailboxID]

920 ch.UIDs = append(ch.UIDs, m.UID)

921 ch.MailboxID = m.MailboxID

922 ch.ModSeq = modseq

923 removeChanges[m.MailboxID] = ch

924 remove = append(remove, m)

925 }

926

927 if mb.ID != 0 {

928 err := tx.Update(&mb)

929 xcheckf(ctx, err, "updating count in mailbox")

930 changes = append(changes, mb.ChangeCounts())

931 }

932

933 err = acc.AddMessageSize(log, tx, -totalSize)

934 xcheckf(ctx, err, "updating disk usage")

935

936 // Mark removed messages as not needing training, then retrain them, so if they

937 // were trained, they get untrained.

938 for i := range remove {

939 remove[i].Junk = false

940 remove[i].Notjunk = false

941 }

942 err = acc.RetrainMessages(ctx, log, tx, remove, true)

943 xcheckf(ctx, err, "untraining deleted messages")

944 })

945

946 for _, ch := range removeChanges {

947 sort.Slice(ch.UIDs, func(i, j int) bool {

948 return ch.UIDs[i] < ch.UIDs[j]

949 })

950 changes = append(changes, ch)

951 }

952 store.BroadcastChanges(acc, changes)

953 })

954

955 for _, mID := range messageIDs {

956 p := acc.MessagePath(mID)

957 err := os.Remove(p)

958 log.Check(err, "removing message file for expunge")

959 }

960}

961

962// FlagsAdd adds flags, either system flags like \Seen or custom keywords. The

963// flags should be lower-case, but will be converted and verified.

964func (Webmail) FlagsAdd(ctx context.Context, messageIDs []int64, flaglist []string) {

965 log := pkglog.WithContext(ctx)

966 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

967 acc, err := store.OpenAccount(log, reqInfo.AccountName)

968 xcheckf(ctx, err, "open account")

969 defer func() {

970 err := acc.Close()

971 log.Check(err, "closing account")

972 }()

973

974 flags, keywords, err := store.ParseFlagsKeywords(flaglist)

975 xcheckuserf(ctx, err, "parsing flags")

976

977 acc.WithRLock(func() {

978 var changes []store.Change

979

980 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

981 var modseq store.ModSeq

982 var retrain []store.Message

983 var mb, origmb store.Mailbox

984

985 for _, mid := range messageIDs {

986 m := xmessageID(ctx, tx, mid)

987

988 if mb.ID != m.MailboxID {

989 if mb.ID != 0 {

990 err := tx.Update(&mb)

991 xcheckf(ctx, err, "updating mailbox")

992 if mb.MailboxCounts != origmb.MailboxCounts {

993 changes = append(changes, mb.ChangeCounts())

994 }

995 if mb.KeywordsChanged(origmb) {

996 changes = append(changes, mb.ChangeKeywords())

997 }

998 }

999 mb = xmailboxID(ctx, tx, m.MailboxID)

1000 origmb = mb

1001 }

1002 mb.Keywords, _ = store.MergeKeywords(mb.Keywords, keywords)

1003

1004 mb.Sub(m.MailboxCounts())

1005 oflags := m.Flags

1006 m.Flags = m.Flags.Set(flags, flags)

1007 var kwChanged bool

1008 m.Keywords, kwChanged = store.MergeKeywords(m.Keywords, keywords)

1009 mb.Add(m.MailboxCounts())

1010

1011 if m.Flags == oflags && !kwChanged {

1012 continue

1013 }

1014

1015 if modseq == 0 {

1016 modseq, err = acc.NextModSeq(tx)

1017 xcheckf(ctx, err, "assigning next modseq")

1018 }

1019 m.ModSeq = modseq

1020 err = tx.Update(&m)

1021 xcheckf(ctx, err, "updating message")

1022

1023 changes = append(changes, m.ChangeFlags(oflags))

1024 retrain = append(retrain, m)

1025 }

1026

1027 if mb.ID != 0 {

1028 err := tx.Update(&mb)

1029 xcheckf(ctx, err, "updating mailbox")

1030 if mb.MailboxCounts != origmb.MailboxCounts {

1031 changes = append(changes, mb.ChangeCounts())

1032 }

1033 if mb.KeywordsChanged(origmb) {

1034 changes = append(changes, mb.ChangeKeywords())

1035 }

1036 }

1037

1038 err = acc.RetrainMessages(ctx, log, tx, retrain, false)

1039 xcheckf(ctx, err, "retraining messages")

1040 })

1041

1042 store.BroadcastChanges(acc, changes)

1043 })

1044}

1045

1046// FlagsClear clears flags, either system flags like \Seen or custom keywords.

1047func (Webmail) FlagsClear(ctx context.Context, messageIDs []int64, flaglist []string) {

1048 log := pkglog.WithContext(ctx)

1049 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1050 acc, err := store.OpenAccount(log, reqInfo.AccountName)

1051 xcheckf(ctx, err, "open account")

1052 defer func() {

1053 err := acc.Close()

1054 log.Check(err, "closing account")

1055 }()

1056

1057 flags, keywords, err := store.ParseFlagsKeywords(flaglist)

1058 xcheckuserf(ctx, err, "parsing flags")

1059

1060 acc.WithRLock(func() {

1061 var retrain []store.Message

1062 var changes []store.Change

1063

1064 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1065 var modseq store.ModSeq

1066 var mb, origmb store.Mailbox

1067

1068 for _, mid := range messageIDs {

1069 m := xmessageID(ctx, tx, mid)

1070

1071 if mb.ID != m.MailboxID {

1072 if mb.ID != 0 {

1073 err := tx.Update(&mb)

1074 xcheckf(ctx, err, "updating counts for mailbox")

1075 if mb.MailboxCounts != origmb.MailboxCounts {

1076 changes = append(changes, mb.ChangeCounts())

1077 }

1078 // note: cannot remove keywords from mailbox by removing keywords from message.

1079 }

1080 mb = xmailboxID(ctx, tx, m.MailboxID)

1081 origmb = mb

1082 }

1083

1084 oflags := m.Flags

1085 mb.Sub(m.MailboxCounts())

1086 m.Flags = m.Flags.Set(flags, store.Flags{})

1087 var changed bool

1088 m.Keywords, changed = store.RemoveKeywords(m.Keywords, keywords)

1089 mb.Add(m.MailboxCounts())

1090

1091 if m.Flags == oflags && !changed {

1092 continue

1093 }

1094

1095 if modseq == 0 {

1096 modseq, err = acc.NextModSeq(tx)

1097 xcheckf(ctx, err, "assigning next modseq")

1098 }

1099 m.ModSeq = modseq

1100 err = tx.Update(&m)

1101 xcheckf(ctx, err, "updating message")

1102

1103 changes = append(changes, m.ChangeFlags(oflags))

1104 retrain = append(retrain, m)

1105 }

1106

1107 if mb.ID != 0 {

1108 err := tx.Update(&mb)

1109 xcheckf(ctx, err, "updating keywords in mailbox")

1110 if mb.MailboxCounts != origmb.MailboxCounts {

1111 changes = append(changes, mb.ChangeCounts())

1112 }

1113 // note: cannot remove keywords from mailbox by removing keywords from message.

1114 }

1115

1116 err = acc.RetrainMessages(ctx, log, tx, retrain, false)

1117 xcheckf(ctx, err, "retraining messages")

1118 })

1119

1120 store.BroadcastChanges(acc, changes)

1121 })

1122}

1123

1124// MailboxCreate creates a new mailbox.

1125func (Webmail) MailboxCreate(ctx context.Context, name string) {

1126 log := pkglog.WithContext(ctx)

1127 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1128 acc, err := store.OpenAccount(log, reqInfo.AccountName)

1129 xcheckf(ctx, err, "open account")

1130 defer func() {

1131 err := acc.Close()

1132 log.Check(err, "closing account")

1133 }()

1134

1135 name, _, err = store.CheckMailboxName(name, false)

1136 xcheckuserf(ctx, err, "checking mailbox name")

1137

1138 acc.WithWLock(func() {

1139 var changes []store.Change

1140 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1141 var exists bool

1142 var err error

1143 changes, _, exists, err = acc.MailboxCreate(tx, name)

1144 if exists {

1145 xcheckuserf(ctx, errors.New("mailbox already exists"), "creating mailbox")

1146 }

1147 xcheckf(ctx, err, "creating mailbox")

1148 })

1149

1150 store.BroadcastChanges(acc, changes)

1151 })

1152}

1153

1154// MailboxDelete deletes a mailbox and all its messages.

1155func (Webmail) MailboxDelete(ctx context.Context, mailboxID int64) {

1156 log := pkglog.WithContext(ctx)

1157 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1158 acc, err := store.OpenAccount(log, reqInfo.AccountName)

1159 xcheckf(ctx, err, "open account")

1160 defer func() {

1161 err := acc.Close()

1162 log.Check(err, "closing account")

1163 }()

1164

1165 // Messages to remove after having broadcasted the removal of messages.

1166 var removeMessageIDs []int64

1167

1168 acc.WithWLock(func() {

1169 var changes []store.Change

1170

1171 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1172 mb := xmailboxID(ctx, tx, mailboxID)

1173 if mb.Name == "Inbox" {

1174 // Inbox is special in IMAP and cannot be removed.

1175 xcheckuserf(ctx, errors.New("cannot remove special Inbox"), "checking mailbox")

1176 }

1177

1178 var hasChildren bool

1179 var err error

1180 changes, removeMessageIDs, hasChildren, err = acc.MailboxDelete(ctx, log, tx, mb)

1181 if hasChildren {

1182 xcheckuserf(ctx, errors.New("mailbox has children"), "deleting mailbox")

1183 }

1184 xcheckf(ctx, err, "deleting mailbox")

1185 })

1186

1187 store.BroadcastChanges(acc, changes)

1188 })

1189

1190 for _, mID := range removeMessageIDs {

1191 p := acc.MessagePath(mID)

1192 err := os.Remove(p)

1193 log.Check(err, "removing message file for mailbox delete", slog.String("path", p))

1194 }

1195}

1196

1197// MailboxEmpty empties a mailbox, removing all messages from the mailbox, but not

1198// its child mailboxes.

1199func (Webmail) MailboxEmpty(ctx context.Context, mailboxID int64) {

1200 log := pkglog.WithContext(ctx)

1201 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1202 acc, err := store.OpenAccount(log, reqInfo.AccountName)

1203 xcheckf(ctx, err, "open account")

1204 defer func() {

1205 err := acc.Close()

1206 log.Check(err, "closing account")

1207 }()

1208

1209 var expunged []store.Message

1210

1211 acc.WithWLock(func() {

1212 var changes []store.Change

1213

1214 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1215 mb := xmailboxID(ctx, tx, mailboxID)

1216

1217 modseq, err := acc.NextModSeq(tx)

1218 xcheckf(ctx, err, "next modseq")

1219

1220 // Mark messages as expunged.

1221 qm := bstore.QueryTx[store.Message](tx)

1222 qm.FilterNonzero(store.Message{MailboxID: mb.ID})

1223 qm.FilterEqual("Expunged", false)

1224 qm.SortAsc("UID")

1225 qm.Gather(&expunged)

1226 _, err = qm.UpdateNonzero(store.Message{ModSeq: modseq, Expunged: true})

1227 xcheckf(ctx, err, "deleting messages")

1228

1229 // Remove Recipients.

1230 anyIDs := make([]any, len(expunged))

1231 for i, m := range expunged {

1232 anyIDs[i] = m.ID

1233 }

1234 qmr := bstore.QueryTx[store.Recipient](tx)

1235 qmr.FilterEqual("MessageID", anyIDs...)

1236 _, err = qmr.Delete()

1237 xcheckf(ctx, err, "removing message recipients")

1238

1239 // Adjust mailbox counts, gather UIDs for broadcasted change, prepare for untraining.

1240 var totalSize int64

1241 uids := make([]store.UID, len(expunged))

1242 for i, m := range expunged {

1243 m.Expunged = false // Gather returns updated values.

1244 mb.Sub(m.MailboxCounts())

1245 totalSize += m.Size

1246 uids[i] = m.UID

1247

1248 expunged[i].Junk = false

1249 expunged[i].Notjunk = false

1250 }

1251

1252 err = tx.Update(&mb)

1253 xcheckf(ctx, err, "updating mailbox for counts")

1254

1255 err = acc.AddMessageSize(log, tx, -totalSize)

1256 xcheckf(ctx, err, "updating disk usage")

1257

1258 err = acc.RetrainMessages(ctx, log, tx, expunged, true)

1259 xcheckf(ctx, err, "retraining expunged messages")

1260

1261 chremove := store.ChangeRemoveUIDs{MailboxID: mb.ID, UIDs: uids, ModSeq: modseq}

1262 changes = []store.Change{chremove, mb.ChangeCounts()}

1263 })

1264

1265 store.BroadcastChanges(acc, changes)

1266 })

1267

1268 for _, m := range expunged {

1269 p := acc.MessagePath(m.ID)

1270 err := os.Remove(p)

1271 log.Check(err, "removing message file after emptying mailbox", slog.String("path", p))

1272 }

1273}

1274

1275// MailboxRename renames a mailbox, possibly moving it to a new parent. The mailbox

1276// ID and its messages are unchanged.

1277func (Webmail) MailboxRename(ctx context.Context, mailboxID int64, newName string) {

1278 log := pkglog.WithContext(ctx)

1279 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1280 acc, err := store.OpenAccount(log, reqInfo.AccountName)

1281 xcheckf(ctx, err, "open account")

1282 defer func() {

1283 err := acc.Close()

1284 log.Check(err, "closing account")

1285 }()

1286

1287 // Renaming Inbox is special for IMAP. For IMAP we have to implement it per the

1288 // standard. We can just say no.

1289 newName, _, err = store.CheckMailboxName(newName, false)

1290 xcheckuserf(ctx, err, "checking new mailbox name")

1291

1292 acc.WithWLock(func() {

1293 var changes []store.Change

1294

1295 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1296 mbsrc := xmailboxID(ctx, tx, mailboxID)

1297 var err error

1298 var isInbox, notExists, alreadyExists bool

1299 changes, isInbox, notExists, alreadyExists, err = acc.MailboxRename(tx, mbsrc, newName)

1300 if isInbox || notExists || alreadyExists {

1301 xcheckuserf(ctx, err, "renaming mailbox")

1302 }

1303 xcheckf(ctx, err, "renaming mailbox")

1304 })

1305

1306 store.BroadcastChanges(acc, changes)

1307 })

1308}

1309

1310// CompleteRecipient returns autocomplete matches for a recipient, returning the

1311// matches, most recently used first, and whether this is the full list and further

1312// requests for longer prefixes aren't necessary.

1313func (Webmail) CompleteRecipient(ctx context.Context, search string) ([]string, bool) {

1314 log := pkglog.WithContext(ctx)

1315 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1316 acc, err := store.OpenAccount(log, reqInfo.AccountName)

1317 xcheckf(ctx, err, "open account")

1318 defer func() {

1319 err := acc.Close()

1320 log.Check(err, "closing account")

1321 }()

1322

1323 search = strings.ToLower(search)

1324

1325 var matches []string

1326 all := true

1327 acc.WithRLock(func() {

1328 xdbread(ctx, acc, func(tx *bstore.Tx) {

1329 type key struct {

1330 localpart smtp.Localpart

1331 domain string

1332 }

1333 seen := map[key]bool{}

1334

1335 q := bstore.QueryTx[store.Recipient](tx)

1336 q.SortDesc("Sent")

1337 err := q.ForEach(func(r store.Recipient) error {

1338 k := key{r.Localpart, r.Domain}

1339 if seen[k] {

1340 return nil

1341 }

1342 // todo: we should have the address including name available in the database for searching. Will result in better matching, and also for the name.

1343 address := fmt.Sprintf("<%s@%s>", r.Localpart.String(), r.Domain)

1344 if !strings.Contains(strings.ToLower(address), search) {

1345 return nil

1346 }

1347 if len(matches) >= 20 {

1348 all = false

1349 return bstore.StopForEach

1350 }

1351

1352 // Look in the message that was sent for a name along with the address.

1353 m := store.Message{ID: r.MessageID}

1354 err := tx.Get(&m)

1355 xcheckf(ctx, err, "get sent message")

1356 if !m.Expunged && m.ParsedBuf != nil {

1357 var part message.Part

1358 err := json.Unmarshal(m.ParsedBuf, &part)

1359 xcheckf(ctx, err, "parsing part")

1360

1361 dom, err := dns.ParseDomain(r.Domain)

1362 xcheckf(ctx, err, "parsing domain of recipient")

1363

1364 var found bool

1365 lp := r.Localpart.String()

1366 checkAddrs := func(l []message.Address) {

1367 if found {

1368 return

1369 }

1370 for _, a := range l {

1371 if a.Name != "" && a.User == lp && strings.EqualFold(a.Host, dom.ASCII) {

1372 found = true

1373 address = addressString(a, false)

1374 return

1375 }

1376 }

1377 }

1378 if part.Envelope != nil {

1379 env := part.Envelope

1380 checkAddrs(env.To)

1381 checkAddrs(env.CC)

1382 checkAddrs(env.BCC)

1383 }

1384 }

1385

1386 matches = append(matches, address)

1387 seen[k] = true

1388 return nil

1389 })

1390 xcheckf(ctx, err, "listing recipients")

1391 })

1392 })

1393 return matches, all

1394}

1395

1396// addressString returns an address into a string as it could be used in a message header.

1397func addressString(a message.Address, smtputf8 bool) string {

1398 host := a.Host

1399 dom, err := dns.ParseDomain(a.Host)

1400 if err == nil {

1401 if smtputf8 && dom.Unicode != "" {

1402 host = dom.Unicode

1403 } else {

1404 host = dom.ASCII

1405 }

1406 }

1407 s := "<" + a.User + "@" + host + ">"

1408 if a.Name != "" {

1409 // todo: properly encoded/escaped name

1410 s = a.Name + " " + s

1411 }

1412 return s

1413}

1414

1415// MailboxSetSpecialUse sets the special use flags of a mailbox.

1416func (Webmail) MailboxSetSpecialUse(ctx context.Context, mb store.Mailbox) {

1417 log := pkglog.WithContext(ctx)

1418 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1419 acc, err := store.OpenAccount(log, reqInfo.AccountName)

1420 xcheckf(ctx, err, "open account")

1421 defer func() {

1422 err := acc.Close()

1423 log.Check(err, "closing account")

1424 }()

1425

1426 acc.WithWLock(func() {

1427 var changes []store.Change

1428

1429 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1430 xmb := xmailboxID(ctx, tx, mb.ID)

1431

1432 // We only allow a single mailbox for each flag (JMAP requirement). So for any flag

1433 // we set, we clear it for the mailbox(es) that had it, if any.

1434 clearPrevious := func(clear bool, specialUse string) {

1435 if !clear {

1436 return

1437 }

1438 var ombl []store.Mailbox

1439 q := bstore.QueryTx[store.Mailbox](tx)

1440 q.FilterNotEqual("ID", mb.ID)

1441 q.FilterEqual(specialUse, true)

1442 q.Gather(&ombl)

1443 _, err := q.UpdateField(specialUse, false)

1444 xcheckf(ctx, err, "updating previous special-use mailboxes")

1445

1446 for _, omb := range ombl {

1447 changes = append(changes, omb.ChangeSpecialUse())

1448 }

1449 }

1450 clearPrevious(mb.Archive, "Archive")

1451 clearPrevious(mb.Draft, "Draft")

1452 clearPrevious(mb.Junk, "Junk")

1453 clearPrevious(mb.Sent, "Sent")

1454 clearPrevious(mb.Trash, "Trash")

1455

1456 xmb.SpecialUse = mb.SpecialUse

1457 err = tx.Update(&xmb)

1458 xcheckf(ctx, err, "updating special-use flags for mailbox")

1459 changes = append(changes, xmb.ChangeSpecialUse())

1460 })

1461

1462 store.BroadcastChanges(acc, changes)

1463 })

1464}

1465

1466// ThreadCollapse saves the ThreadCollapse field for the messages and its

1467// children. The messageIDs are typically thread roots. But not all roots

1468// (without parent) of a thread need to have the same collapsed state.

1469func (Webmail) ThreadCollapse(ctx context.Context, messageIDs []int64, collapse bool) {

1470 log := pkglog.WithContext(ctx)

1471 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1472 acc, err := store.OpenAccount(log, reqInfo.AccountName)

1473 xcheckf(ctx, err, "open account")

1474 defer func() {

1475 err := acc.Close()

1476 log.Check(err, "closing account")

1477 }()

1478

1479 if len(messageIDs) == 0 {

1480 xcheckuserf(ctx, errors.New("no messages"), "setting collapse")

1481 }

1482

1483 acc.WithWLock(func() {

1484 changes := make([]store.Change, 0, len(messageIDs))

1485 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1486 // Gather ThreadIDs to list all potential messages, for a way to get all potential

1487 // (child) messages. Further refined in FilterFn.

1488 threadIDs := map[int64]struct{}{}

1489 msgIDs := map[int64]struct{}{}

1490 for _, id := range messageIDs {

1491 m := store.Message{ID: id}

1492 err := tx.Get(&m)

1493 if err == bstore.ErrAbsent {

1494 xcheckuserf(ctx, err, "get message")

1495 }

1496 xcheckf(ctx, err, "get message")

1497 threadIDs[m.ThreadID] = struct{}{}

1498 msgIDs[id] = struct{}{}

1499 }

1500

1501 var updated []store.Message

1502 q := bstore.QueryTx[store.Message](tx)

1503 q.FilterEqual("ThreadID", slicesAny(maps.Keys(threadIDs))...)

1504 q.FilterNotEqual("ThreadCollapsed", collapse)

1505 q.FilterFn(func(tm store.Message) bool {

1506 for _, id := range tm.ThreadParentIDs {

1507 if _, ok := msgIDs[id]; ok {

1508 return true

1509 }

1510 }

1511 _, ok := msgIDs[tm.ID]

1512 return ok

1513 })

1514 q.Gather(&updated)

1515 q.SortAsc("ID") // Consistent order for testing.

1516 _, err = q.UpdateFields(map[string]any{"ThreadCollapsed": collapse})

1517 xcheckf(ctx, err, "updating collapse in database")

1518

1519 for _, m := range updated {

1520 changes = append(changes, m.ChangeThread())

1521 }

1522 })

1523 store.BroadcastChanges(acc, changes)

1524 })

1525}

1526

1527// ThreadMute saves the ThreadMute field for the messages and their children.

1528// If messages are muted, they are also marked collapsed.

1529func (Webmail) ThreadMute(ctx context.Context, messageIDs []int64, mute bool) {

1530 log := pkglog.WithContext(ctx)

1531 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1532 acc, err := store.OpenAccount(log, reqInfo.AccountName)

1533 xcheckf(ctx, err, "open account")

1534 defer func() {

1535 err := acc.Close()

1536 log.Check(err, "closing account")

1537 }()

1538

1539 if len(messageIDs) == 0 {

1540 xcheckuserf(ctx, errors.New("no messages"), "setting mute")

1541 }

1542

1543 acc.WithWLock(func() {

1544 changes := make([]store.Change, 0, len(messageIDs))

1545 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1546 threadIDs := map[int64]struct{}{}

1547 msgIDs := map[int64]struct{}{}

1548 for _, id := range messageIDs {

1549 m := store.Message{ID: id}

1550 err := tx.Get(&m)

1551 if err == bstore.ErrAbsent {

1552 xcheckuserf(ctx, err, "get message")

1553 }

1554 xcheckf(ctx, err, "get message")

1555 threadIDs[m.ThreadID] = struct{}{}

1556 msgIDs[id] = struct{}{}

1557 }

1558

1559 var updated []store.Message

1560

1561 q := bstore.QueryTx[store.Message](tx)

1562 q.FilterEqual("ThreadID", slicesAny(maps.Keys(threadIDs))...)

1563 q.FilterFn(func(tm store.Message) bool {

1564 if tm.ThreadMuted == mute && (!mute || tm.ThreadCollapsed) {

1565 return false

1566 }

1567 for _, id := range tm.ThreadParentIDs {

1568 if _, ok := msgIDs[id]; ok {

1569 return true

1570 }

1571 }

1572 _, ok := msgIDs[tm.ID]

1573 return ok

1574 })

1575 q.Gather(&updated)

1576 fields := map[string]any{"ThreadMuted": mute}

1577 if mute {

1578 fields["ThreadCollapsed"] = true

1579 }

1580 _, err = q.UpdateFields(fields)

1581 xcheckf(ctx, err, "updating mute in database")

1582

1583 for _, m := range updated {

1584 changes = append(changes, m.ChangeThread())

1585 }

1586 })

1587 store.BroadcastChanges(acc, changes)

1588 })

1589}

1590

1591// SecurityResult indicates whether a security feature is supported.

1592type SecurityResult string

1593

1594const (

1595 SecurityResultError SecurityResult = "error"

1596 SecurityResultNo SecurityResult = "no"

1597 SecurityResultYes SecurityResult = "yes"

1598 // Unknown whether supported. Finding out may only be (reasonably) possible when

1599 // trying (e.g. SMTP STARTTLS). Once tried, the result may be cached for future

1600 // lookups.

1601 SecurityResultUnknown SecurityResult = "unknown"

1602)

1603

1604// RecipientSecurity is a quick analysis of the security properties of delivery to

1605// the recipient (domain).

1606type RecipientSecurity struct {

1607 // Whether recipient domain supports (opportunistic) STARTTLS, as seen during most

1608 // recent delivery attempt. Will be "unknown" if no delivery to the domain has been

1609 // attempted yet.

1610 STARTTLS SecurityResult

1611

1612 // Whether we have a stored enforced MTA-STS policy, or domain has MTA-STS DNS

1613 // record.

1614 MTASTS SecurityResult

1615

1616 // Whether MX lookup response was DNSSEC-signed.

1617 DNSSEC SecurityResult

1618

1619 // Whether first delivery destination has DANE records.

1620 DANE SecurityResult

1621

1622 // Whether recipient domain is known to implement the REQUIRETLS SMTP extension.

1623 // Will be "unknown" if no delivery to the domain has been attempted yet.

1624 RequireTLS SecurityResult

1625}

1626

1627// RecipientSecurity looks up security properties of the address in the

1628// single-address message addressee (as it appears in a To/Cc/Bcc/etc header).

1629func (Webmail) RecipientSecurity(ctx context.Context, messageAddressee string) (RecipientSecurity, error) {

1630 resolver := dns.StrictResolver{Pkg: "webmail", Log: pkglog.WithContext(ctx).Logger}

1631 return recipientSecurity(ctx, resolver, messageAddressee)

1632}

1633

1634// logPanic can be called with a defer from a goroutine to prevent the entire program from being shutdown in case of a panic.

1635func logPanic(ctx context.Context) {

1636 x := recover()

1637 if x == nil {

1638 return

1639 }

1640 log := pkglog.WithContext(ctx)

1641 log.Error("recover from panic", slog.Any("panic", x))

1642 debug.PrintStack()

1643 metrics.PanicInc(metrics.Webmail)

1644}

1645

1646// separate function for testing with mocked resolver.

1647func recipientSecurity(ctx context.Context, resolver dns.Resolver, messageAddressee string) (RecipientSecurity, error) {

1648 log := pkglog.WithContext(ctx)

1649

1650 rs := RecipientSecurity{

1651 SecurityResultUnknown,

1652 SecurityResultUnknown,

1653 SecurityResultUnknown,

1654 SecurityResultUnknown,

1655 SecurityResultUnknown,

1656 }

1657

1658 msgAddr, err := mail.ParseAddress(messageAddressee)

1659 if err != nil {

1660 return rs, fmt.Errorf("parsing message addressee: %v", err)

1661 }

1662

1663 addr, err := smtp.ParseAddress(msgAddr.Address)

1664 if err != nil {

1665 return rs, fmt.Errorf("parsing address: %v", err)

1666 }

1667

1668 var wg sync.WaitGroup

1669

1670 // MTA-STS.

1671 wg.Add(1)

1672 go func() {

1673 defer logPanic(ctx)

1674 defer wg.Done()

1675

1676 policy, _, _, err := mtastsdb.Get(ctx, log.Logger, resolver, addr.Domain)

1677 if policy != nil && policy.Mode == mtasts.ModeEnforce {

1678 rs.MTASTS = SecurityResultYes

1679 } else if err == nil {

1680 rs.MTASTS = SecurityResultNo

1681 } else {

1682 rs.MTASTS = SecurityResultError

1683 }

1684 }()

1685

1686 // DNSSEC and DANE.

1687 wg.Add(1)

1688 go func() {

1689 defer logPanic(ctx)

1690 defer wg.Done()

1691

1692 _, origNextHopAuthentic, expandedNextHopAuthentic, _, hosts, _, err := smtpclient.GatherDestinations(ctx, log.Logger, resolver, dns.IPDomain{Domain: addr.Domain})

1693 if err != nil {

1694 rs.DNSSEC = SecurityResultError

1695 return

1696 }

1697 if origNextHopAuthentic && expandedNextHopAuthentic {

1698 rs.DNSSEC = SecurityResultYes

1699 } else {

1700 rs.DNSSEC = SecurityResultNo

1701 }

1702

1703 if !origNextHopAuthentic {

1704 rs.DANE = SecurityResultNo

1705 return

1706 }

1707

1708 // We're only looking at the first host to deliver to (typically first mx destination).

1709 if len(hosts) == 0 || hosts[0].Domain.IsZero() {

1710 return // Should not happen.

1711 }

1712 host := hosts[0]

1713

1714 // Resolve the IPs. Required for DANE to prevent bad DNS servers from causing an

1715 // error result instead of no-DANE result.

1716 authentic, expandedAuthentic, expandedHost, _, _, err := smtpclient.GatherIPs(ctx, log.Logger, resolver, host, map[string][]net.IP{})

1717 if err != nil {

1718 rs.DANE = SecurityResultError

1719 return

1720 }

1721 if !authentic {

1722 rs.DANE = SecurityResultNo

1723 return

1724 }

1725

1726 daneRequired, _, _, err := smtpclient.GatherTLSA(ctx, log.Logger, resolver, host.Domain, expandedAuthentic, expandedHost)

1727 if err != nil {

1728 rs.DANE = SecurityResultError

1729 return

1730 } else if daneRequired {

1731 rs.DANE = SecurityResultYes

1732 } else {

1733 rs.DANE = SecurityResultNo

1734 }

1735 }()

1736

1737 // STARTTLS and RequireTLS

1738 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1739 acc, err := store.OpenAccount(log, reqInfo.AccountName)

1740 xcheckf(ctx, err, "open account")

1741 defer func() {

1742 if acc != nil {

1743 err := acc.Close()

1744 log.Check(err, "closing account")

1745 }

1746 }()

1747

1748 err = acc.DB.Read(ctx, func(tx *bstore.Tx) error {

1749 q := bstore.QueryTx[store.RecipientDomainTLS](tx)

1750 q.FilterNonzero(store.RecipientDomainTLS{Domain: addr.Domain.Name()})

1751 rd, err := q.Get()

1752 if err == bstore.ErrAbsent {

1753 return nil

1754 } else if err != nil {

1755 rs.STARTTLS = SecurityResultError

1756 rs.RequireTLS = SecurityResultError

1757 log.Errorx("looking up recipient domain", err, slog.Any("domain", addr.Domain))

1758 return nil

1759 }

1760 if rd.STARTTLS {

1761 rs.STARTTLS = SecurityResultYes

1762 } else {

1763 rs.STARTTLS = SecurityResultNo

1764 }

1765 if rd.RequireTLS {

1766 rs.RequireTLS = SecurityResultYes

1767 } else {

1768 rs.RequireTLS = SecurityResultNo

1769 }

1770 return nil

1771 })

1772 xcheckf(ctx, err, "lookup recipient domain")

1773

1774 // Close account as soon as possible, not after waiting for MTA-STS/DNSSEC/DANE

1775 // checks to complete, which can take a while.

1776 err = acc.Close()

1777 log.Check(err, "closing account")

1778 acc = nil

1779

1780 wg.Wait()

1781

1782 return rs, nil

1783}

1784

1785func slicesAny[T any](l []T) []any {

1786 r := make([]any, len(l))

1787 for i, v := range l {

1788 r[i] = v

1789 }

1790 return r

1791}

1792

1793// SSETypes exists to ensure the generated API contains the types, for use in SSE events.

1794func (Webmail) SSETypes() (start EventStart, viewErr EventViewErr, viewReset EventViewReset, viewMsgs EventViewMsgs, viewChanges EventViewChanges, msgAdd ChangeMsgAdd, msgRemove ChangeMsgRemove, msgFlags ChangeMsgFlags, msgThread ChangeMsgThread, mailboxRemove ChangeMailboxRemove, mailboxAdd ChangeMailboxAdd, mailboxRename ChangeMailboxRename, mailboxCounts ChangeMailboxCounts, mailboxSpecialUse ChangeMailboxSpecialUse, mailboxKeywords ChangeMailboxKeywords, flags store.Flags) {

1795 return

1796}

1797