1package webmail

2

3import (

4 "context"

5 cryptorand "crypto/rand"

6 "encoding/base64"

7 "encoding/json"

8 "errors"

9 "fmt"

10 "io"

11 "log/slog"

12 "mime"

13 "mime/multipart"

14 "net"

15 "net/http"

16 "net/mail"

17 "net/textproto"

18 "os"

19 "runtime/debug"

20 "sort"

21 "strings"

22 "sync"

23 "time"

24

25 _ "embed"

26

27 "golang.org/x/exp/maps"

28

29 "github.com/mjl-/bstore"

30 "github.com/mjl-/sherpa"

31 "github.com/mjl-/sherpadoc"

32 "github.com/mjl-/sherpaprom"

33

34 "github.com/mjl-/mox/dkim"

35 "github.com/mjl-/mox/dns"

36 "github.com/mjl-/mox/message"

37 "github.com/mjl-/mox/metrics"

38 "github.com/mjl-/mox/mox-"

39 "github.com/mjl-/mox/moxio"

40 "github.com/mjl-/mox/moxvar"

41 "github.com/mjl-/mox/mtasts"

42 "github.com/mjl-/mox/mtastsdb"

43 "github.com/mjl-/mox/queue"

44 "github.com/mjl-/mox/smtp"

45 "github.com/mjl-/mox/smtpclient"

46 "github.com/mjl-/mox/store"

47 "github.com/mjl-/mox/webauth"

48)

49

50//go:embed api.json

51var webmailapiJSON []byte

52

53type Webmail struct {

54 maxMessageSize int64 // From listener.

55 cookiePath string // From listener.

56 isForwarded bool // From listener, whether we look at X-Forwarded-* headers.

57}

58

59func mustParseAPI(api string, buf []byte) (doc sherpadoc.Section) {

60 err := json.Unmarshal(buf, &doc)

61 if err != nil {

62 pkglog.Fatalx("parsing webmail api docs", err, slog.String("api", api))

63 }

64 return doc

65}

66

67var webmailDoc = mustParseAPI("webmail", webmailapiJSON)

68

69var sherpaHandlerOpts *sherpa.HandlerOpts

70

71func makeSherpaHandler(maxMessageSize int64, cookiePath string, isForwarded bool) (http.Handler, error) {

72 return sherpa.NewHandler("/api/", moxvar.Version, Webmail{maxMessageSize, cookiePath, isForwarded}, &webmailDoc, sherpaHandlerOpts)

73}

74

75func init() {

76 collector, err := sherpaprom.NewCollector("moxwebmail", nil)

77 if err != nil {

78 pkglog.Fatalx("creating sherpa prometheus collector", err)

79 }

80

81 sherpaHandlerOpts = &sherpa.HandlerOpts{Collector: collector, AdjustFunctionNames: "none", NoCORS: true}

82 // Just to validate.

83 _, err = makeSherpaHandler(0, "", false)

84 if err != nil {

85 pkglog.Fatalx("sherpa handler", err)

86 }

87}

88

89// LoginPrep returns a login token, and also sets it as cookie. Both must be

90// present in the call to Login.

91func (w Webmail) LoginPrep(ctx context.Context) string {

92 log := pkglog.WithContext(ctx)

93 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

94

95 var data [8]byte

96 _, err := cryptorand.Read(data[:])

97 xcheckf(ctx, err, "generate token")

98 loginToken := base64.RawURLEncoding.EncodeToString(data[:])

99

100 webauth.LoginPrep(ctx, log, "webmail", w.cookiePath, w.isForwarded, reqInfo.Response, reqInfo.Request, loginToken)

101

102 return loginToken

103}

104

105// Login returns a session token for the credentials, or fails with error code

106// "user:badLogin". Call LoginPrep to get a loginToken.

107func (w Webmail) Login(ctx context.Context, loginToken, username, password string) store.CSRFToken {

108 log := pkglog.WithContext(ctx)

109 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

110

111 csrfToken, err := webauth.Login(ctx, log, webauth.Accounts, "webmail", w.cookiePath, w.isForwarded, reqInfo.Response, reqInfo.Request, loginToken, username, password)

112 if _, ok := err.(*sherpa.Error); ok {

113 panic(err)

114 }

115 xcheckf(ctx, err, "login")

116 return csrfToken

117}

118

119// Logout invalidates the session token.

120func (w Webmail) Logout(ctx context.Context) {

121 log := pkglog.WithContext(ctx)

122 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

123

124 err := webauth.Logout(ctx, log, webauth.Accounts, "webmail", w.cookiePath, w.isForwarded, reqInfo.Response, reqInfo.Request, reqInfo.AccountName, reqInfo.SessionToken)

125 xcheckf(ctx, err, "logout")

126}

127

128// Token returns a token to use for an SSE connection. A token can only be used for

129// a single SSE connection. Tokens are stored in memory for a maximum of 1 minute,

130// with at most 10 unused tokens (the most recently created) per account.

131func (Webmail) Token(ctx context.Context) string {

132 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

133 return sseTokens.xgenerate(ctx, reqInfo.AccountName, reqInfo.LoginAddress, reqInfo.SessionToken)

134}

135

136// Requests sends a new request for an open SSE connection. Any currently active

137// request for the connection will be canceled, but this is done asynchrously, so

138// the SSE connection may still send results for the previous request. Callers

139// should take care to ignore such results. If req.Cancel is set, no new request is

140// started.

141func (Webmail) Request(ctx context.Context, req Request) {

142 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

143

144 if !req.Cancel && req.Page.Count <= 0 {

145 xcheckuserf(ctx, errors.New("Page.Count must be >= 1"), "checking request")

146 }

147

148 sse, ok := sseGet(req.SSEID, reqInfo.AccountName)

149 if !ok {

150 xcheckuserf(ctx, errors.New("unknown sseid"), "looking up connection")

151 }

152 sse.Request <- req

153}

154

155// ParsedMessage returns enough to render the textual body of a message. It is

156// assumed the client already has other fields through MessageItem.

157func (Webmail) ParsedMessage(ctx context.Context, msgID int64) (pm ParsedMessage) {

158 log := pkglog.WithContext(ctx)

159 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

160 acc, err := store.OpenAccount(log, reqInfo.AccountName)

161 xcheckf(ctx, err, "open account")

162 defer func() {

163 err := acc.Close()

164 log.Check(err, "closing account")

165 }()

166

167 var m store.Message

168 xdbread(ctx, acc, func(tx *bstore.Tx) {

169 m = xmessageID(ctx, tx, msgID)

170 })

171

172 state := msgState{acc: acc}

173 defer state.clear()

174 pm, err = parsedMessage(log, m, &state, true, false)

175 xcheckf(ctx, err, "parsing message")

176 return

177}

178

179// Attachment is a MIME part is an existing message that is not intended as

180// viewable text or HTML part.

181type Attachment struct {

182 Path []int // Indices into top-level message.Part.Parts.

183

184 // File name based on "name" attribute of "Content-Type", or the "filename"

185 // attribute of "Content-Disposition".

186 Filename string

187

188 Part message.Part

189}

190

191// SubmitMessage is an email message to be sent to one or more recipients.

192// Addresses are formatted as just email address, or with a name like "name

193// <user@host>".

194type SubmitMessage struct {

195 From string

196 To []string

197 Cc []string

198 Bcc []string

199 Subject string

200 TextBody string

201 Attachments []File

202 ForwardAttachments ForwardAttachments

203 IsForward bool

204 ResponseMessageID int64 // If set, this was a reply or forward, based on IsForward.

205 ReplyTo string // If non-empty, Reply-To header to add to message.

206 UserAgent string // User-Agent header added if not empty.

207 RequireTLS *bool // For "Require TLS" extension during delivery.

208 FutureRelease *time.Time // If set, time (in the future) when message should be delivered from queue.

209}

210

211// ForwardAttachments references attachments by a list of message.Part paths.

212type ForwardAttachments struct {

213 MessageID int64 // Only relevant if MessageID is not 0.

214 Paths [][]int // List of attachments, each path is a list of indices into the top-level message.Part.Parts.

215}

216

217// File is a new attachment (not from an existing message that is being

218// forwarded) to send with a SubmitMessage.

219type File struct {

220 Filename string

221 DataURI string // Full data of the attachment, with base64 encoding and including content-type.

222}

223

224// parseAddress expects either a plain email address like "user@domain", or a

225// single address as used in a message header, like "name <user@domain>".

226func parseAddress(msghdr string) (message.NameAddress, error) {

227 a, err := mail.ParseAddress(msghdr)

228 if err != nil {

229 return message.NameAddress{}, err

230 }

231

232 // todo: parse more fully according to ../rfc/5322:959

233 path, err := smtp.ParseAddress(a.Address)

234 if err != nil {

235 return message.NameAddress{}, err

236 }

237 return message.NameAddress{DisplayName: a.Name, Address: path}, nil

238}

239

240func xmailboxID(ctx context.Context, tx *bstore.Tx, mailboxID int64) store.Mailbox {

241 if mailboxID == 0 {

242 xcheckuserf(ctx, errors.New("invalid zero mailbox ID"), "getting mailbox")

243 }

244 mb := store.Mailbox{ID: mailboxID}

245 err := tx.Get(&mb)

246 if err == bstore.ErrAbsent {

247 xcheckuserf(ctx, err, "getting mailbox")

248 }

249 xcheckf(ctx, err, "getting mailbox")

250 return mb

251}

252

253// xmessageID returns a non-expunged message or panics with a sherpa error.

254func xmessageID(ctx context.Context, tx *bstore.Tx, messageID int64) store.Message {

255 if messageID == 0 {

256 xcheckuserf(ctx, errors.New("invalid zero message id"), "getting message")

257 }

258 m := store.Message{ID: messageID}

259 err := tx.Get(&m)

260 if err == bstore.ErrAbsent {

261 xcheckuserf(ctx, errors.New("message does not exist"), "getting message")

262 } else if err == nil && m.Expunged {

263 xcheckuserf(ctx, errors.New("message was removed"), "getting message")

264 }

265 xcheckf(ctx, err, "getting message")

266 return m

267}

268

269// MessageSubmit sends a message by submitting it the outgoing email queue. The

270// message is sent to all addresses listed in the To, Cc and Bcc addresses, without

271// Bcc message header.

272//

273// If a Sent mailbox is configured, messages are added to it after submitting

274// to the delivery queue.

275func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {

276 // Similar between ../smtpserver/server.go:/submit\( and ../webmail/webmail.go:/MessageSubmit\(

277

278 // todo: consider making this an HTTP POST, so we can upload as regular form, which is probably more efficient for encoding for the client and we can stream the data in.

279

280 // Prevent any accidental control characters, or attempts at getting bare \r or \n

281 // into messages.

282 for _, l := range [][]string{m.To, m.Cc, m.Bcc, {m.From, m.Subject, m.ReplyTo, m.UserAgent}} {

283 for _, s := range l {

284 for _, c := range s {

285 if c < 0x20 {

286 xcheckuserf(ctx, errors.New("control characters not allowed"), "checking header values")

287 }

288 }

289 }

290 }

291

292 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

293 log := pkglog.WithContext(ctx).With(slog.String("account", reqInfo.AccountName))

294 acc, err := store.OpenAccount(log, reqInfo.AccountName)

295 xcheckf(ctx, err, "open account")

296 defer func() {

297 err := acc.Close()

298 log.Check(err, "closing account")

299 }()

300

301 log.Debug("message submit")

302

303 fromAddr, err := parseAddress(m.From)

304 xcheckuserf(ctx, err, "parsing From address")

305

306 var replyTo *message.NameAddress

307 if m.ReplyTo != "" {

308 a, err := parseAddress(m.ReplyTo)

309 xcheckuserf(ctx, err, "parsing Reply-To address")

310 replyTo = &a

311 }

312

313 var recipients []smtp.Address

314

315 var toAddrs []message.NameAddress

316 for _, s := range m.To {

317 addr, err := parseAddress(s)

318 xcheckuserf(ctx, err, "parsing To address")

319 toAddrs = append(toAddrs, addr)

320 recipients = append(recipients, addr.Address)

321 }

322

323 var ccAddrs []message.NameAddress

324 for _, s := range m.Cc {

325 addr, err := parseAddress(s)

326 xcheckuserf(ctx, err, "parsing Cc address")

327 ccAddrs = append(ccAddrs, addr)

328 recipients = append(recipients, addr.Address)

329 }

330

331 for _, s := range m.Bcc {

332 addr, err := parseAddress(s)

333 xcheckuserf(ctx, err, "parsing Bcc address")

334 recipients = append(recipients, addr.Address)

335 }

336

337 // Check if from address is allowed for account.

338 fromAccName, _, _, err := mox.FindAccount(fromAddr.Address.Localpart, fromAddr.Address.Domain, false)

339 if err == nil && fromAccName != reqInfo.AccountName {

340 err = mox.ErrAccountNotFound

341 }

342 if err != nil && (errors.Is(err, mox.ErrAccountNotFound) || errors.Is(err, mox.ErrDomainNotFound)) {

343 metricSubmission.WithLabelValues("badfrom").Inc()

344 xcheckuserf(ctx, errors.New("address not found"), `looking up "from" address for account`)

345 }

346 xcheckf(ctx, err, "checking if from address is allowed")

347

348 if len(recipients) == 0 {

349 xcheckuserf(ctx, errors.New("no recipients"), "composing message")

350 }

351

352 // Check outgoing message rate limit.

353 xdbread(ctx, acc, func(tx *bstore.Tx) {

354 rcpts := make([]smtp.Path, len(recipients))

355 for i, r := range recipients {

356 rcpts[i] = smtp.Path{Localpart: r.Localpart, IPDomain: dns.IPDomain{Domain: r.Domain}}

357 }

358 msglimit, rcptlimit, err := acc.SendLimitReached(tx, rcpts)

359 if msglimit >= 0 {

360 metricSubmission.WithLabelValues("messagelimiterror").Inc()

361 xcheckuserf(ctx, errors.New("send message limit reached"), "checking outgoing rate limit")

362 } else if rcptlimit >= 0 {

363 metricSubmission.WithLabelValues("recipientlimiterror").Inc()

364 xcheckuserf(ctx, errors.New("send message limit reached"), "checking outgoing rate limit")

365 }

366 xcheckf(ctx, err, "checking send limit")

367 })

368

369 has8bit := false // We update this later on.

370

371 // We only use smtputf8 if we have to, with a utf-8 localpart. For IDNA, we use ASCII domains.

372 smtputf8 := false

373 for _, a := range recipients {

374 if a.Localpart.IsInternational() {

375 smtputf8 = true

376 break

377 }

378 }

379 if !smtputf8 && fromAddr.Address.Localpart.IsInternational() {

380 // todo: may want to warn user that they should consider sending with a ascii-only localpart, in case receiver doesn't support smtputf8.

381 smtputf8 = true

382 }

383

384 // Create file to compose message into.

385 dataFile, err := store.CreateMessageTemp(log, "webmail-submit")

386 xcheckf(ctx, err, "creating temporary file for message")

387 defer store.CloseRemoveTempFile(log, dataFile, "message to submit")

388

389 // If writing to the message file fails, we abort immediately.

390 xc := message.NewComposer(dataFile, w.maxMessageSize)

391 defer func() {

392 x := recover()

393 if x == nil {

394 return

395 }

396 if err, ok := x.(error); ok && errors.Is(err, message.ErrMessageSize) {

397 xcheckuserf(ctx, err, "making message")

398 } else if ok && errors.Is(err, message.ErrCompose) {

399 xcheckf(ctx, err, "making message")

400 }

401 panic(x)

402 }()

403

404 // todo spec: can we add an Authentication-Results header that indicates this is an authenticated message? the "auth" method is for SMTP AUTH, which this isn't. ../rfc/8601 https://www.iana.org/assignments/email-auth/email-auth.xhtml

405

406 // Each queued message gets a Received header.

407 // We don't have access to the local IP for adding.

408 // We cannot use VIA, because there is no registered method. We would like to use

409 // it to add the ascii domain name in case of smtputf8 and IDNA host name.

410 recvFrom := message.HeaderCommentDomain(mox.Conf.Static.HostnameDomain, smtputf8)

411 recvBy := mox.Conf.Static.HostnameDomain.XName(smtputf8)

412 recvID := mox.ReceivedID(mox.CidFromCtx(ctx))

413 recvHdrFor := func(rcptTo string) string {

414 recvHdr := &message.HeaderWriter{}

415 // For additional Received-header clauses, see:

416 // https://www.iana.org/assignments/mail-parameters/mail-parameters.xhtml#table-mail-parameters-8

417 // Note: we don't have "via" or "with", there is no registered for webmail.

418 recvHdr.Add(" ", "Received:", "from", recvFrom, "by", recvBy, "id", recvID) // ../rfc/5321:3158

419 if reqInfo.Request.TLS != nil {

420 recvHdr.Add(" ", mox.TLSReceivedComment(log, *reqInfo.Request.TLS)...)

421 }

422 recvHdr.Add(" ", "for", "<"+rcptTo+">;", time.Now().Format(message.RFC5322Z))

423 return recvHdr.String()

424 }

425

426 // Outer message headers.

427 xc.HeaderAddrs("From", []message.NameAddress{fromAddr})

428 if replyTo != nil {

429 xc.HeaderAddrs("Reply-To", []message.NameAddress{*replyTo})

430 }

431 xc.HeaderAddrs("To", toAddrs)

432 xc.HeaderAddrs("Cc", ccAddrs)

433 if m.Subject != "" {

434 xc.Subject(m.Subject)

435 }

436

437 messageID := fmt.Sprintf("<%s>", mox.MessageIDGen(smtputf8))

438 xc.Header("Message-Id", messageID)

439 xc.Header("Date", time.Now().Format(message.RFC5322Z))

440 // Add In-Reply-To and References headers.

441 if m.ResponseMessageID > 0 {

442 xdbread(ctx, acc, func(tx *bstore.Tx) {

443 rm := xmessageID(ctx, tx, m.ResponseMessageID)

444 msgr := acc.MessageReader(rm)

445 defer func() {

446 err := msgr.Close()

447 log.Check(err, "closing message reader")

448 }()

449 rp, err := rm.LoadPart(msgr)

450 xcheckf(ctx, err, "load parsed message")

451 h, err := rp.Header()

452 xcheckf(ctx, err, "parsing header")

453

454 if rp.Envelope == nil {

455 return

456 }

457 xc.Header("In-Reply-To", rp.Envelope.MessageID)

458 ref := h.Get("References")

459 if ref == "" {

460 ref = h.Get("In-Reply-To")

461 }

462 if ref != "" {

463 xc.Header("References", ref+"\r\n\t"+rp.Envelope.MessageID)

464 } else {

465 xc.Header("References", rp.Envelope.MessageID)

466 }

467 })

468 }

469 if m.UserAgent != "" {

470 xc.Header("User-Agent", m.UserAgent)

471 }

472 if m.RequireTLS != nil && !*m.RequireTLS {

473 xc.Header("TLS-Required", "No")

474 }

475 xc.Header("MIME-Version", "1.0")

476

477 if len(m.Attachments) > 0 || len(m.ForwardAttachments.Paths) > 0 {

478 mp := multipart.NewWriter(xc)

479 xc.Header("Content-Type", fmt.Sprintf(`multipart/mixed; boundary="%s"`, mp.Boundary()))

480 xc.Line()

481

482 textBody, ct, cte := xc.TextPart(m.TextBody)

483 textHdr := textproto.MIMEHeader{}

484 textHdr.Set("Content-Type", ct)

485 textHdr.Set("Content-Transfer-Encoding", cte)

486

487 textp, err := mp.CreatePart(textHdr)

488 xcheckf(ctx, err, "adding text part to message")

489 _, err = textp.Write(textBody)

490 xcheckf(ctx, err, "writing text part")

491

492 xaddPart := func(ct, filename string) io.Writer {

493 ahdr := textproto.MIMEHeader{}

494 cd := mime.FormatMediaType("attachment", map[string]string{"filename": filename})

495

496 ahdr.Set("Content-Type", ct)

497 ahdr.Set("Content-Transfer-Encoding", "base64")

498 ahdr.Set("Content-Disposition", cd)

499 ap, err := mp.CreatePart(ahdr)

500 xcheckf(ctx, err, "adding attachment part to message")

501 return ap

502 }

503

504 xaddAttachmentBase64 := func(ct, filename string, base64Data []byte) {

505 ap := xaddPart(ct, filename)

506

507 for len(base64Data) > 0 {

508 line := base64Data

509 n := len(line)

510 if n > 78 {

511 n = 78

512 }

513 line, base64Data = base64Data[:n], base64Data[n:]

514 _, err := ap.Write(line)

515 xcheckf(ctx, err, "writing attachment")

516 _, err = ap.Write([]byte("\r\n"))

517 xcheckf(ctx, err, "writing attachment")

518 }

519 }

520

521 xaddAttachment := func(ct, filename string, r io.Reader) {

522 ap := xaddPart(ct, filename)

523 wc := moxio.Base64Writer(ap)

524 _, err := io.Copy(wc, r)

525 xcheckf(ctx, err, "adding attachment")

526 err = wc.Close()

527 xcheckf(ctx, err, "flushing attachment")

528 }

529

530 for _, a := range m.Attachments {

531 s := a.DataURI

532 if !strings.HasPrefix(s, "data:") {

533 xcheckuserf(ctx, errors.New("missing data: in datauri"), "parsing attachment")

534 }

535 s = s[len("data:"):]

536 t := strings.SplitN(s, ",", 2)

537 if len(t) != 2 {

538 xcheckuserf(ctx, errors.New("missing comma in datauri"), "parsing attachment")

539 }

540 if !strings.HasSuffix(t[0], "base64") {

541 xcheckuserf(ctx, errors.New("missing base64 in datauri"), "parsing attachment")

542 }

543 ct := strings.TrimSuffix(t[0], "base64")

544 ct = strings.TrimSuffix(ct, ";")

545 if ct == "" {

546 ct = "application/octet-stream"

547 }

548 filename := a.Filename

549 if filename == "" {

550 filename = "unnamed.bin"

551 }

552 params := map[string]string{"name": filename}

553 ct = mime.FormatMediaType(ct, params)

554

555 // Ensure base64 is valid, then we'll write the original string.

556 _, err := io.Copy(io.Discard, base64.NewDecoder(base64.StdEncoding, strings.NewReader(t[1])))

557 xcheckuserf(ctx, err, "parsing attachment as base64")

558

559 xaddAttachmentBase64(ct, filename, []byte(t[1]))

560 }

561

562 if len(m.ForwardAttachments.Paths) > 0 {

563 acc.WithRLock(func() {

564 xdbread(ctx, acc, func(tx *bstore.Tx) {

565 fm := xmessageID(ctx, tx, m.ForwardAttachments.MessageID)

566 msgr := acc.MessageReader(fm)

567 defer func() {

568 err := msgr.Close()

569 log.Check(err, "closing message reader")

570 }()

571

572 fp, err := fm.LoadPart(msgr)

573 xcheckf(ctx, err, "load parsed message")

574

575 for _, path := range m.ForwardAttachments.Paths {

576 ap := fp

577 for _, xp := range path {

578 if xp < 0 || xp >= len(ap.Parts) {

579 xcheckuserf(ctx, errors.New("unknown part"), "looking up attachment")

580 }

581 ap = ap.Parts[xp]

582 }

583

584 filename := tryDecodeParam(log, ap.ContentTypeParams["name"])

585 if filename == "" {

586 filename = "unnamed.bin"

587 }

588 params := map[string]string{"name": filename}

589 if pcharset := ap.ContentTypeParams["charset"]; pcharset != "" {

590 params["charset"] = pcharset

591 }

592 ct := strings.ToLower(ap.MediaType + "/" + ap.MediaSubType)

593 ct = mime.FormatMediaType(ct, params)

594 xaddAttachment(ct, filename, ap.Reader())

595 }

596 })

597 })

598 }

599

600 err = mp.Close()

601 xcheckf(ctx, err, "writing mime multipart")

602 } else {

603 textBody, ct, cte := xc.TextPart(m.TextBody)

604 xc.Header("Content-Type", ct)

605 xc.Header("Content-Transfer-Encoding", cte)

606 xc.Line()

607 xc.Write([]byte(textBody))

608 }

609

610 xc.Flush()

611

612 // Add DKIM-Signature headers.

613 var msgPrefix string

614 fd := fromAddr.Address.Domain

615 confDom, _ := mox.Conf.Domain(fd)

616 selectors := mox.DKIMSelectors(confDom.DKIM)

617 if len(selectors) > 0 {

618 dkimHeaders, err := dkim.Sign(ctx, log.Logger, fromAddr.Address.Localpart, fd, selectors, smtputf8, dataFile)

619 if err != nil {

620 metricServerErrors.WithLabelValues("dkimsign").Inc()

621 }

622 xcheckf(ctx, err, "sign dkim")

623

624 msgPrefix = dkimHeaders

625 }

626

627 fromPath := smtp.Path{

628 Localpart: fromAddr.Address.Localpart,

629 IPDomain: dns.IPDomain{Domain: fromAddr.Address.Domain},

630 }

631 qml := make([]queue.Msg, len(recipients))

632 now := time.Now()

633 for i, rcpt := range recipients {

634 // Don't use per-recipient unique message prefix when multiple recipients are

635 // present, or the queue cannot deliver it in a single smtp transaction.

636 var recvRcpt string

637 if len(recipients) == 1 {

638 recvRcpt = rcpt.Pack(smtputf8)

639 }

640 rcptMsgPrefix := recvHdrFor(recvRcpt) + msgPrefix

641 msgSize := int64(len(rcptMsgPrefix)) + xc.Size

642 toPath := smtp.Path{

643 Localpart: rcpt.Localpart,

644 IPDomain: dns.IPDomain{Domain: rcpt.Domain},

645 }

646 qm := queue.MakeMsg(fromPath, toPath, has8bit, smtputf8, msgSize, messageID, []byte(rcptMsgPrefix), m.RequireTLS, now)

647 if m.FutureRelease != nil {

648 ival := time.Until(*m.FutureRelease)

649 if ival < 0 {

650 xcheckuserf(ctx, errors.New("date/time is in the past"), "scheduling delivery")

651 } else if ival > queue.FutureReleaseIntervalMax {

652 xcheckuserf(ctx, fmt.Errorf("date/time can not be further than %v in the future", queue.FutureReleaseIntervalMax), "scheduling delivery")

653 }

654 qm.NextAttempt = *m.FutureRelease

655 qm.FutureReleaseRequest = "until;" + m.FutureRelease.Format(time.RFC3339)

656 // todo: possibly add a header to the message stored in the Sent mailbox to indicate it was scheduled for later delivery.

657 }

658 qml[i] = qm

659 }

660 err = queue.Add(ctx, log, reqInfo.AccountName, dataFile, qml...)

661 if err != nil {

662 metricSubmission.WithLabelValues("queueerror").Inc()

663 }

664 xcheckf(ctx, err, "adding messages to the delivery queue")

665 metricSubmission.WithLabelValues("ok").Inc()

666

667 var modseq store.ModSeq // Only set if needed.

668

669 // Append message to Sent mailbox and mark original messages as answered/forwarded.

670 acc.WithRLock(func() {

671 var changes []store.Change

672

673 metricked := false

674 defer func() {

675 if x := recover(); x != nil {

676 if !metricked {

677 metricServerErrors.WithLabelValues("submit").Inc()

678 }

679 panic(x)

680 }

681 }()

682 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

683 if m.ResponseMessageID > 0 {

684 rm := xmessageID(ctx, tx, m.ResponseMessageID)

685 oflags := rm.Flags

686 if m.IsForward {

687 rm.Forwarded = true

688 } else {

689 rm.Answered = true

690 }

691 if !rm.Junk && !rm.Notjunk {

692 rm.Notjunk = true

693 }

694 if rm.Flags != oflags {

695 modseq, err = acc.NextModSeq(tx)

696 xcheckf(ctx, err, "next modseq")

697 rm.ModSeq = modseq

698 err := tx.Update(&rm)

699 xcheckf(ctx, err, "updating flags of replied/forwarded message")

700 changes = append(changes, rm.ChangeFlags(oflags))

701

702 err = acc.RetrainMessages(ctx, log, tx, []store.Message{rm}, false)

703 xcheckf(ctx, err, "retraining messages after reply/forward")

704 }

705 }

706

707 sentmb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Sent", true).Get()

708 if err == bstore.ErrAbsent {

709 // There is no mailbox designated as Sent mailbox, so we're done.

710 return

711 }

712 xcheckf(ctx, err, "message submitted to queue, adding to Sent mailbox")

713

714 if modseq == 0 {

715 modseq, err = acc.NextModSeq(tx)

716 xcheckf(ctx, err, "next modseq")

717 }

718

719 sentm := store.Message{

720 CreateSeq: modseq,

721 ModSeq: modseq,

722 MailboxID: sentmb.ID,

723 MailboxOrigID: sentmb.ID,

724 Flags: store.Flags{Notjunk: true, Seen: true},

725 Size: int64(len(msgPrefix)) + xc.Size,

726 MsgPrefix: []byte(msgPrefix),

727 }

728

729 if ok, maxSize, err := acc.CanAddMessageSize(tx, sentm.Size); err != nil {

730 xcheckf(ctx, err, "checking quota")

731 } else if !ok {

732 xcheckuserf(ctx, fmt.Errorf("account over maximum total message size %d", maxSize), "checking quota")

733 }

734

735 // Update mailbox before delivery, which changes uidnext.

736 sentmb.Add(sentm.MailboxCounts())

737 err = tx.Update(&sentmb)

738 xcheckf(ctx, err, "updating sent mailbox for counts")

739

740 err = acc.DeliverMessage(log, tx, &sentm, dataFile, true, false, false, true)

741 if err != nil {

742 metricSubmission.WithLabelValues("storesenterror").Inc()

743 metricked = true

744 }

745 xcheckf(ctx, err, "message submitted to queue, appending message to Sent mailbox")

746

747 changes = append(changes, sentm.ChangeAddUID(), sentmb.ChangeCounts())

748 })

749

750 store.BroadcastChanges(acc, changes)

751 })

752}

753

754// MessageMove moves messages to another mailbox. If the message is already in

755// the mailbox an error is returned.

756func (Webmail) MessageMove(ctx context.Context, messageIDs []int64, mailboxID int64) {

757 log := pkglog.WithContext(ctx)

758 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

759 acc, err := store.OpenAccount(log, reqInfo.AccountName)

760 xcheckf(ctx, err, "open account")

761 defer func() {

762 err := acc.Close()

763 log.Check(err, "closing account")

764 }()

765

766 acc.WithRLock(func() {

767 retrain := make([]store.Message, 0, len(messageIDs))

768 removeChanges := map[int64]store.ChangeRemoveUIDs{}

769 // n adds, 1 remove, 2 mailboxcounts, optimistic and at least for a single message.

770 changes := make([]store.Change, 0, len(messageIDs)+3)

771

772 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

773 var mbSrc store.Mailbox

774 var modseq store.ModSeq

775

776 mbDst := xmailboxID(ctx, tx, mailboxID)

777

778 if len(messageIDs) == 0 {

779 return

780 }

781

782 keywords := map[string]struct{}{}

783

784 for _, mid := range messageIDs {

785 m := xmessageID(ctx, tx, mid)

786

787 // We may have loaded this mailbox in the previous iteration of this loop.

788 if m.MailboxID != mbSrc.ID {

789 if mbSrc.ID != 0 {

790 err = tx.Update(&mbSrc)

791 xcheckf(ctx, err, "updating source mailbox counts")

792 changes = append(changes, mbSrc.ChangeCounts())

793 }

794 mbSrc = xmailboxID(ctx, tx, m.MailboxID)

795 }

796

797 if mbSrc.ID == mailboxID {

798 // Client should filter out messages that are already in mailbox.

799 xcheckuserf(ctx, errors.New("already in destination mailbox"), "moving message")

800 }

801

802 if modseq == 0 {

803 modseq, err = acc.NextModSeq(tx)

804 xcheckf(ctx, err, "assigning next modseq")

805 }

806

807 ch := removeChanges[m.MailboxID]

808 ch.UIDs = append(ch.UIDs, m.UID)

809 ch.ModSeq = modseq

810 ch.MailboxID = m.MailboxID

811 removeChanges[m.MailboxID] = ch

812

813 // Copy of message record that we'll insert when UID is freed up.

814 om := m

815 om.PrepareExpunge()

816 om.ID = 0 // Assign new ID.

817 om.ModSeq = modseq

818

819 mbSrc.Sub(m.MailboxCounts())

820

821 if mbDst.Trash {

822 m.Seen = true

823 }

824 conf, _ := acc.Conf()

825 m.MailboxID = mbDst.ID

826 if m.IsReject && m.MailboxDestinedID != 0 {

827 // Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message

828 // is used for reputation calculation during future deliveries.

829 m.MailboxOrigID = m.MailboxDestinedID

830 m.IsReject = false

831 m.Seen = false

832 }

833 m.UID = mbDst.UIDNext

834 m.ModSeq = modseq

835 mbDst.UIDNext++

836 m.JunkFlagsForMailbox(mbDst, conf)

837 err = tx.Update(&m)

838 xcheckf(ctx, err, "updating moved message in database")

839

840 // Now that UID is unused, we can insert the old record again.

841 err = tx.Insert(&om)

842 xcheckf(ctx, err, "inserting record for expunge after moving message")

843

844 mbDst.Add(m.MailboxCounts())

845

846 changes = append(changes, m.ChangeAddUID())

847 retrain = append(retrain, m)

848

849 for _, kw := range m.Keywords {

850 keywords[kw] = struct{}{}

851 }

852 }

853

854 err = tx.Update(&mbSrc)

855 xcheckf(ctx, err, "updating source mailbox counts")

856

857 changes = append(changes, mbSrc.ChangeCounts(), mbDst.ChangeCounts())

858

859 // Ensure destination mailbox has keywords of the moved messages.

860 var mbKwChanged bool

861 mbDst.Keywords, mbKwChanged = store.MergeKeywords(mbDst.Keywords, maps.Keys(keywords))

862 if mbKwChanged {

863 changes = append(changes, mbDst.ChangeKeywords())

864 }

865

866 err = tx.Update(&mbDst)

867 xcheckf(ctx, err, "updating mailbox with uidnext")

868

869 err = acc.RetrainMessages(ctx, log, tx, retrain, false)

870 xcheckf(ctx, err, "retraining messages after move")

871 })

872

873 // Ensure UIDs of the removed message are in increasing order. It is quite common

874 // for all messages to be from a single source mailbox, meaning this is just one

875 // change, for which we preallocated space.

876 for _, ch := range removeChanges {

877 sort.Slice(ch.UIDs, func(i, j int) bool {

878 return ch.UIDs[i] < ch.UIDs[j]

879 })

880 changes = append(changes, ch)

881 }

882 store.BroadcastChanges(acc, changes)

883 })

884}

885

886// MessageDelete permanently deletes messages, without moving them to the Trash mailbox.

887func (Webmail) MessageDelete(ctx context.Context, messageIDs []int64) {

888 log := pkglog.WithContext(ctx)

889 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

890 acc, err := store.OpenAccount(log, reqInfo.AccountName)

891 xcheckf(ctx, err, "open account")

892 defer func() {

893 err := acc.Close()

894 log.Check(err, "closing account")

895 }()

896

897 if len(messageIDs) == 0 {

898 return

899 }

900

901 acc.WithWLock(func() {

902 removeChanges := map[int64]store.ChangeRemoveUIDs{}

903 changes := make([]store.Change, 0, len(messageIDs)+1) // n remove, 1 mailbox counts

904

905 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

906 var modseq store.ModSeq

907 var mb store.Mailbox

908 remove := make([]store.Message, 0, len(messageIDs))

909

910 var totalSize int64

911 for _, mid := range messageIDs {

912 m := xmessageID(ctx, tx, mid)

913 totalSize += m.Size

914

915 if m.MailboxID != mb.ID {

916 if mb.ID != 0 {

917 err := tx.Update(&mb)

918 xcheckf(ctx, err, "updating mailbox counts")

919 changes = append(changes, mb.ChangeCounts())

920 }

921 mb = xmailboxID(ctx, tx, m.MailboxID)

922 }

923

924 qmr := bstore.QueryTx[store.Recipient](tx)

925 qmr.FilterEqual("MessageID", m.ID)

926 _, err = qmr.Delete()

927 xcheckf(ctx, err, "removing message recipients")

928

929 mb.Sub(m.MailboxCounts())

930

931 if modseq == 0 {

932 modseq, err = acc.NextModSeq(tx)

933 xcheckf(ctx, err, "assigning next modseq")

934 }

935 m.Expunged = true

936 m.ModSeq = modseq

937 err = tx.Update(&m)

938 xcheckf(ctx, err, "marking message as expunged")

939

940 ch := removeChanges[m.MailboxID]

941 ch.UIDs = append(ch.UIDs, m.UID)

942 ch.MailboxID = m.MailboxID

943 ch.ModSeq = modseq

944 removeChanges[m.MailboxID] = ch

945 remove = append(remove, m)

946 }

947

948 if mb.ID != 0 {

949 err := tx.Update(&mb)

950 xcheckf(ctx, err, "updating count in mailbox")

951 changes = append(changes, mb.ChangeCounts())

952 }

953

954 err = acc.AddMessageSize(log, tx, -totalSize)

955 xcheckf(ctx, err, "updating disk usage")

956

957 // Mark removed messages as not needing training, then retrain them, so if they

958 // were trained, they get untrained.

959 for i := range remove {

960 remove[i].Junk = false

961 remove[i].Notjunk = false

962 }

963 err = acc.RetrainMessages(ctx, log, tx, remove, true)

964 xcheckf(ctx, err, "untraining deleted messages")

965 })

966

967 for _, ch := range removeChanges {

968 sort.Slice(ch.UIDs, func(i, j int) bool {

969 return ch.UIDs[i] < ch.UIDs[j]

970 })

971 changes = append(changes, ch)

972 }

973 store.BroadcastChanges(acc, changes)

974 })

975

976 for _, mID := range messageIDs {

977 p := acc.MessagePath(mID)

978 err := os.Remove(p)

979 log.Check(err, "removing message file for expunge")

980 }

981}

982

983// FlagsAdd adds flags, either system flags like \Seen or custom keywords. The

984// flags should be lower-case, but will be converted and verified.

985func (Webmail) FlagsAdd(ctx context.Context, messageIDs []int64, flaglist []string) {

986 log := pkglog.WithContext(ctx)

987 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

988 acc, err := store.OpenAccount(log, reqInfo.AccountName)

989 xcheckf(ctx, err, "open account")

990 defer func() {

991 err := acc.Close()

992 log.Check(err, "closing account")

993 }()

994

995 flags, keywords, err := store.ParseFlagsKeywords(flaglist)

996 xcheckuserf(ctx, err, "parsing flags")

997

998 acc.WithRLock(func() {

999 var changes []store.Change

1000

1001 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1002 var modseq store.ModSeq

1003 var retrain []store.Message

1004 var mb, origmb store.Mailbox

1005

1006 for _, mid := range messageIDs {

1007 m := xmessageID(ctx, tx, mid)

1008

1009 if mb.ID != m.MailboxID {

1010 if mb.ID != 0 {

1011 err := tx.Update(&mb)

1012 xcheckf(ctx, err, "updating mailbox")

1013 if mb.MailboxCounts != origmb.MailboxCounts {

1014 changes = append(changes, mb.ChangeCounts())

1015 }

1016 if mb.KeywordsChanged(origmb) {

1017 changes = append(changes, mb.ChangeKeywords())

1018 }

1019 }

1020 mb = xmailboxID(ctx, tx, m.MailboxID)

1021 origmb = mb

1022 }

1023 mb.Keywords, _ = store.MergeKeywords(mb.Keywords, keywords)

1024

1025 mb.Sub(m.MailboxCounts())

1026 oflags := m.Flags

1027 m.Flags = m.Flags.Set(flags, flags)

1028 var kwChanged bool

1029 m.Keywords, kwChanged = store.MergeKeywords(m.Keywords, keywords)

1030 mb.Add(m.MailboxCounts())

1031

1032 if m.Flags == oflags && !kwChanged {

1033 continue

1034 }

1035

1036 if modseq == 0 {

1037 modseq, err = acc.NextModSeq(tx)

1038 xcheckf(ctx, err, "assigning next modseq")

1039 }

1040 m.ModSeq = modseq

1041 err = tx.Update(&m)

1042 xcheckf(ctx, err, "updating message")

1043

1044 changes = append(changes, m.ChangeFlags(oflags))

1045 retrain = append(retrain, m)

1046 }

1047

1048 if mb.ID != 0 {

1049 err := tx.Update(&mb)

1050 xcheckf(ctx, err, "updating mailbox")

1051 if mb.MailboxCounts != origmb.MailboxCounts {

1052 changes = append(changes, mb.ChangeCounts())

1053 }

1054 if mb.KeywordsChanged(origmb) {

1055 changes = append(changes, mb.ChangeKeywords())

1056 }

1057 }

1058

1059 err = acc.RetrainMessages(ctx, log, tx, retrain, false)

1060 xcheckf(ctx, err, "retraining messages")

1061 })

1062

1063 store.BroadcastChanges(acc, changes)

1064 })

1065}

1066

1067// FlagsClear clears flags, either system flags like \Seen or custom keywords.

1068func (Webmail) FlagsClear(ctx context.Context, messageIDs []int64, flaglist []string) {

1069 log := pkglog.WithContext(ctx)

1070 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1071 acc, err := store.OpenAccount(log, reqInfo.AccountName)

1072 xcheckf(ctx, err, "open account")

1073 defer func() {

1074 err := acc.Close()

1075 log.Check(err, "closing account")

1076 }()

1077

1078 flags, keywords, err := store.ParseFlagsKeywords(flaglist)

1079 xcheckuserf(ctx, err, "parsing flags")

1080

1081 acc.WithRLock(func() {

1082 var retrain []store.Message

1083 var changes []store.Change

1084

1085 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1086 var modseq store.ModSeq

1087 var mb, origmb store.Mailbox

1088

1089 for _, mid := range messageIDs {

1090 m := xmessageID(ctx, tx, mid)

1091

1092 if mb.ID != m.MailboxID {

1093 if mb.ID != 0 {

1094 err := tx.Update(&mb)

1095 xcheckf(ctx, err, "updating counts for mailbox")

1096 if mb.MailboxCounts != origmb.MailboxCounts {

1097 changes = append(changes, mb.ChangeCounts())

1098 }

1099 // note: cannot remove keywords from mailbox by removing keywords from message.

1100 }

1101 mb = xmailboxID(ctx, tx, m.MailboxID)

1102 origmb = mb

1103 }

1104

1105 oflags := m.Flags

1106 mb.Sub(m.MailboxCounts())

1107 m.Flags = m.Flags.Set(flags, store.Flags{})

1108 var changed bool

1109 m.Keywords, changed = store.RemoveKeywords(m.Keywords, keywords)

1110 mb.Add(m.MailboxCounts())

1111

1112 if m.Flags == oflags && !changed {

1113 continue

1114 }

1115

1116 if modseq == 0 {

1117 modseq, err = acc.NextModSeq(tx)

1118 xcheckf(ctx, err, "assigning next modseq")

1119 }

1120 m.ModSeq = modseq

1121 err = tx.Update(&m)

1122 xcheckf(ctx, err, "updating message")

1123

1124 changes = append(changes, m.ChangeFlags(oflags))

1125 retrain = append(retrain, m)

1126 }

1127

1128 if mb.ID != 0 {

1129 err := tx.Update(&mb)

1130 xcheckf(ctx, err, "updating keywords in mailbox")

1131 if mb.MailboxCounts != origmb.MailboxCounts {

1132 changes = append(changes, mb.ChangeCounts())

1133 }

1134 // note: cannot remove keywords from mailbox by removing keywords from message.

1135 }

1136

1137 err = acc.RetrainMessages(ctx, log, tx, retrain, false)

1138 xcheckf(ctx, err, "retraining messages")

1139 })

1140

1141 store.BroadcastChanges(acc, changes)

1142 })

1143}

1144

1145// MailboxCreate creates a new mailbox.

1146func (Webmail) MailboxCreate(ctx context.Context, name string) {

1147 log := pkglog.WithContext(ctx)

1148 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1149 acc, err := store.OpenAccount(log, reqInfo.AccountName)

1150 xcheckf(ctx, err, "open account")

1151 defer func() {

1152 err := acc.Close()

1153 log.Check(err, "closing account")

1154 }()

1155

1156 name, _, err = store.CheckMailboxName(name, false)

1157 xcheckuserf(ctx, err, "checking mailbox name")

1158

1159 acc.WithWLock(func() {

1160 var changes []store.Change

1161 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1162 var exists bool

1163 var err error

1164 changes, _, exists, err = acc.MailboxCreate(tx, name)

1165 if exists {

1166 xcheckuserf(ctx, errors.New("mailbox already exists"), "creating mailbox")

1167 }

1168 xcheckf(ctx, err, "creating mailbox")

1169 })

1170

1171 store.BroadcastChanges(acc, changes)

1172 })

1173}

1174

1175// MailboxDelete deletes a mailbox and all its messages.

1176func (Webmail) MailboxDelete(ctx context.Context, mailboxID int64) {

1177 log := pkglog.WithContext(ctx)

1178 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1179 acc, err := store.OpenAccount(log, reqInfo.AccountName)

1180 xcheckf(ctx, err, "open account")

1181 defer func() {

1182 err := acc.Close()

1183 log.Check(err, "closing account")

1184 }()

1185

1186 // Messages to remove after having broadcasted the removal of messages.

1187 var removeMessageIDs []int64

1188

1189 acc.WithWLock(func() {

1190 var changes []store.Change

1191

1192 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1193 mb := xmailboxID(ctx, tx, mailboxID)

1194 if mb.Name == "Inbox" {

1195 // Inbox is special in IMAP and cannot be removed.

1196 xcheckuserf(ctx, errors.New("cannot remove special Inbox"), "checking mailbox")

1197 }

1198

1199 var hasChildren bool

1200 var err error

1201 changes, removeMessageIDs, hasChildren, err = acc.MailboxDelete(ctx, log, tx, mb)

1202 if hasChildren {

1203 xcheckuserf(ctx, errors.New("mailbox has children"), "deleting mailbox")

1204 }

1205 xcheckf(ctx, err, "deleting mailbox")

1206 })

1207

1208 store.BroadcastChanges(acc, changes)

1209 })

1210

1211 for _, mID := range removeMessageIDs {

1212 p := acc.MessagePath(mID)

1213 err := os.Remove(p)

1214 log.Check(err, "removing message file for mailbox delete", slog.String("path", p))

1215 }

1216}

1217

1218// MailboxEmpty empties a mailbox, removing all messages from the mailbox, but not

1219// its child mailboxes.

1220func (Webmail) MailboxEmpty(ctx context.Context, mailboxID int64) {

1221 log := pkglog.WithContext(ctx)

1222 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1223 acc, err := store.OpenAccount(log, reqInfo.AccountName)

1224 xcheckf(ctx, err, "open account")

1225 defer func() {

1226 err := acc.Close()

1227 log.Check(err, "closing account")

1228 }()

1229

1230 var expunged []store.Message

1231

1232 acc.WithWLock(func() {

1233 var changes []store.Change

1234

1235 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1236 mb := xmailboxID(ctx, tx, mailboxID)

1237

1238 modseq, err := acc.NextModSeq(tx)

1239 xcheckf(ctx, err, "next modseq")

1240

1241 // Mark messages as expunged.

1242 qm := bstore.QueryTx[store.Message](tx)

1243 qm.FilterNonzero(store.Message{MailboxID: mb.ID})

1244 qm.FilterEqual("Expunged", false)

1245 qm.SortAsc("UID")

1246 qm.Gather(&expunged)

1247 _, err = qm.UpdateNonzero(store.Message{ModSeq: modseq, Expunged: true})

1248 xcheckf(ctx, err, "deleting messages")

1249

1250 // Remove Recipients.

1251 anyIDs := make([]any, len(expunged))

1252 for i, m := range expunged {

1253 anyIDs[i] = m.ID

1254 }

1255 qmr := bstore.QueryTx[store.Recipient](tx)

1256 qmr.FilterEqual("MessageID", anyIDs...)

1257 _, err = qmr.Delete()

1258 xcheckf(ctx, err, "removing message recipients")

1259

1260 // Adjust mailbox counts, gather UIDs for broadcasted change, prepare for untraining.

1261 var totalSize int64

1262 uids := make([]store.UID, len(expunged))

1263 for i, m := range expunged {

1264 m.Expunged = false // Gather returns updated values.

1265 mb.Sub(m.MailboxCounts())

1266 totalSize += m.Size

1267 uids[i] = m.UID

1268

1269 expunged[i].Junk = false

1270 expunged[i].Notjunk = false

1271 }

1272

1273 err = tx.Update(&mb)

1274 xcheckf(ctx, err, "updating mailbox for counts")

1275

1276 err = acc.AddMessageSize(log, tx, -totalSize)

1277 xcheckf(ctx, err, "updating disk usage")

1278

1279 err = acc.RetrainMessages(ctx, log, tx, expunged, true)

1280 xcheckf(ctx, err, "retraining expunged messages")

1281

1282 chremove := store.ChangeRemoveUIDs{MailboxID: mb.ID, UIDs: uids, ModSeq: modseq}

1283 changes = []store.Change{chremove, mb.ChangeCounts()}

1284 })

1285

1286 store.BroadcastChanges(acc, changes)

1287 })

1288

1289 for _, m := range expunged {

1290 p := acc.MessagePath(m.ID)

1291 err := os.Remove(p)

1292 log.Check(err, "removing message file after emptying mailbox", slog.String("path", p))

1293 }

1294}

1295

1296// MailboxRename renames a mailbox, possibly moving it to a new parent. The mailbox

1297// ID and its messages are unchanged.

1298func (Webmail) MailboxRename(ctx context.Context, mailboxID int64, newName string) {

1299 log := pkglog.WithContext(ctx)

1300 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1301 acc, err := store.OpenAccount(log, reqInfo.AccountName)

1302 xcheckf(ctx, err, "open account")

1303 defer func() {

1304 err := acc.Close()

1305 log.Check(err, "closing account")

1306 }()

1307

1308 // Renaming Inbox is special for IMAP. For IMAP we have to implement it per the

1309 // standard. We can just say no.

1310 newName, _, err = store.CheckMailboxName(newName, false)

1311 xcheckuserf(ctx, err, "checking new mailbox name")

1312

1313 acc.WithWLock(func() {

1314 var changes []store.Change

1315

1316 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1317 mbsrc := xmailboxID(ctx, tx, mailboxID)

1318 var err error

1319 var isInbox, notExists, alreadyExists bool

1320 changes, isInbox, notExists, alreadyExists, err = acc.MailboxRename(tx, mbsrc, newName)

1321 if isInbox || notExists || alreadyExists {

1322 xcheckuserf(ctx, err, "renaming mailbox")

1323 }

1324 xcheckf(ctx, err, "renaming mailbox")

1325 })

1326

1327 store.BroadcastChanges(acc, changes)

1328 })

1329}

1330

1331// CompleteRecipient returns autocomplete matches for a recipient, returning the

1332// matches, most recently used first, and whether this is the full list and further

1333// requests for longer prefixes aren't necessary.

1334func (Webmail) CompleteRecipient(ctx context.Context, search string) ([]string, bool) {

1335 log := pkglog.WithContext(ctx)

1336 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1337 acc, err := store.OpenAccount(log, reqInfo.AccountName)

1338 xcheckf(ctx, err, "open account")

1339 defer func() {

1340 err := acc.Close()

1341 log.Check(err, "closing account")

1342 }()

1343

1344 search = strings.ToLower(search)

1345

1346 var matches []string

1347 all := true

1348 acc.WithRLock(func() {

1349 xdbread(ctx, acc, func(tx *bstore.Tx) {

1350 type key struct {

1351 localpart string

1352 domain string

1353 }

1354 seen := map[key]bool{}

1355

1356 q := bstore.QueryTx[store.Recipient](tx)

1357 q.SortDesc("Sent")

1358 err := q.ForEach(func(r store.Recipient) error {

1359 k := key{r.Localpart, r.Domain}

1360 if seen[k] {

1361 return nil

1362 }

1363 // todo: we should have the address including name available in the database for searching. Will result in better matching, and also for the name.

1364 address := fmt.Sprintf("<%s@%s>", r.Localpart, r.Domain)

1365 if !strings.Contains(strings.ToLower(address), search) {

1366 return nil

1367 }

1368 if len(matches) >= 20 {

1369 all = false

1370 return bstore.StopForEach

1371 }

1372

1373 // Look in the message that was sent for a name along with the address.

1374 m := store.Message{ID: r.MessageID}

1375 err := tx.Get(&m)

1376 xcheckf(ctx, err, "get sent message")

1377 if !m.Expunged && m.ParsedBuf != nil {

1378 var part message.Part

1379 err := json.Unmarshal(m.ParsedBuf, &part)

1380 xcheckf(ctx, err, "parsing part")

1381

1382 dom, err := dns.ParseDomain(r.Domain)

1383 xcheckf(ctx, err, "parsing domain of recipient")

1384

1385 var found bool

1386 lp := r.Localpart

1387 checkAddrs := func(l []message.Address) {

1388 if found {

1389 return

1390 }

1391 for _, a := range l {

1392 if a.Name != "" && a.User == lp && strings.EqualFold(a.Host, dom.ASCII) {

1393 found = true

1394 address = addressString(a, false)

1395 return

1396 }

1397 }

1398 }

1399 if part.Envelope != nil {

1400 env := part.Envelope

1401 checkAddrs(env.To)

1402 checkAddrs(env.CC)

1403 checkAddrs(env.BCC)

1404 }

1405 }

1406

1407 matches = append(matches, address)

1408 seen[k] = true

1409 return nil

1410 })

1411 xcheckf(ctx, err, "listing recipients")

1412 })

1413 })

1414 return matches, all

1415}

1416

1417// addressString returns an address into a string as it could be used in a message header.

1418func addressString(a message.Address, smtputf8 bool) string {

1419 host := a.Host

1420 dom, err := dns.ParseDomain(a.Host)

1421 if err == nil {

1422 if smtputf8 && dom.Unicode != "" {

1423 host = dom.Unicode

1424 } else {

1425 host = dom.ASCII

1426 }

1427 }

1428 s := "<" + a.User + "@" + host + ">"

1429 if a.Name != "" {

1430 // todo: properly encoded/escaped name

1431 s = a.Name + " " + s

1432 }

1433 return s

1434}

1435

1436// MailboxSetSpecialUse sets the special use flags of a mailbox.

1437func (Webmail) MailboxSetSpecialUse(ctx context.Context, mb store.Mailbox) {

1438 log := pkglog.WithContext(ctx)

1439 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1440 acc, err := store.OpenAccount(log, reqInfo.AccountName)

1441 xcheckf(ctx, err, "open account")

1442 defer func() {

1443 err := acc.Close()

1444 log.Check(err, "closing account")

1445 }()

1446

1447 acc.WithWLock(func() {

1448 var changes []store.Change

1449

1450 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1451 xmb := xmailboxID(ctx, tx, mb.ID)

1452

1453 // We only allow a single mailbox for each flag (JMAP requirement). So for any flag

1454 // we set, we clear it for the mailbox(es) that had it, if any.

1455 clearPrevious := func(clear bool, specialUse string) {

1456 if !clear {

1457 return

1458 }

1459 var ombl []store.Mailbox

1460 q := bstore.QueryTx[store.Mailbox](tx)

1461 q.FilterNotEqual("ID", mb.ID)

1462 q.FilterEqual(specialUse, true)

1463 q.Gather(&ombl)

1464 _, err := q.UpdateField(specialUse, false)

1465 xcheckf(ctx, err, "updating previous special-use mailboxes")

1466

1467 for _, omb := range ombl {

1468 changes = append(changes, omb.ChangeSpecialUse())

1469 }

1470 }

1471 clearPrevious(mb.Archive, "Archive")

1472 clearPrevious(mb.Draft, "Draft")

1473 clearPrevious(mb.Junk, "Junk")

1474 clearPrevious(mb.Sent, "Sent")

1475 clearPrevious(mb.Trash, "Trash")

1476

1477 xmb.SpecialUse = mb.SpecialUse

1478 err = tx.Update(&xmb)

1479 xcheckf(ctx, err, "updating special-use flags for mailbox")

1480 changes = append(changes, xmb.ChangeSpecialUse())

1481 })

1482

1483 store.BroadcastChanges(acc, changes)

1484 })

1485}

1486

1487// ThreadCollapse saves the ThreadCollapse field for the messages and its

1488// children. The messageIDs are typically thread roots. But not all roots

1489// (without parent) of a thread need to have the same collapsed state.

1490func (Webmail) ThreadCollapse(ctx context.Context, messageIDs []int64, collapse bool) {

1491 log := pkglog.WithContext(ctx)

1492 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1493 acc, err := store.OpenAccount(log, reqInfo.AccountName)

1494 xcheckf(ctx, err, "open account")

1495 defer func() {

1496 err := acc.Close()

1497 log.Check(err, "closing account")

1498 }()

1499

1500 if len(messageIDs) == 0 {

1501 xcheckuserf(ctx, errors.New("no messages"), "setting collapse")

1502 }

1503

1504 acc.WithWLock(func() {

1505 changes := make([]store.Change, 0, len(messageIDs))

1506 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1507 // Gather ThreadIDs to list all potential messages, for a way to get all potential

1508 // (child) messages. Further refined in FilterFn.

1509 threadIDs := map[int64]struct{}{}

1510 msgIDs := map[int64]struct{}{}

1511 for _, id := range messageIDs {

1512 m := store.Message{ID: id}

1513 err := tx.Get(&m)

1514 if err == bstore.ErrAbsent {

1515 xcheckuserf(ctx, err, "get message")

1516 }

1517 xcheckf(ctx, err, "get message")

1518 threadIDs[m.ThreadID] = struct{}{}

1519 msgIDs[id] = struct{}{}

1520 }

1521

1522 var updated []store.Message

1523 q := bstore.QueryTx[store.Message](tx)

1524 q.FilterEqual("ThreadID", slicesAny(maps.Keys(threadIDs))...)

1525 q.FilterNotEqual("ThreadCollapsed", collapse)

1526 q.FilterFn(func(tm store.Message) bool {

1527 for _, id := range tm.ThreadParentIDs {

1528 if _, ok := msgIDs[id]; ok {

1529 return true

1530 }

1531 }

1532 _, ok := msgIDs[tm.ID]

1533 return ok

1534 })

1535 q.Gather(&updated)

1536 q.SortAsc("ID") // Consistent order for testing.

1537 _, err = q.UpdateFields(map[string]any{"ThreadCollapsed": collapse})

1538 xcheckf(ctx, err, "updating collapse in database")

1539

1540 for _, m := range updated {

1541 changes = append(changes, m.ChangeThread())

1542 }

1543 })

1544 store.BroadcastChanges(acc, changes)

1545 })

1546}

1547

1548// ThreadMute saves the ThreadMute field for the messages and their children.

1549// If messages are muted, they are also marked collapsed.

1550func (Webmail) ThreadMute(ctx context.Context, messageIDs []int64, mute bool) {

1551 log := pkglog.WithContext(ctx)

1552 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1553 acc, err := store.OpenAccount(log, reqInfo.AccountName)

1554 xcheckf(ctx, err, "open account")

1555 defer func() {

1556 err := acc.Close()

1557 log.Check(err, "closing account")

1558 }()

1559

1560 if len(messageIDs) == 0 {

1561 xcheckuserf(ctx, errors.New("no messages"), "setting mute")

1562 }

1563

1564 acc.WithWLock(func() {

1565 changes := make([]store.Change, 0, len(messageIDs))

1566 xdbwrite(ctx, acc, func(tx *bstore.Tx) {

1567 threadIDs := map[int64]struct{}{}

1568 msgIDs := map[int64]struct{}{}

1569 for _, id := range messageIDs {

1570 m := store.Message{ID: id}

1571 err := tx.Get(&m)

1572 if err == bstore.ErrAbsent {

1573 xcheckuserf(ctx, err, "get message")

1574 }

1575 xcheckf(ctx, err, "get message")

1576 threadIDs[m.ThreadID] = struct{}{}

1577 msgIDs[id] = struct{}{}

1578 }

1579

1580 var updated []store.Message

1581

1582 q := bstore.QueryTx[store.Message](tx)

1583 q.FilterEqual("ThreadID", slicesAny(maps.Keys(threadIDs))...)

1584 q.FilterFn(func(tm store.Message) bool {

1585 if tm.ThreadMuted == mute && (!mute || tm.ThreadCollapsed) {

1586 return false

1587 }

1588 for _, id := range tm.ThreadParentIDs {

1589 if _, ok := msgIDs[id]; ok {

1590 return true

1591 }

1592 }

1593 _, ok := msgIDs[tm.ID]

1594 return ok

1595 })

1596 q.Gather(&updated)

1597 fields := map[string]any{"ThreadMuted": mute}

1598 if mute {

1599 fields["ThreadCollapsed"] = true

1600 }

1601 _, err = q.UpdateFields(fields)

1602 xcheckf(ctx, err, "updating mute in database")

1603

1604 for _, m := range updated {

1605 changes = append(changes, m.ChangeThread())

1606 }

1607 })

1608 store.BroadcastChanges(acc, changes)

1609 })

1610}

1611

1612// SecurityResult indicates whether a security feature is supported.

1613type SecurityResult string

1614

1615const (

1616 SecurityResultError SecurityResult = "error"

1617 SecurityResultNo SecurityResult = "no"

1618 SecurityResultYes SecurityResult = "yes"

1619 // Unknown whether supported. Finding out may only be (reasonably) possible when

1620 // trying (e.g. SMTP STARTTLS). Once tried, the result may be cached for future

1621 // lookups.

1622 SecurityResultUnknown SecurityResult = "unknown"

1623)

1624

1625// RecipientSecurity is a quick analysis of the security properties of delivery to

1626// the recipient (domain).

1627type RecipientSecurity struct {

1628 // Whether recipient domain supports (opportunistic) STARTTLS, as seen during most

1629 // recent delivery attempt. Will be "unknown" if no delivery to the domain has been

1630 // attempted yet.

1631 STARTTLS SecurityResult

1632

1633 // Whether we have a stored enforced MTA-STS policy, or domain has MTA-STS DNS

1634 // record.

1635 MTASTS SecurityResult

1636

1637 // Whether MX lookup response was DNSSEC-signed.

1638 DNSSEC SecurityResult

1639

1640 // Whether first delivery destination has DANE records.

1641 DANE SecurityResult

1642

1643 // Whether recipient domain is known to implement the REQUIRETLS SMTP extension.

1644 // Will be "unknown" if no delivery to the domain has been attempted yet.

1645 RequireTLS SecurityResult

1646}

1647

1648// RecipientSecurity looks up security properties of the address in the

1649// single-address message addressee (as it appears in a To/Cc/Bcc/etc header).

1650func (Webmail) RecipientSecurity(ctx context.Context, messageAddressee string) (RecipientSecurity, error) {

1651 resolver := dns.StrictResolver{Pkg: "webmail", Log: pkglog.WithContext(ctx).Logger}

1652 return recipientSecurity(ctx, resolver, messageAddressee)

1653}

1654

1655// logPanic can be called with a defer from a goroutine to prevent the entire program from being shutdown in case of a panic.

1656func logPanic(ctx context.Context) {

1657 x := recover()

1658 if x == nil {

1659 return

1660 }

1661 log := pkglog.WithContext(ctx)

1662 log.Error("recover from panic", slog.Any("panic", x))

1663 debug.PrintStack()

1664 metrics.PanicInc(metrics.Webmail)

1665}

1666

1667// separate function for testing with mocked resolver.

1668func recipientSecurity(ctx context.Context, resolver dns.Resolver, messageAddressee string) (RecipientSecurity, error) {

1669 log := pkglog.WithContext(ctx)

1670

1671 rs := RecipientSecurity{

1672 SecurityResultUnknown,

1673 SecurityResultUnknown,

1674 SecurityResultUnknown,

1675 SecurityResultUnknown,

1676 SecurityResultUnknown,

1677 }

1678

1679 msgAddr, err := mail.ParseAddress(messageAddressee)

1680 if err != nil {

1681 return rs, fmt.Errorf("parsing message addressee: %v", err)

1682 }

1683

1684 addr, err := smtp.ParseAddress(msgAddr.Address)

1685 if err != nil {

1686 return rs, fmt.Errorf("parsing address: %v", err)

1687 }

1688

1689 var wg sync.WaitGroup

1690

1691 // MTA-STS.

1692 wg.Add(1)

1693 go func() {

1694 defer logPanic(ctx)

1695 defer wg.Done()

1696

1697 policy, _, _, err := mtastsdb.Get(ctx, log.Logger, resolver, addr.Domain)

1698 if policy != nil && policy.Mode == mtasts.ModeEnforce {

1699 rs.MTASTS = SecurityResultYes

1700 } else if err == nil {

1701 rs.MTASTS = SecurityResultNo

1702 } else {

1703 rs.MTASTS = SecurityResultError

1704 }

1705 }()

1706

1707 // DNSSEC and DANE.

1708 wg.Add(1)

1709 go func() {

1710 defer logPanic(ctx)

1711 defer wg.Done()

1712

1713 _, origNextHopAuthentic, expandedNextHopAuthentic, _, hosts, _, err := smtpclient.GatherDestinations(ctx, log.Logger, resolver, dns.IPDomain{Domain: addr.Domain})

1714 if err != nil {

1715 rs.DNSSEC = SecurityResultError

1716 return

1717 }

1718 if origNextHopAuthentic && expandedNextHopAuthentic {

1719 rs.DNSSEC = SecurityResultYes

1720 } else {

1721 rs.DNSSEC = SecurityResultNo

1722 }

1723

1724 if !origNextHopAuthentic {

1725 rs.DANE = SecurityResultNo

1726 return

1727 }

1728

1729 // We're only looking at the first host to deliver to (typically first mx destination).

1730 if len(hosts) == 0 || hosts[0].Domain.IsZero() {

1731 return // Should not happen.

1732 }

1733 host := hosts[0]

1734

1735 // Resolve the IPs. Required for DANE to prevent bad DNS servers from causing an

1736 // error result instead of no-DANE result.

1737 authentic, expandedAuthentic, expandedHost, _, _, err := smtpclient.GatherIPs(ctx, log.Logger, resolver, host, map[string][]net.IP{})

1738 if err != nil {

1739 rs.DANE = SecurityResultError

1740 return

1741 }

1742 if !authentic {

1743 rs.DANE = SecurityResultNo

1744 return

1745 }

1746

1747 daneRequired, _, _, err := smtpclient.GatherTLSA(ctx, log.Logger, resolver, host.Domain, expandedAuthentic, expandedHost)

1748 if err != nil {

1749 rs.DANE = SecurityResultError

1750 return

1751 } else if daneRequired {

1752 rs.DANE = SecurityResultYes

1753 } else {

1754 rs.DANE = SecurityResultNo

1755 }

1756 }()

1757

1758 // STARTTLS and RequireTLS

1759 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

1760 acc, err := store.OpenAccount(log, reqInfo.AccountName)

1761 xcheckf(ctx, err, "open account")

1762 defer func() {

1763 if acc != nil {

1764 err := acc.Close()

1765 log.Check(err, "closing account")

1766 }

1767 }()

1768

1769 err = acc.DB.Read(ctx, func(tx *bstore.Tx) error {

1770 q := bstore.QueryTx[store.RecipientDomainTLS](tx)

1771 q.FilterNonzero(store.RecipientDomainTLS{Domain: addr.Domain.Name()})

1772 rd, err := q.Get()

1773 if err == bstore.ErrAbsent {

1774 return nil

1775 } else if err != nil {

1776 rs.STARTTLS = SecurityResultError

1777 rs.RequireTLS = SecurityResultError

1778 log.Errorx("looking up recipient domain", err, slog.Any("domain", addr.Domain))

1779 return nil

1780 }

1781 if rd.STARTTLS {

1782 rs.STARTTLS = SecurityResultYes

1783 } else {

1784 rs.STARTTLS = SecurityResultNo

1785 }

1786 if rd.RequireTLS {

1787 rs.RequireTLS = SecurityResultYes

1788 } else {

1789 rs.RequireTLS = SecurityResultNo

1790 }

1791 return nil

1792 })

1793 xcheckf(ctx, err, "lookup recipient domain")

1794

1795 // Close account as soon as possible, not after waiting for MTA-STS/DNSSEC/DANE

1796 // checks to complete, which can take a while.

1797 err = acc.Close()

1798 log.Check(err, "closing account")

1799 acc = nil

1800

1801 wg.Wait()

1802

1803 return rs, nil

1804}

1805

1806// DecodeMIMEWords decodes Q/B-encoded words for a mime headers into UTF-8 text.

1807func (Webmail) DecodeMIMEWords(ctx context.Context, text string) string {

1808 s, err := wordDecoder.DecodeHeader(text)

1809 xcheckuserf(ctx, err, "decoding mime q/b-word encoded header")

1810 return s

1811}

1812

1813func slicesAny[T any](l []T) []any {

1814 r := make([]any, len(l))

1815 for i, v := range l {

1816 r[i] = v

1817 }

1818 return r

1819}

1820

1821// SSETypes exists to ensure the generated API contains the types, for use in SSE events.

1822func (Webmail) SSETypes() (start EventStart, viewErr EventViewErr, viewReset EventViewReset, viewMsgs EventViewMsgs, viewChanges EventViewChanges, msgAdd ChangeMsgAdd, msgRemove ChangeMsgRemove, msgFlags ChangeMsgFlags, msgThread ChangeMsgThread, mailboxRemove ChangeMailboxRemove, mailboxAdd ChangeMailboxAdd, mailboxRename ChangeMailboxRename, mailboxCounts ChangeMailboxCounts, mailboxSpecialUse ChangeMailboxSpecialUse, mailboxKeywords ChangeMailboxKeywords, flags store.Flags) {

1823 return

1824}

1825