1// Package webapisrv implements the server-side of the webapi.

2package webapisrv

4// In a separate package from webapi, so webapi.Client can be used and imported

5// without including all mox internals. Documentation for the functions is in

6// ../webapi/client.go.

8import (

9 "bytes"

10 "context"

11 cryptorand "crypto/rand"

12 "encoding/base64"

13 "encoding/json"

14 "errors"

15 "fmt"

16 htmltemplate "html/template"

17 "io"

18 "log/slog"

19 "mime"

20 "mime/multipart"

21 "net/http"

22 "net/textproto"

23 "reflect"

24 "runtime/debug"

25 "slices"

26 "strings"

27 "time"

29 "github.com/prometheus/client_golang/prometheus"

30 "github.com/prometheus/client_golang/prometheus/promauto"

32 "github.com/mjl-/bstore"

34 "github.com/mjl-/mox/dkim"

35 "github.com/mjl-/mox/dns"

36 "github.com/mjl-/mox/message"

37 "github.com/mjl-/mox/metrics"

38 "github.com/mjl-/mox/mlog"

39 "github.com/mjl-/mox/mox-"

40 "github.com/mjl-/mox/moxio"

41 "github.com/mjl-/mox/moxvar"

42 "github.com/mjl-/mox/queue"

43 "github.com/mjl-/mox/smtp"

44 "github.com/mjl-/mox/store"

45 "github.com/mjl-/mox/webapi"

46 "github.com/mjl-/mox/webauth"

47 "github.com/mjl-/mox/webops"

48)

50var pkglog = mlog.New("webapi", nil)

52var (

53 // Similar between ../webmail/webmail.go:/metricSubmission and ../smtpserver/server.go:/metricSubmission and ../webapisrv/server.go:/metricSubmission

54 metricSubmission = promauto.NewCounterVec(

55 prometheus.CounterOpts{

56 Name: "mox_webapi_submission_total",

57 Help: "Webapi message submission results, known values (those ending with error are server errors): ok, badfrom, messagelimiterror, recipientlimiterror, queueerror, storesenterror.",

58 },

59 []string{

60 "result",

61 },

62 )

63 metricServerErrors = promauto.NewCounterVec(

64 prometheus.CounterOpts{

65 Name: "mox_webapi_errors_total",

66 Help: "Webapi server errors, known values: dkimsign, submit.",

67 },

68 []string{

69 "error",

70 },

71 )

72 metricResults = promauto.NewCounterVec(

73 prometheus.CounterOpts{

74 Name: "mox_webapi_results_total",

75 Help: "HTTP webapi results by method and result.",

76 },

77 []string{"method", "result"}, // result: "badauth", "ok", or error code

78 )

79 metricDuration = promauto.NewHistogramVec(

80 prometheus.HistogramOpts{

81 Name: "mox_webapi_duration_seconds",

82 Help: "HTTP webhook call duration.",

83 Buckets: []float64{0.01, 0.05, 0.1, 0.5, 1, 5, 10, 20, 30},

84 },

85 []string{"method"},

86 )

87)

89// We pass the request to the handler so the TLS info can be used for

90// the Received header in submitted messages. Most API calls need just the

91// account name.

92type ctxKey string

94var requestInfoCtxKey ctxKey = "requestInfo"

96type requestInfo struct {

97 Log mlog.Log

98 LoginAddress string

99 Account *store.Account

100 Response http.ResponseWriter // For setting headers for non-JSON responses.

101 Request *http.Request // For Proto and TLS connection state during message submit.

102}

103

104// todo: show a curl invocation on the method pages

105

106var docsMethodTemplate = htmltemplate.Must(htmltemplate.New("method").Parse(`<!doctype html>

107 <head>

108 <meta charset="utf-8" />

109 <meta name="robots" content="noindex,nofollow" />

110 <title>Method {{ .Method }} - WebAPI - Mox</title>

111 <style>

112body, html { padding: 1em; font-size: 16px; }

113* { font-size: inherit; font-family: ubuntu, lato, sans-serif; margin: 0; padding: 0; box-sizing: border-box; }

114h1, h2, h3, h4 { margin-bottom: 1ex; }

115h1 { font-size: 1.2rem; }

116h2 { font-size: 1.1rem; }

117h3, h4 { font-size: 1rem; }

118ul { padding-left: 1rem; }

119p { margin-bottom: 1em; max-width: 50em; }

120[title] { text-decoration: underline; text-decoration-style: dotted; }

121fieldset { border: 0; }

122textarea { width: 100%; max-width: 50em; }

123 </style>

124 </head>

125 <body>

126 <h1><a href="../">WebAPI</a> - Method {{ .Method }}</h1>

127 <form id="webapicall" method="POST">

128 <fieldset id="webapifieldset">

129 <h2>Request JSON</h2>

130 <div><textarea id="webapirequest" name="request" required rows="20">{{ .Request }}</textarea></div>

131 <br/>

132 <div>

133 <button type="reset">Reset</button>

134 <button type="submit">Call</button>

135 </div>

136 <br/>

137{{ if .ReturnsBytes }}

138 <p>Method has a non-JSON response.</p>

139{{ else }}

140 <h2>Response JSON</h2>

141 <div><textarea id="webapiresponse" rows="20">{{ .Response }}</textarea></div>

142{{ end }}

143 </fieldset>

144 </form>

145 <script>

146window.addEventListener('load', () => {

147 window.webapicall.addEventListener('submit', async (e) => {

148 const stop = () => {

149 e.stopPropagation()

150 e.preventDefault()

151 }

152

153 let req

154 try {

155 req = JSON.parse(window.webapirequest.value)

156 } catch (err) {

157 window.alert('Error parsing request: ' + err.message)

158 stop()

159 return

160 }

161 if (!req) {

162 window.alert('Empty request')

163 stop()

164 return

165 }

166

167 if ({{ .ReturnsBytes }}) {

168 // Just POST to this URL.

169 return

170 }

171

172 stop()

173 // Do call ourselves, get response and put it in the response textarea.

174 window.webapifieldset.disabled = true

175 let data = new window.FormData()

176 data.append("request", window.webapirequest.value)

177 try {

178 const response = await fetch("{{ .Method }}", {body: data, method: "POST"})

179 const text = await response.text()

180 try {

181 window.webapiresponse.value = JSON.stringify(JSON.parse(text), undefined, '\t')

182 } catch (err) {

183 window.webapiresponse.value = text

184 }

185 } catch (err) {

186 window.alert('Error: ' + err.message)

187 } finally {

188 window.webapifieldset.disabled = false

189 }

190 })

191})

192 </script>

193 </body>

194</html>

195`))

196

197var docsIndex []byte

198

199func init() {

200 var methods []string

201 mt := reflect.TypeFor[webapi.Methods]()

202 n := mt.NumMethod()

203 for i := 0; i < n; i++ {

204 methods = append(methods, mt.Method(i).Name)

205 }

206 docsIndexTmpl := htmltemplate.Must(htmltemplate.New("index").Parse(`<!doctype html>

207<html>

208 <head>

209 <meta charset="utf-8" />

210 <meta name="robots" content="noindex,nofollow" />

211 <title>Webapi - Mox</title>

212 <style>

213body, html { padding: 1em; font-size: 16px; }

214* { font-size: inherit; font-family: ubuntu, lato, sans-serif; margin: 0; padding: 0; box-sizing: border-box; }

215h1, h2, h3, h4 { margin-bottom: 1ex; }

216h1 { font-size: 1.2rem; }

217h2 { font-size: 1.1rem; }

218h3, h4 { font-size: 1rem; }

219ul { padding-left: 1rem; }

220p { margin-bottom: 1em; max-width: 50em; }

221[title] { text-decoration: underline; text-decoration-style: dotted; }

222fieldset { border: 0; }

223 </style>

224 </head>

225 <body>

226 <h1>Webapi and webhooks</h1>

227 <p>The mox webapi is a simple HTTP/JSON-based API for sending messages and processing incoming messages.</p>

228 <p>Configure webhooks in mox to receive notifications about outgoing delivery event, and/or incoming deliveries of messages.</p>

229 <p>Documentation and examples:</p>

230 <p><a href="{{ .WebapiDocsURL }}">{{ .WebapiDocsURL }}</a></p>

231 <h2>Methods</h2>

232 <p>The methods below are available in this version of mox. Follow a link for an example request/response JSON, and a button to make an API call.</p>

233 <ul>

234{{ range $i, $method := .Methods }}

235 <li><a href="{{ $method }}">{{ $method }}</a></li>

236{{ end }}

237 </ul>

238 </body>

239</html>

240`))

241 webapiDocsURL := "https://pkg.go.dev/github.com/mjl-/mox@" + moxvar.VersionBare + "/webapi/"

242 webhookDocsURL := "https://pkg.go.dev/github.com/mjl-/mox@" + moxvar.VersionBare + "/webhook/"

243 indexArgs := struct {

244 WebapiDocsURL string

245 WebhookDocsURL string

246 Methods []string

247 }{webapiDocsURL, webhookDocsURL, methods}

248 var b bytes.Buffer

249 err := docsIndexTmpl.Execute(&b, indexArgs)

250 if err != nil {

251 panic("executing api docs index template: " + err.Error())

252 }

253 docsIndex = b.Bytes()

254

255 mox.NewWebapiHandler = func(maxMsgSize int64, basePath string, isForwarded bool) http.Handler {

256 return NewServer(maxMsgSize, basePath, isForwarded)

257 }

258}

259

260// NewServer returns a new http.Handler for a webapi server.

261func NewServer(maxMsgSize int64, path string, isForwarded bool) http.Handler {

262 return server{maxMsgSize, path, isForwarded}

263}

264

265// server implements the webapi methods.

266type server struct {

267 maxMsgSize int64 // Of outgoing messages.

268 path string // Path webapi is configured under, typically /webapi/, with methods at /webapi/v0/<method>.

269 isForwarded bool // Whether incoming requests are reverse-proxied. Used for getting remote IPs for rate limiting.

270}

271

272var _ webapi.Methods = server{}

273

274// ServeHTTP implements http.Handler.

275func (s server) ServeHTTP(w http.ResponseWriter, r *http.Request) {

276 log := pkglog.WithContext(r.Context()) // Take cid from webserver.

277

278 // Send requests to /webapi/ to /webapi/v0/.

279 if r.URL.Path == "/" {

280 if r.Method != "GET" {

281 http.Error(w, "405 - method not allow", http.StatusMethodNotAllowed)

282 return

283 }

284 http.Redirect(w, r, s.path+"v0/", http.StatusSeeOther)

285 return

286 }

287 // Serve short introduction and list to methods at /webapi/v0/.

288 if r.URL.Path == "/v0/" {

289 w.Header().Set("Content-Type", "text/html; charset=utf-8")

290 w.Write(docsIndex)

291 return

292 }

293

294 // Anything else must be a method endpoint.

295 if !strings.HasPrefix(r.URL.Path, "/v0/") {

296 http.NotFound(w, r)

297 return

298 }

299 fn := r.URL.Path[len("/v0/"):]

300 log = log.With(slog.String("method", fn))

301 rfn := reflect.ValueOf(s).MethodByName(fn)

302 var zero reflect.Value

303 if rfn == zero || rfn.Type().NumIn() != 2 || rfn.Type().NumOut() != 2 {

304 log.Debug("unknown webapi method")

305 http.NotFound(w, r)

306 return

307 }

308

309 // GET on method returns an example request JSON, a button to call the method,

310 // which either fills a textarea with the response (in case of JSON) or posts to

311 // the URL letting the browser handle the response (e.g. raw message or part).

312 if r.Method == "GET" {

313 formatJSON := func(v any) (string, error) {

314 var b bytes.Buffer

315 enc := json.NewEncoder(&b)

316 enc.SetIndent("", "\t")

317 enc.SetEscapeHTML(false)

318 err := enc.Encode(v)

319 return string(b.String()), err

320 }

321

322 req, err := formatJSON(mox.FillExample(nil, reflect.New(rfn.Type().In(1))).Interface())

323 if err != nil {

324 log.Errorx("formatting request as json", err)

325 http.Error(w, "500 - internal server error - marshal request: "+err.Error(), http.StatusInternalServerError)

326 return

327 }

328 // todo: could check for io.ReadCloser, but we don't return other interfaces than that one.

329 returnsBytes := rfn.Type().Out(0).Kind() == reflect.Interface

330 var resp string

331 if !returnsBytes {

332 resp, err = formatJSON(mox.FillExample(nil, reflect.New(rfn.Type().Out(0))).Interface())

333 if err != nil {

334 log.Errorx("formatting response as json", err)

335 http.Error(w, "500 - internal server error - marshal response: "+err.Error(), http.StatusInternalServerError)

336 return

337 }

338 }

339 args := struct {

340 Method string

341 Request string

342 Response string

343 ReturnsBytes bool

344 }{fn, req, resp, returnsBytes}

345 w.Header().Set("Content-Type", "text/html; charset=utf-8")

346 err = docsMethodTemplate.Execute(w, args)

347 log.Check(err, "executing webapi method template")

348 return

349 } else if r.Method != "POST" {

350 http.Error(w, "405 - method not allowed - use get or post", http.StatusMethodNotAllowed)

351 return

352 }

353

354 // Account is available during call, but we close it before we start writing a

355 // response, to prevent slow readers from holding a reference for a long time.

356 var acc *store.Account

357 closeAccount := func() {

358 if acc != nil {

359 err := acc.Close()

360 log.Check(err, "closing account")

361 acc = nil

362 }

363 }

364 defer closeAccount()

365

366 email, password, aok := r.BasicAuth()

367 if !aok {

368 metricResults.WithLabelValues(fn, "badauth").Inc()

369 log.Debug("missing http basic authentication credentials")

370 w.Header().Set("WWW-Authenticate", "Basic realm=webapi")

371 http.Error(w, "401 - unauthorized - use http basic auth with email address as username", http.StatusUnauthorized)

372 return

373 }

374 log = log.With(slog.String("username", email))

375

376 t0 := time.Now()

377

378 // If remote IP/network resulted in too many authentication failures, refuse to serve.

379 remoteIP := webauth.RemoteIP(log, s.isForwarded, r)

380 if remoteIP == nil {

381 metricResults.WithLabelValues(fn, "internal").Inc()

382 log.Debug("cannot find remote ip for rate limiter")

383 http.Error(w, "500 - internal server error - cannot find remote ip", http.StatusInternalServerError)

384 return

385 }

386 if !mox.LimiterFailedAuth.CanAdd(remoteIP, t0, 1) {

387 metrics.AuthenticationRatelimitedInc("webapi")

388 log.Debug("refusing connection due to many auth failures", slog.Any("remoteip", remoteIP))

389 http.Error(w, "429 - too many auth attempts", http.StatusTooManyRequests)

390 return

391 }

392

393 writeError := func(err webapi.Error) {

394 closeAccount()

395 metricResults.WithLabelValues(fn, err.Code).Inc()

396

397 if err.Code == "server" {

398 log.Errorx("webapi call result", err, slog.String("resultcode", err.Code))

399 } else {

400 log.Infox("webapi call result", err, slog.String("resultcode", err.Code))

401 }

402

403 w.Header().Set("Content-Type", "application/json; charset=utf-8")

404 w.WriteHeader(http.StatusBadRequest)

405 enc := json.NewEncoder(w)

406 enc.SetEscapeHTML(false)

407 werr := enc.Encode(err)

408 if werr != nil && !moxio.IsClosed(werr) {

409 log.Infox("writing error response", werr)

410 }

411 }

412

413 // Called for all successful JSON responses, not non-JSON responses.

414 writeResponse := func(resp any) {

415 closeAccount()

416 metricResults.WithLabelValues(fn, "ok").Inc()

417 log.Debug("webapi call result", slog.String("resultcode", "ok"))

418 w.Header().Set("Content-Type", "application/json; charset=utf-8")

419 enc := json.NewEncoder(w)

420 enc.SetEscapeHTML(false)

421 werr := enc.Encode(resp)

422 if werr != nil && !moxio.IsClosed(werr) {

423 log.Infox("writing error response", werr)

424 }

425 }

426

427 authResult := "error"

428 defer func() {

429 metricDuration.WithLabelValues(fn).Observe(float64(time.Since(t0)) / float64(time.Second))

430 metrics.AuthenticationInc("webapi", "httpbasic", authResult)

431 }()

432

433 var err error

434 acc, err = store.OpenEmailAuth(log, email, password)

435 if err != nil {

436 mox.LimiterFailedAuth.Add(remoteIP, t0, 1)

437 if errors.Is(err, mox.ErrDomainNotFound) || errors.Is(err, mox.ErrAddressNotFound) || errors.Is(err, store.ErrUnknownCredentials) {

438 log.Debug("bad http basic authentication credentials")

439 metricResults.WithLabelValues(fn, "badauth").Inc()

440 authResult = "badcreds"

441 w.Header().Set("WWW-Authenticate", "Basic realm=webapi")

442 http.Error(w, "401 - unauthorized - use http basic auth with email address as username", http.StatusUnauthorized)

443 return

444 }

445 writeError(webapi.Error{Code: "server", Message: "error verifying credentials"})

446 return

447 }

448 authResult = "ok"

449 mox.LimiterFailedAuth.Reset(remoteIP, t0)

450

451 ct := r.Header.Get("Content-Type")

452 ct, _, err = mime.ParseMediaType(ct)

453 if err != nil {

454 writeError(webapi.Error{Code: "protocol", Message: "unknown content-type " + r.Header.Get("Content-Type")})

455 return

456 }

457 if ct == "multipart/form-data" {

458 err = r.ParseMultipartForm(200 * 1024)

459 } else {

460 err = r.ParseForm()

461 }

462 if err != nil {

463 writeError(webapi.Error{Code: "protocol", Message: "parsing form: " + err.Error()})

464 return

465 }

466

467 reqstr := r.PostFormValue("request")

468 if reqstr == "" {

469 writeError(webapi.Error{Code: "protocol", Message: "missing/empty request"})

470 return

471 }

472

473 defer func() {

474 x := recover()

475 if x == nil {

476 return

477 }

478 if err, eok := x.(webapi.Error); eok {

479 writeError(err)

480 return

481 }

482 log.Error("unhandled panic in webapi call", slog.Any("x", x), slog.String("resultcode", "server"))

483 metrics.PanicInc(metrics.Webapi)

484 debug.PrintStack()

485 writeError(webapi.Error{Code: "server", Message: "unhandled error"})

486 }()

487 req := reflect.New(rfn.Type().In(1))

488 dec := json.NewDecoder(strings.NewReader(reqstr))

489 dec.DisallowUnknownFields()

490 if err := dec.Decode(req.Interface()); err != nil {

491 writeError(webapi.Error{Code: "protocol", Message: fmt.Sprintf("parsing request: %s", err)})

492 return

493 }

494

495 reqInfo := requestInfo{log, email, acc, w, r}

496 nctx := context.WithValue(r.Context(), requestInfoCtxKey, reqInfo)

497 resp := rfn.Call([]reflect.Value{reflect.ValueOf(nctx), req.Elem()})

498 if !resp[1].IsZero() {

499 var e webapi.Error

500 err := resp[1].Interface().(error)

501 if x, eok := err.(webapi.Error); eok {

502 e = x

503 } else {

504 e = webapi.Error{Code: "error", Message: err.Error()}

505 }

506 writeError(e)

507 return

508 }

509 rc, ok := resp[0].Interface().(io.ReadCloser)

510 if !ok {

511 rv, _ := mox.FillNil(resp[0])

512 writeResponse(rv.Interface())

513 return

514 }

515 closeAccount()

516 log.Debug("webapi call result", slog.String("resultcode", "ok"))

517 metricResults.WithLabelValues(fn, "ok").Inc()

518 defer rc.Close()

519 if _, err := io.Copy(w, rc); err != nil && !moxio.IsClosed(err) {

520 log.Errorx("writing response to client", err)

521 }

522}

523

524func xcheckf(err error, format string, args ...any) {

525 if err != nil {

526 msg := fmt.Sprintf(format, args...)

527 panic(webapi.Error{Code: "server", Message: fmt.Sprintf("%s: %s", msg, err)})

528 }

529}

530

531func xcheckuserf(err error, format string, args ...any) {

532 if err != nil {

533 msg := fmt.Sprintf(format, args...)

534 panic(webapi.Error{Code: "user", Message: fmt.Sprintf("%s: %s", msg, err)})

535 }

536}

537

538func xdbwrite(ctx context.Context, acc *store.Account, fn func(tx *bstore.Tx)) {

539 err := acc.DB.Write(ctx, func(tx *bstore.Tx) error {

540 fn(tx)

541 return nil

542 })

543 xcheckf(err, "transaction")

544}

545

546func xdbread(ctx context.Context, acc *store.Account, fn func(tx *bstore.Tx)) {

547 err := acc.DB.Read(ctx, func(tx *bstore.Tx) error {

548 fn(tx)

549 return nil

550 })

551 xcheckf(err, "transaction")

552}

553

554func xcheckcontrol(s string) {

555 for _, c := range s {

556 if c < 0x20 {

557 xcheckuserf(errors.New("control characters not allowed"), "checking header values")

558 }

559 }

560}

561

562func xparseAddress(addr string) smtp.Address {

563 a, err := smtp.ParseAddress(addr)

564 if err != nil {

565 panic(webapi.Error{Code: "badAddress", Message: fmt.Sprintf("parsing address %q: %s", addr, err)})

566 }

567 return a

568}

569

570func xparseAddresses(l []webapi.NameAddress) ([]message.NameAddress, []smtp.Path) {

571 r := make([]message.NameAddress, len(l))

572 paths := make([]smtp.Path, len(l))

573 for i, a := range l {

574 xcheckcontrol(a.Name)

575 addr := xparseAddress(a.Address)

576 r[i] = message.NameAddress{DisplayName: a.Name, Address: addr}

577 paths[i] = addr.Path()

578 }

579 return r, paths

580}

581

582func xrandomID(n int) string {

583 return base64.RawURLEncoding.EncodeToString(xrandom(n))

584}

585

586func xrandom(n int) []byte {

587 buf := make([]byte, n)

588 x, err := cryptorand.Read(buf)

589 if err != nil {

590 panic("read random")

591 } else if x != n {

592 panic("short random read")

593 }

594 return buf

595}

596

597func (s server) Send(ctx context.Context, req webapi.SendRequest) (resp webapi.SendResult, err error) {

598 // Similar between ../smtpserver/server.go:/submit\( and ../webmail/api.go:/MessageSubmit\( and ../webapisrv/server.go:/Send\(

599

600 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

601 log := reqInfo.Log

602 acc := reqInfo.Account

603

604 m := req.Message

605

606 accConf, _ := acc.Conf()

607

608 if m.Text == "" && m.HTML == "" {

609 return resp, webapi.Error{Code: "missingBody", Message: "at least text or html body required"}

610 }

611

612 if len(m.From) == 0 {

613 m.From = []webapi.NameAddress{{Name: accConf.FullName, Address: reqInfo.LoginAddress}}

614 } else if len(m.From) > 1 {

615 return resp, webapi.Error{Code: "multipleFrom", Message: "multiple from-addresses not allowed"}

616 }

617 froms, fromPaths := xparseAddresses(m.From)

618 from, fromPath := froms[0], fromPaths[0]

619 to, toPaths := xparseAddresses(m.To)

620 cc, ccPaths := xparseAddresses(m.CC)

621 bcc, bccPaths := xparseAddresses(m.BCC)

622

623 recipients := append(append(toPaths, ccPaths...), bccPaths...)

624 addresses := append(append(m.To, m.CC...), m.BCC...)

625

626 // Check if from address is allowed for account.

627 if !mox.AllowMsgFrom(acc.Name, from.Address) {

628 metricSubmission.WithLabelValues("badfrom").Inc()

629 return resp, webapi.Error{Code: "badFrom", Message: "from-address not configured for account"}

630 }

631

632 if len(recipients) == 0 {

633 return resp, webapi.Error{Code: "noRecipients", Message: "no recipients"}

634 }

635

636 // Check outgoing message rate limit.

637 xdbread(ctx, acc, func(tx *bstore.Tx) {

638 msglimit, rcptlimit, err := acc.SendLimitReached(tx, recipients)

639 if msglimit >= 0 {

640 metricSubmission.WithLabelValues("messagelimiterror").Inc()

641 panic(webapi.Error{Code: "messageLimitReached", Message: "outgoing message rate limit reached"})

642 } else if rcptlimit >= 0 {

643 metricSubmission.WithLabelValues("recipientlimiterror").Inc()

644 panic(webapi.Error{Code: "recipientLimitReached", Message: "outgoing new recipient rate limit reached"})

645 }

646 xcheckf(err, "checking send limit")

647 })

648

649 // If we have a non-ascii localpart, we will be sending with smtputf8. We'll go

650 // full utf-8 then.

651 intl := func(l []smtp.Path) bool {

652 for _, p := range l {

653 if p.Localpart.IsInternational() {

654 return true

655 }

656 }

657 return false

658 }

659 smtputf8 := intl([]smtp.Path{fromPath}) || intl(toPaths) || intl(ccPaths) || intl(bccPaths)

660

661 replyTos, replyToPaths := xparseAddresses(m.ReplyTo)

662 for _, rt := range replyToPaths {

663 if rt.Localpart.IsInternational() {

664 smtputf8 = true

665 }

666 }

667

668 // Create file to compose message into.

669 dataFile, err := store.CreateMessageTemp(log, "webapi-submit")

670 xcheckf(err, "creating temporary file for message")

671 defer store.CloseRemoveTempFile(log, dataFile, "message to submit")

672

673 // If writing to the message file fails, we abort immediately.

674 xc := message.NewComposer(dataFile, s.maxMsgSize, smtputf8)

675 defer func() {

676 x := recover()

677 if x == nil {

678 return

679 }

680 if err, ok := x.(error); ok && errors.Is(err, message.ErrMessageSize) {

681 panic(webapi.Error{Code: "messageTooLarge", Message: "message too large"})

682 } else if ok && errors.Is(err, message.ErrCompose) {

683 xcheckf(err, "making message")

684 }

685 panic(x)

686 }()

687

688 // Each queued message gets a Received header.

689 // We cannot use VIA, because there is no registered method. We would like to use

690 // it to add the ascii domain name in case of smtputf8 and IDNA host name.

691 // We don't add the IP address of the submitter. Exposing likely not desirable.

692 recvFrom := message.HeaderCommentDomain(mox.Conf.Static.HostnameDomain, smtputf8)

693 recvBy := mox.Conf.Static.HostnameDomain.XName(smtputf8)

694 recvID := mox.ReceivedID(mox.CidFromCtx(ctx))

695 recvHdrFor := func(rcptTo string) string {

696 recvHdr := &message.HeaderWriter{}

697 // For additional Received-header clauses, see:

698 // https://www.iana.org/assignments/mail-parameters/mail-parameters.xhtml#table-mail-parameters-8

699 // Note: we don't have "via" or "with", there is no registered for webmail.

700 recvHdr.Add(" ", "Received:", "from", recvFrom, "by", recvBy, "id", recvID) // ../rfc/5321:3158

701 if reqInfo.Request.TLS != nil {

702 recvHdr.Add(" ", mox.TLSReceivedComment(log, *reqInfo.Request.TLS)...)

703 }

704 recvHdr.Add(" ", "for", "<"+rcptTo+">;", time.Now().Format(message.RFC5322Z))

705 return recvHdr.String()

706 }

707

708 // Outer message headers.

709 xc.HeaderAddrs("From", []message.NameAddress{from})

710 if len(replyTos) > 0 {

711 xc.HeaderAddrs("Reply-To", replyTos)

712 }

713 xc.HeaderAddrs("To", to)

714 xc.HeaderAddrs("Cc", cc)

715 // We prepend Bcc headers to the message when adding to the Sent mailbox.

716 if m.Subject != "" {

717 xcheckcontrol(m.Subject)

718 xc.Subject(m.Subject)

719 }

720

721 var date time.Time

722 if m.Date != nil {

723 date = *m.Date

724 } else {

725 date = time.Now()

726 }

727 xc.Header("Date", date.Format(message.RFC5322Z))

728

729 if m.MessageID == "" {

730 m.MessageID = fmt.Sprintf("<%s>", mox.MessageIDGen(smtputf8))

731 } else if !strings.HasPrefix(m.MessageID, "<") || !strings.HasSuffix(m.MessageID, ">") {

732 return resp, webapi.Error{Code: "malformedMessageID", Message: "missing <> in message-id"}

733 }

734 xcheckcontrol(m.MessageID)

735 xc.Header("Message-Id", m.MessageID)

736

737 if len(m.References) > 0 {

738 for _, ref := range m.References {

739 xcheckcontrol(ref)

740 // We don't check for <>'s. If caller just puts in what they got, we don't want to

741 // reject the message.

742 }

743 xc.Header("References", strings.Join(m.References, "\r\n\t"))

744 xc.Header("In-Reply-To", m.References[len(m.References)-1])

745 }

746 xc.Header("MIME-Version", "1.0")

747

748 var haveUserAgent bool

749 for _, kv := range req.Headers {

750 xcheckcontrol(kv[0])

751 xcheckcontrol(kv[1])

752 xc.Header(kv[0], kv[1])

753 if strings.EqualFold(kv[0], "User-Agent") || strings.EqualFold(kv[0], "X-Mailer") {

754 haveUserAgent = true

755 }

756 }

757 if !haveUserAgent {

758 xc.Header("User-Agent", "mox/"+moxvar.Version)

759 }

760

761 // Whether we have additional separately alternative/inline/attached file(s).

762 mpf := reqInfo.Request.MultipartForm

763 formAlternative := mpf != nil && len(mpf.File["alternativefile"]) > 0

764 formInline := mpf != nil && len(mpf.File["inlinefile"]) > 0

765 formAttachment := mpf != nil && len(mpf.File["attachedfile"]) > 0

766

767 // MIME structure we'll build:

768 // - multipart/mixed (in case of attached files)

769 // - multipart/related (in case of inline files, we assume they are relevant both text and html part if present)

770 // - multipart/alternative (in case we have both text and html bodies)

771 // - text/plain (optional)

772 // - text/html (optional)

773 // - alternative file, ...

774 // - inline file, ...

775 // - attached file, ...

776

777 // We keep track of cur, which is where we add new parts to, whether the text or

778 // html part, or the inline or attached files.

779 var cur, mixed, related, alternative *multipart.Writer

780 xcreateMultipart := func(subtype string) *multipart.Writer {

781 mp := multipart.NewWriter(xc)

782 if cur == nil {

783 xc.Header("Content-Type", fmt.Sprintf(`multipart/%s; boundary="%s"`, subtype, mp.Boundary()))

784 xc.Line()

785 } else {

786 _, err := cur.CreatePart(textproto.MIMEHeader{"Content-Type": []string{fmt.Sprintf(`multipart/%s; boundary="%s"`, subtype, mp.Boundary())}})

787 xcheckf(err, "adding multipart")

788 }

789 return mp

790 }

791 xcreatePart := func(header textproto.MIMEHeader) io.Writer {

792 if cur == nil {

793 for k, vl := range header {

794 for _, v := range vl {

795 xc.Header(k, v)

796 }

797 }

798 xc.Line()

799 return xc

800 }

801 p, err := cur.CreatePart(header)

802 xcheckf(err, "adding part")

803 return p

804 }

805 // We create multiparts from outer structure to inner. Then for each we add its

806 // inner parts and close the multipart.

807 if len(req.AttachedFiles) > 0 || formAttachment {

808 mixed = xcreateMultipart("mixed")

809 cur = mixed

810 }

811 if len(req.InlineFiles) > 0 || formInline {

812 related = xcreateMultipart("related")

813 cur = related

814 }

815 if m.Text != "" && m.HTML != "" || len(req.AlternativeFiles) > 0 || formAlternative {

816 alternative = xcreateMultipart("alternative")

817 cur = alternative

818 }

819 if m.Text != "" {

820 textBody, ct, cte := xc.TextPart("plain", m.Text)

821 tp := xcreatePart(textproto.MIMEHeader{"Content-Type": []string{ct}, "Content-Transfer-Encoding": []string{cte}})

822 _, err := tp.Write([]byte(textBody))

823 xcheckf(err, "write text part")

824 }

825 if m.HTML != "" {

826 htmlBody, ct, cte := xc.TextPart("html", m.HTML)

827 tp := xcreatePart(textproto.MIMEHeader{"Content-Type": []string{ct}, "Content-Transfer-Encoding": []string{cte}})

828 _, err := tp.Write([]byte(htmlBody))

829 xcheckf(err, "write html part")

830 }

831

832 xaddFileBase64 := func(ct string, inline bool, filename string, cid string, base64Data string) {

833 h := textproto.MIMEHeader{}

834 disp := "attachment"

835 if inline {

836 disp = "inline"

837 }

838 cd := mime.FormatMediaType(disp, map[string]string{"filename": filename})

839

840 h.Set("Content-Type", ct)

841 h.Set("Content-Disposition", cd)

842 if cid != "" {

843 h.Set("Content-ID", cid)

844 }

845 h.Set("Content-Transfer-Encoding", "base64")

846 p := xcreatePart(h)

847

848 for len(base64Data) > 0 {

849 line := base64Data

850 n := len(line)

851 if n > 78 {

852 n = 78

853 }

854 line, base64Data = base64Data[:n], base64Data[n:]

855 _, err := p.Write([]byte(line))

856 xcheckf(err, "writing attachment")

857 _, err = p.Write([]byte("\r\n"))

858 xcheckf(err, "writing attachment")

859 }

860 }

861 xaddJSONFiles := func(l []webapi.File, inline bool) {

862 for _, f := range l {

863 if f.ContentType == "" {

864 buf, _ := io.ReadAll(io.LimitReader(base64.NewDecoder(base64.StdEncoding, strings.NewReader(f.Data)), 512))

865 f.ContentType = http.DetectContentType(buf)

866 if f.ContentType == "application/octet-stream" {

867 f.ContentType = ""

868 }

869 }

870

871 // Ensure base64 is valid, then we'll write the original string.

872 _, err := io.Copy(io.Discard, base64.NewDecoder(base64.StdEncoding, strings.NewReader(f.Data)))

873 xcheckuserf(err, "parsing attachment as base64")

874

875 xaddFileBase64(f.ContentType, inline, f.Name, f.ContentID, f.Data)

876 }

877 }

878 xaddFile := func(fh *multipart.FileHeader, inline bool) {

879 f, err := fh.Open()

880 xcheckf(err, "open uploaded file")

881 defer func() {

882 err := f.Close()

883 log.Check(err, "closing uploaded file")

884 }()

885

886 ct := fh.Header.Get("Content-Type")

887 if ct == "" {

888 buf, err := io.ReadAll(io.LimitReader(f, 512))

889 if err == nil {

890 ct = http.DetectContentType(buf)

891 }

892 _, err = f.Seek(0, 0)

893 xcheckf(err, "rewind uploaded file after content-detection")

894 if ct == "application/octet-stream" {

895 ct = ""

896 }

897 }

898

899 h := textproto.MIMEHeader{}

900 disp := "attachment"

901 if inline {

902 disp = "inline"

903 }

904 cd := mime.FormatMediaType(disp, map[string]string{"filename": fh.Filename})

905

906 if ct != "" {

907 h.Set("Content-Type", ct)

908 }

909 h.Set("Content-Disposition", cd)

910 cid := fh.Header.Get("Content-ID")

911 if cid != "" {

912 h.Set("Content-ID", cid)

913 }

914 h.Set("Content-Transfer-Encoding", "base64")

915 p := xcreatePart(h)

916 bw := moxio.Base64Writer(p)

917 _, err = io.Copy(bw, f)

918 xcheckf(err, "adding uploaded file")

919 err = bw.Close()

920 xcheckf(err, "flushing uploaded file")

921 }

922

923 cur = alternative

924 xaddJSONFiles(req.AlternativeFiles, true)

925 if mpf != nil {

926 for _, fh := range mpf.File["alternativefile"] {

927 xaddFile(fh, true)

928 }

929 }

930 if alternative != nil {

931 alternative.Close()

932 alternative = nil

933 }

934

935 cur = related

936 xaddJSONFiles(req.InlineFiles, true)

937 if mpf != nil {

938 for _, fh := range mpf.File["inlinefile"] {

939 xaddFile(fh, true)

940 }

941 }

942 if related != nil {

943 related.Close()

944 related = nil

945 }

946 cur = mixed

947 xaddJSONFiles(req.AttachedFiles, false)

948 if mpf != nil {

949 for _, fh := range mpf.File["attachedfile"] {

950 xaddFile(fh, false)

951 }

952 }

953 if mixed != nil {

954 mixed.Close()

955 mixed = nil

956 }

957 cur = nil

958 xc.Flush()

959

960 // Add DKIM-Signature headers.

961 var msgPrefix string

962 fd := from.Address.Domain

963 confDom, _ := mox.Conf.Domain(fd)

964 selectors := mox.DKIMSelectors(confDom.DKIM)

965 if len(selectors) > 0 {

966 dkimHeaders, err := dkim.Sign(ctx, log.Logger, from.Address.Localpart, fd, selectors, smtputf8, dataFile)

967 if err != nil {

968 metricServerErrors.WithLabelValues("dkimsign").Inc()

969 }

970 xcheckf(err, "sign dkim")

971

972 msgPrefix = dkimHeaders

973 }

974

975 loginAddr, err := smtp.ParseAddress(reqInfo.LoginAddress)

976 xcheckf(err, "parsing login address")

977 useFromID := slices.Contains(accConf.ParsedFromIDLoginAddresses, loginAddr)

978 var localpartBase string

979 if useFromID {

980 if confDom.LocalpartCatchallSeparator == "" {

981 xcheckuserf(errors.New(`localpart catchall separator must be configured for domain`), `composing unique "from" address`)

982 }

983 localpartBase = strings.SplitN(string(fromPath.Localpart), confDom.LocalpartCatchallSeparator, 2)[0]

984 }

985 fromIDs := make([]string, len(recipients))

986 qml := make([]queue.Msg, len(recipients))

987 now := time.Now()

988 for i, rcpt := range recipients {

989 fp := fromPath

990 if useFromID {

991 fromIDs[i] = xrandomID(16)

992 fp.Localpart = smtp.Localpart(localpartBase + confDom.LocalpartCatchallSeparator + fromIDs[i])

993 }

994

995 // Don't use per-recipient unique message prefix when multiple recipients are

996 // present, we want to keep the message identical.

997 var recvRcpt string

998 if len(recipients) == 1 {

999 recvRcpt = rcpt.XString(smtputf8)

1000 }

1001 rcptMsgPrefix := recvHdrFor(recvRcpt) + msgPrefix

1002 msgSize := int64(len(rcptMsgPrefix)) + xc.Size

1003 qm := queue.MakeMsg(fp, rcpt, xc.Has8bit, xc.SMTPUTF8, msgSize, m.MessageID, []byte(rcptMsgPrefix), req.RequireTLS, now, m.Subject)

1004 qm.FromID = fromIDs[i]

1005 qm.Extra = req.Extra

1006 if req.FutureRelease != nil {

1007 ival := time.Until(*req.FutureRelease)

1008 if ival > queue.FutureReleaseIntervalMax {

1009 xcheckuserf(fmt.Errorf("date/time can not be further than %v in the future", queue.FutureReleaseIntervalMax), "scheduling delivery")

1010 }

1011 qm.NextAttempt = *req.FutureRelease

1012 qm.FutureReleaseRequest = "until;" + req.FutureRelease.Format(time.RFC3339)

1013 // todo: possibly add a header to the message stored in the Sent mailbox to indicate it was scheduled for later delivery.

1014 }

1015 qml[i] = qm

1016 }

1017 err = queue.Add(ctx, log, acc.Name, dataFile, qml...)

1018 if err != nil {

1019 metricSubmission.WithLabelValues("queueerror").Inc()

1020 }

1021 xcheckf(err, "adding messages to the delivery queue")

1022 metricSubmission.WithLabelValues("ok").Inc()

1023

1024 if req.SaveSent {

1025 // Append message to Sent mailbox and mark original messages as answered/forwarded.

1026 acc.WithRLock(func() {

1027 var changes []store.Change

1028

1029 metricked := false

1030 defer func() {

1031 if x := recover(); x != nil {

1032 if !metricked {

1033 metricServerErrors.WithLabelValues("submit").Inc()

1034 }

1035 panic(x)

1036 }

1037 }()

1038 xdbwrite(ctx, reqInfo.Account, func(tx *bstore.Tx) {

1039 sentmb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Sent", true).Get()

1040 if err == bstore.ErrAbsent {

1041 // There is no mailbox designated as Sent mailbox, so we're done.

1042 return

1043 }

1044 xcheckf(err, "message submitted to queue, adding to Sent mailbox")

1045

1046 modseq, err := acc.NextModSeq(tx)

1047 xcheckf(err, "next modseq")

1048

1049 // If there were bcc headers, prepend those to the stored message only, before the

1050 // DKIM signature. The DKIM-signature oversigns the bcc header, so this stored message

1051 // won't validate with DKIM anymore, which is fine.

1052 if len(bcc) > 0 {

1053 var sb strings.Builder

1054 xbcc := message.NewComposer(&sb, 100*1024, smtputf8)

1055 xbcc.HeaderAddrs("Bcc", bcc)

1056 xbcc.Flush()

1057 msgPrefix = sb.String() + msgPrefix

1058 }

1059

1060 sentm := store.Message{

1061 CreateSeq: modseq,

1062 ModSeq: modseq,

1063 MailboxID: sentmb.ID,

1064 MailboxOrigID: sentmb.ID,

1065 Flags: store.Flags{Notjunk: true, Seen: true},

1066 Size: int64(len(msgPrefix)) + xc.Size,

1067 MsgPrefix: []byte(msgPrefix),

1068 }

1069

1070 if ok, maxSize, err := acc.CanAddMessageSize(tx, sentm.Size); err != nil {

1071 xcheckf(err, "checking quota")

1072 } else if !ok {

1073 panic(webapi.Error{Code: "sentOverQuota", Message: fmt.Sprintf("message was sent, but not stored in sent mailbox due to quota of total %d bytes reached", maxSize)})

1074 }

1075

1076 // Update mailbox before delivery, which changes uidnext.

1077 sentmb.Add(sentm.MailboxCounts())

1078 err = tx.Update(&sentmb)

1079 xcheckf(err, "updating sent mailbox for counts")

1080

1081 err = acc.DeliverMessage(log, tx, &sentm, dataFile, true, false, false, true)

1082 if err != nil {

1083 metricSubmission.WithLabelValues("storesenterror").Inc()

1084 metricked = true

1085 }