1// Package webapisrv implements the server-side of the webapi.

2package webapisrv

4// In a separate package from webapi, so webapi.Client can be used and imported

5// without including all mox internals. Documentation for the functions is in

6// ../webapi/client.go.

8import (

9 "bytes"

10 "context"

11 cryptorand "crypto/rand"

12 "encoding/base64"

13 "encoding/json"

14 "errors"

15 "fmt"

16 htmltemplate "html/template"

17 "io"

18 "log/slog"

19 "mime"

20 "mime/multipart"

21 "net/http"

22 "net/textproto"

23 "reflect"

24 "runtime/debug"

25 "slices"

26 "strings"

27 "time"

29 "github.com/prometheus/client_golang/prometheus"

30 "github.com/prometheus/client_golang/prometheus/promauto"

32 "github.com/mjl-/bstore"

34 "github.com/mjl-/mox/dkim"

35 "github.com/mjl-/mox/dns"

36 "github.com/mjl-/mox/message"

37 "github.com/mjl-/mox/metrics"

38 "github.com/mjl-/mox/mlog"

39 "github.com/mjl-/mox/mox-"

40 "github.com/mjl-/mox/moxio"

41 "github.com/mjl-/mox/moxvar"

42 "github.com/mjl-/mox/queue"

43 "github.com/mjl-/mox/smtp"

44 "github.com/mjl-/mox/store"

45 "github.com/mjl-/mox/webapi"

46 "github.com/mjl-/mox/webauth"

47 "github.com/mjl-/mox/webhook"

48 "github.com/mjl-/mox/webops"

49)

51var pkglog = mlog.New("webapi", nil)

53var (

54 // Similar between ../webmail/webmail.go:/metricSubmission and ../smtpserver/server.go:/metricSubmission and ../webapisrv/server.go:/metricSubmission

55 metricSubmission = promauto.NewCounterVec(

56 prometheus.CounterOpts{

57 Name: "mox_webapi_submission_total",

58 Help: "Webapi message submission results, known values (those ending with error are server errors): ok, badfrom, messagelimiterror, recipientlimiterror, queueerror, storesenterror.",

59 },

60 []string{

61 "result",

62 },

63 )

64 metricServerErrors = promauto.NewCounterVec(

65 prometheus.CounterOpts{

66 Name: "mox_webapi_errors_total",

67 Help: "Webapi server errors, known values: dkimsign, submit.",

68 },

69 []string{

70 "error",

71 },

72 )

73 metricResults = promauto.NewCounterVec(

74 prometheus.CounterOpts{

75 Name: "mox_webapi_results_total",

76 Help: "HTTP webapi results by method and result.",

77 },

78 []string{"method", "result"}, // result: "badauth", "ok", or error code

79 )

80 metricDuration = promauto.NewHistogramVec(

81 prometheus.HistogramOpts{

82 Name: "mox_webapi_duration_seconds",

83 Help: "HTTP webhook call duration.",

84 Buckets: []float64{0.01, 0.05, 0.1, 0.5, 1, 5, 10, 20, 30},

85 },

86 []string{"method"},

87 )

88)

90// We pass the request to the handler so the TLS info can be used for

91// the Received header in submitted messages. Most API calls need just the

92// account name.

93type ctxKey string

95var requestInfoCtxKey ctxKey = "requestInfo"

97type requestInfo struct {

98 Log mlog.Log

99 LoginAddress string

100 Account *store.Account

101 Response http.ResponseWriter // For setting headers for non-JSON responses.

102 Request *http.Request // For Proto and TLS connection state during message submit.

103}

104

105// todo: show a curl invocation on the method pages

106

107var docsMethodTemplate = htmltemplate.Must(htmltemplate.New("method").Parse(`<!doctype html>

108 <head>

109 <meta charset="utf-8" />

110 <meta name="robots" content="noindex,nofollow" />

111 <title>Method {{ .Method }} - WebAPI - Mox</title>

112 <style>

113body, html { padding: 1em; font-size: 16px; }

114* { font-size: inherit; font-family: ubuntu, lato, sans-serif; margin: 0; padding: 0; box-sizing: border-box; }

115h1, h2, h3, h4 { margin-bottom: 1ex; }

116h1 { font-size: 1.2rem; }

117h2 { font-size: 1.1rem; }

118h3, h4 { font-size: 1rem; }

119ul { padding-left: 1rem; }

120p { margin-bottom: 1em; max-width: 50em; }

121[title] { text-decoration: underline; text-decoration-style: dotted; }

122fieldset { border: 0; }

123textarea { width: 100%; max-width: 50em; }

124 </style>

125 </head>

126 <body>

127 <h1><a href="../">WebAPI</a> - Method {{ .Method }}</h1>

128 <form id="webapicall" method="POST">

129 <fieldset id="webapifieldset">

130 <h2>Request JSON</h2>

131 <div><textarea id="webapirequest" name="request" required rows="20">{{ .Request }}</textarea></div>

132 <br/>

133 <div>

134 <button type="reset">Reset</button>

135 <button type="submit">Call</button>

136 </div>

137 <br/>

138{{ if .ReturnsBytes }}

139 <p>Method has a non-JSON response.</p>

140{{ else }}

141 <h2>Response JSON</h2>

142 <div><textarea id="webapiresponse" rows="20">{{ .Response }}</textarea></div>

143{{ end }}

144 </fieldset>

145 </form>

146 <script>

147window.addEventListener('load', () => {

148 window.webapicall.addEventListener('submit', async (e) => {

149 const stop = () => {

150 e.stopPropagation()

151 e.preventDefault()

152 }

153

154 let req

155 try {

156 req = JSON.parse(window.webapirequest.value)

157 } catch (err) {

158 window.alert('Error parsing request: ' + err.message)

159 stop()

160 return

161 }

162 if (!req) {

163 window.alert('Empty request')

164 stop()

165 return

166 }

167

168 if ({{ .ReturnsBytes }}) {

169 // Just POST to this URL.

170 return

171 }

172

173 stop()

174 // Do call ourselves, get response and put it in the response textarea.

175 window.webapifieldset.disabled = true

176 let data = new window.FormData()

177 data.append("request", window.webapirequest.value)

178 try {

179 const response = await fetch("{{ .Method }}", {body: data, method: "POST"})

180 const text = await response.text()

181 try {

182 window.webapiresponse.value = JSON.stringify(JSON.parse(text), undefined, '\t')

183 } catch (err) {

184 window.webapiresponse.value = text

185 }

186 } catch (err) {

187 window.alert('Error: ' + err.message)

188 } finally {

189 window.webapifieldset.disabled = false

190 }

191 })

192})

193 </script>

194 </body>

195</html>

196`))

197

198var docsIndex []byte

199

200func init() {

201 var methods []string

202 mt := reflect.TypeOf((*webapi.Methods)(nil)).Elem()

203 n := mt.NumMethod()

204 for i := 0; i < n; i++ {

205 methods = append(methods, mt.Method(i).Name)

206 }

207 docsIndexTmpl := htmltemplate.Must(htmltemplate.New("index").Parse(`<!doctype html>

208<html>

209 <head>

210 <meta charset="utf-8" />

211 <meta name="robots" content="noindex,nofollow" />

212 <title>Webapi - Mox</title>

213 <style>

214body, html { padding: 1em; font-size: 16px; }

215* { font-size: inherit; font-family: ubuntu, lato, sans-serif; margin: 0; padding: 0; box-sizing: border-box; }

216h1, h2, h3, h4 { margin-bottom: 1ex; }

217h1 { font-size: 1.2rem; }

218h2 { font-size: 1.1rem; }

219h3, h4 { font-size: 1rem; }

220ul { padding-left: 1rem; }

221p { margin-bottom: 1em; max-width: 50em; }

222[title] { text-decoration: underline; text-decoration-style: dotted; }

223fieldset { border: 0; }

224 </style>

225 </head>

226 <body>

227 <h1>Webapi and webhooks</h1>

228 <p>The mox webapi is a simple HTTP/JSON-based API for sending messages and processing incoming messages.</p>

229 <p>Configure webhooks in mox to receive notifications about outgoing delivery event, and/or incoming deliveries of messages.</p>

230 <p>Documentation and examples:</p>

231 <p><a href="{{ .WebapiDocsURL }}">{{ .WebapiDocsURL }}</a></p>

232 <h2>Methods</h2>

233 <p>The methods below are available in this version of mox. Follow a link for an example request/response JSON, and a button to make an API call.</p>

234 <ul>

235{{ range $i, $method := .Methods }}

236 <li><a href="{{ $method }}">{{ $method }}</a></li>

237{{ end }}

238 </ul>

239 </body>

240</html>

241`))

242 webapiDocsURL := "https://pkg.go.dev/github.com/mjl-/mox@" + moxvar.VersionBare + "/webapi/"

243 webhookDocsURL := "https://pkg.go.dev/github.com/mjl-/mox@" + moxvar.VersionBare + "/webhook/"

244 indexArgs := struct {

245 WebapiDocsURL string

246 WebhookDocsURL string

247 Methods []string

248 }{webapiDocsURL, webhookDocsURL, methods}

249 var b bytes.Buffer

250 err := docsIndexTmpl.Execute(&b, indexArgs)

251 if err != nil {

252 panic("executing api docs index template: " + err.Error())

253 }

254 docsIndex = b.Bytes()

255

256 mox.NewWebapiHandler = func(maxMsgSize int64, basePath string, isForwarded bool) http.Handler {

257 return NewServer(maxMsgSize, basePath, isForwarded)

258 }

259}

260

261// NewServer returns a new http.Handler for a webapi server.

262func NewServer(maxMsgSize int64, path string, isForwarded bool) http.Handler {

263 return server{maxMsgSize, path, isForwarded}

264}

265

266// server implements the webapi methods.

267type server struct {

268 maxMsgSize int64 // Of outgoing messages.

269 path string // Path webapi is configured under, typically /webapi/, with methods at /webapi/v0/<method>.

270 isForwarded bool // Whether incoming requests are reverse-proxied. Used for getting remote IPs for rate limiting.

271}

272

273var _ webapi.Methods = server{}

274

275// ServeHTTP implements http.Handler.

276func (s server) ServeHTTP(w http.ResponseWriter, r *http.Request) {

277 log := pkglog.WithContext(r.Context()) // Take cid from webserver.

278

279 // Send requests to /webapi/ to /webapi/v0/.

280 if r.URL.Path == "/" {

281 if r.Method != "GET" {

282 http.Error(w, "405 - method not allow", http.StatusMethodNotAllowed)

283 return

284 }

285 http.Redirect(w, r, s.path+"v0/", http.StatusSeeOther)

286 return

287 }

288 // Serve short introduction and list to methods at /webapi/v0/.

289 if r.URL.Path == "/v0/" {

290 w.Header().Set("Content-Type", "text/html; charset=utf-8")

291 w.Write(docsIndex)

292 return

293 }

294

295 // Anything else must be a method endpoint.

296 if !strings.HasPrefix(r.URL.Path, "/v0/") {

297 http.NotFound(w, r)

298 return

299 }

300 fn := r.URL.Path[len("/v0/"):]

301 log = log.With(slog.String("method", fn))

302 rfn := reflect.ValueOf(s).MethodByName(fn)

303 var zero reflect.Value

304 if rfn == zero || rfn.Type().NumIn() != 2 || rfn.Type().NumOut() != 2 {

305 log.Debug("unknown webapi method")

306 http.NotFound(w, r)

307 return

308 }

309

310 // GET on method returns an example request JSON, a button to call the method,

311 // which either fills a textarea with the response (in case of JSON) or posts to

312 // the URL letting the browser handle the response (e.g. raw message or part).

313 if r.Method == "GET" {

314 formatJSON := func(v any) (string, error) {

315 var b bytes.Buffer

316 enc := json.NewEncoder(&b)

317 enc.SetIndent("", "\t")

318 enc.SetEscapeHTML(false)

319 err := enc.Encode(v)

320 return string(b.String()), err

321 }

322

323 req, err := formatJSON(mox.FillExample(nil, reflect.New(rfn.Type().In(1))).Interface())

324 if err != nil {

325 log.Errorx("formatting request as json", err)

326 http.Error(w, "500 - internal server error - marshal request: "+err.Error(), http.StatusInternalServerError)

327 return

328 }

329 // todo: could check for io.ReadCloser, but we don't return other interfaces than that one.

330 returnsBytes := rfn.Type().Out(0).Kind() == reflect.Interface

331 var resp string

332 if !returnsBytes {

333 resp, err = formatJSON(mox.FillExample(nil, reflect.New(rfn.Type().Out(0))).Interface())

334 if err != nil {

335 log.Errorx("formatting response as json", err)

336 http.Error(w, "500 - internal server error - marshal response: "+err.Error(), http.StatusInternalServerError)

337 return

338 }

339 }

340 args := struct {

341 Method string

342 Request string

343 Response string

344 ReturnsBytes bool

345 }{fn, req, resp, returnsBytes}

346 w.Header().Set("Content-Type", "text/html; charset=utf-8")

347 err = docsMethodTemplate.Execute(w, args)

348 log.Check(err, "executing webapi method template")

349 return

350 } else if r.Method != "POST" {

351 http.Error(w, "405 - method not allowed - use get or post", http.StatusMethodNotAllowed)

352 return

353 }

354

355 // Account is available during call, but we close it before we start writing a

356 // response, to prevent slow readers from holding a reference for a long time.

357 var acc *store.Account

358 closeAccount := func() {

359 if acc != nil {

360 err := acc.Close()

361 log.Check(err, "closing account")

362 acc = nil

363 }

364 }

365 defer closeAccount()

366

367 email, password, aok := r.BasicAuth()

368 if !aok {

369 metricResults.WithLabelValues(fn, "badauth").Inc()

370 log.Debug("missing http basic authentication credentials")

371 w.Header().Set("WWW-Authenticate", "Basic realm=webapi")

372 http.Error(w, "401 - unauthorized - use http basic auth with email address as username", http.StatusUnauthorized)

373 return

374 }

375 log = log.With(slog.String("username", email))

376

377 t0 := time.Now()

378

379 // If remote IP/network resulted in too many authentication failures, refuse to serve.

380 remoteIP := webauth.RemoteIP(log, s.isForwarded, r)

381 if remoteIP == nil {

382 metricResults.WithLabelValues(fn, "internal").Inc()

383 log.Debug("cannot find remote ip for rate limiter")

384 http.Error(w, "500 - internal server error - cannot find remote ip", http.StatusInternalServerError)

385 return

386 }

387 if !mox.LimiterFailedAuth.CanAdd(remoteIP, t0, 1) {

388 metrics.AuthenticationRatelimitedInc("webapi")

389 log.Debug("refusing connection due to many auth failures", slog.Any("remoteip", remoteIP))

390 http.Error(w, "429 - too many auth attempts", http.StatusTooManyRequests)

391 return

392 }

393

394 writeError := func(err webapi.Error) {

395 closeAccount()

396 metricResults.WithLabelValues(fn, err.Code).Inc()

397

398 if err.Code == "server" {

399 log.Errorx("webapi call result", err, slog.String("resultcode", err.Code))

400 } else {

401 log.Infox("webapi call result", err, slog.String("resultcode", err.Code))

402 }

403

404 w.Header().Set("Content-Type", "application/json; charset=utf-8")

405 w.WriteHeader(http.StatusBadRequest)

406 enc := json.NewEncoder(w)

407 enc.SetEscapeHTML(false)

408 werr := enc.Encode(err)

409 if werr != nil && !moxio.IsClosed(werr) {

410 log.Infox("writing error response", werr)

411 }

412 }

413

414 // Called for all successful JSON responses, not non-JSON responses.

415 writeResponse := func(resp any) {

416 closeAccount()

417 metricResults.WithLabelValues(fn, "ok").Inc()

418 log.Debug("webapi call result", slog.String("resultcode", "ok"))

419 w.Header().Set("Content-Type", "application/json; charset=utf-8")

420 enc := json.NewEncoder(w)

421 enc.SetEscapeHTML(false)

422 werr := enc.Encode(resp)

423 if werr != nil && !moxio.IsClosed(werr) {

424 log.Infox("writing error response", werr)

425 }

426 }

427

428 authResult := "error"

429 defer func() {

430 metricDuration.WithLabelValues(fn).Observe(float64(time.Since(t0)) / float64(time.Second))

431 metrics.AuthenticationInc("webapi", "httpbasic", authResult)

432 }()

433

434 var err error

435 acc, err = store.OpenEmailAuth(log, email, password)

436 if err != nil {

437 mox.LimiterFailedAuth.Add(remoteIP, t0, 1)

438 if errors.Is(err, mox.ErrDomainNotFound) || errors.Is(err, mox.ErrAddressNotFound) || errors.Is(err, store.ErrUnknownCredentials) {

439 log.Debug("bad http basic authentication credentials")

440 metricResults.WithLabelValues(fn, "badauth").Inc()

441 authResult = "badcreds"

442 w.Header().Set("WWW-Authenticate", "Basic realm=webapi")

443 http.Error(w, "401 - unauthorized - use http basic auth with email address as username", http.StatusUnauthorized)

444 return

445 }

446 writeError(webapi.Error{Code: "server", Message: "error verifying credentials"})

447 return

448 }

449 authResult = "ok"

450 mox.LimiterFailedAuth.Reset(remoteIP, t0)

451

452 ct := r.Header.Get("Content-Type")

453 ct, _, err = mime.ParseMediaType(ct)

454 if err != nil {

455 writeError(webapi.Error{Code: "protocol", Message: "unknown content-type " + r.Header.Get("Content-Type")})

456 return

457 }

458 if ct == "multipart/form-data" {

459 err = r.ParseMultipartForm(200 * 1024)

460 } else {

461 err = r.ParseForm()

462 }

463 if err != nil {

464 writeError(webapi.Error{Code: "protocol", Message: "parsing form: " + err.Error()})

465 return

466 }

467

468 reqstr := r.PostFormValue("request")

469 if reqstr == "" {

470 writeError(webapi.Error{Code: "protocol", Message: "missing/empty request"})

471 return

472 }

473

474 defer func() {

475 x := recover()

476 if x == nil {

477 return

478 }

479 if err, eok := x.(webapi.Error); eok {

480 writeError(err)

481 return

482 }

483 log.Error("unhandled panic in webapi call", slog.Any("x", x), slog.String("resultcode", "server"))

484 metrics.PanicInc(metrics.Webapi)

485 debug.PrintStack()

486 writeError(webapi.Error{Code: "server", Message: "unhandled error"})

487 }()

488 req := reflect.New(rfn.Type().In(1))

489 dec := json.NewDecoder(strings.NewReader(reqstr))

490 dec.DisallowUnknownFields()

491 if err := dec.Decode(req.Interface()); err != nil {

492 writeError(webapi.Error{Code: "protocol", Message: fmt.Sprintf("parsing request: %s", err)})

493 return

494 }

495

496 reqInfo := requestInfo{log, email, acc, w, r}

497 nctx := context.WithValue(r.Context(), requestInfoCtxKey, reqInfo)

498 resp := rfn.Call([]reflect.Value{reflect.ValueOf(nctx), req.Elem()})

499 if !resp[1].IsZero() {

500 var e webapi.Error

501 err := resp[1].Interface().(error)

502 if x, eok := err.(webapi.Error); eok {

503 e = x

504 } else {

505 e = webapi.Error{Code: "error", Message: err.Error()}

506 }

507 writeError(e)

508 return

509 }

510 rc, ok := resp[0].Interface().(io.ReadCloser)

511 if !ok {

512 rv, _ := mox.FillNil(resp[0])

513 writeResponse(rv.Interface())

514 return

515 }

516 closeAccount()

517 log.Debug("webapi call result", slog.String("resultcode", "ok"))

518 metricResults.WithLabelValues(fn, "ok").Inc()

519 defer rc.Close()

520 if _, err := io.Copy(w, rc); err != nil && !moxio.IsClosed(err) {

521 log.Errorx("writing response to client", err)

522 }

523}

524

525func xcheckf(err error, format string, args ...any) {

526 if err != nil {

527 msg := fmt.Sprintf(format, args...)

528 panic(webapi.Error{Code: "server", Message: fmt.Sprintf("%s: %s", msg, err)})

529 }

530}

531

532func xcheckuserf(err error, format string, args ...any) {

533 if err != nil {

534 msg := fmt.Sprintf(format, args...)

535 panic(webapi.Error{Code: "user", Message: fmt.Sprintf("%s: %s", msg, err)})

536 }

537}

538

539func xdbwrite(ctx context.Context, acc *store.Account, fn func(tx *bstore.Tx)) {

540 err := acc.DB.Write(ctx, func(tx *bstore.Tx) error {

541 fn(tx)

542 return nil

543 })

544 xcheckf(err, "transaction")

545}

546

547func xdbread(ctx context.Context, acc *store.Account, fn func(tx *bstore.Tx)) {

548 err := acc.DB.Read(ctx, func(tx *bstore.Tx) error {

549 fn(tx)

550 return nil

551 })

552 xcheckf(err, "transaction")

553}

554

555func xcheckcontrol(s string) {

556 for _, c := range s {

557 if c < 0x20 {

558 xcheckuserf(errors.New("control characters not allowed"), "checking header values")

559 }

560 }

561}

562

563func xparseAddress(addr string) smtp.Address {

564 a, err := smtp.ParseAddress(addr)

565 if err != nil {

566 panic(webapi.Error{Code: "badAddress", Message: fmt.Sprintf("parsing address %q: %s", addr, err)})

567 }

568 return a

569}

570

571func xparseAddresses(l []webapi.NameAddress) ([]message.NameAddress, []smtp.Path) {

572 r := make([]message.NameAddress, len(l))

573 paths := make([]smtp.Path, len(l))

574 for i, a := range l {

575 xcheckcontrol(a.Name)

576 addr := xparseAddress(a.Address)

577 r[i] = message.NameAddress{DisplayName: a.Name, Address: addr}

578 paths[i] = addr.Path()

579 }

580 return r, paths

581}

582

583func xrandomID(n int) string {

584 return base64.RawURLEncoding.EncodeToString(xrandom(n))

585}

586

587func xrandom(n int) []byte {

588 buf := make([]byte, n)

589 x, err := cryptorand.Read(buf)

590 if err != nil {

591 panic("read random")

592 } else if x != n {

593 panic("short random read")

594 }

595 return buf

596}

597

598func (s server) Send(ctx context.Context, req webapi.SendRequest) (resp webapi.SendResult, err error) {

599 // Similar between ../smtpserver/server.go:/submit\( and ../webmail/api.go:/MessageSubmit\( and ../webapisrv/server.go:/Send\(

600

601 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

602 log := reqInfo.Log

603 acc := reqInfo.Account

604

605 m := req.Message

606

607 accConf, _ := acc.Conf()

608

609 if m.Text == "" && m.HTML == "" {

610 return resp, webapi.Error{Code: "missingBody", Message: "at least text or html body required"}

611 }

612

613 if len(m.From) == 0 {

614 m.From = []webapi.NameAddress{{Name: accConf.FullName, Address: reqInfo.LoginAddress}}

615 } else if len(m.From) > 1 {

616 return resp, webapi.Error{Code: "multipleFrom", Message: "multiple from-addresses not allowed"}

617 }

618 froms, fromPaths := xparseAddresses(m.From)

619 from, fromPath := froms[0], fromPaths[0]

620 to, toPaths := xparseAddresses(m.To)

621 cc, ccPaths := xparseAddresses(m.CC)

622 bcc, bccPaths := xparseAddresses(m.BCC)

623

624 recipients := append(append(toPaths, ccPaths...), bccPaths...)

625 addresses := append(append(m.To, m.CC...), m.BCC...)

626

627 // Check if from address is allowed for account.

628 if !mox.AllowMsgFrom(acc.Name, from.Address) {

629 metricSubmission.WithLabelValues("badfrom").Inc()

630 return resp, webapi.Error{Code: "badFrom", Message: "from-address not configured for account"}

631 }

632

633 if len(recipients) == 0 {

634 return resp, webapi.Error{Code: "noRecipients", Message: "no recipients"}

635 }

636

637 // Check outgoing message rate limit.

638 xdbread(ctx, acc, func(tx *bstore.Tx) {

639 msglimit, rcptlimit, err := acc.SendLimitReached(tx, recipients)

640 if msglimit >= 0 {

641 metricSubmission.WithLabelValues("messagelimiterror").Inc()

642 panic(webapi.Error{Code: "messageLimitReached", Message: "outgoing message rate limit reached"})

643 } else if rcptlimit >= 0 {

644 metricSubmission.WithLabelValues("recipientlimiterror").Inc()

645 panic(webapi.Error{Code: "recipientLimitReached", Message: "outgoing new recipient rate limit reached"})

646 }

647 xcheckf(err, "checking send limit")

648 })

649

650 // If we have a non-ascii localpart, we will be sending with smtputf8. We'll go

651 // full utf-8 then.

652 intl := func(l []smtp.Path) bool {

653 for _, p := range l {

654 if p.Localpart.IsInternational() {

655 return true

656 }

657 }

658 return false

659 }

660 smtputf8 := intl([]smtp.Path{fromPath}) || intl(toPaths) || intl(ccPaths) || intl(bccPaths)

661

662 replyTos, replyToPaths := xparseAddresses(m.ReplyTo)

663 for _, rt := range replyToPaths {

664 if rt.Localpart.IsInternational() {

665 smtputf8 = true

666 }

667 }

668

669 // Create file to compose message into.

670 dataFile, err := store.CreateMessageTemp(log, "webapi-submit")

671 xcheckf(err, "creating temporary file for message")

672 defer store.CloseRemoveTempFile(log, dataFile, "message to submit")

673

674 // If writing to the message file fails, we abort immediately.

675 xc := message.NewComposer(dataFile, s.maxMsgSize, smtputf8)

676 defer func() {

677 x := recover()

678 if x == nil {

679 return

680 }

681 if err, ok := x.(error); ok && errors.Is(err, message.ErrMessageSize) {

682 panic(webapi.Error{Code: "messageTooLarge", Message: "message too large"})

683 } else if ok && errors.Is(err, message.ErrCompose) {

684 xcheckf(err, "making message")

685 }

686 panic(x)

687 }()

688

689 // Each queued message gets a Received header.

690 // We cannot use VIA, because there is no registered method. We would like to use

691 // it to add the ascii domain name in case of smtputf8 and IDNA host name.

692 // We don't add the IP address of the submitter. Exposing likely not desirable.

693 recvFrom := message.HeaderCommentDomain(mox.Conf.Static.HostnameDomain, smtputf8)

694 recvBy := mox.Conf.Static.HostnameDomain.XName(smtputf8)

695 recvID := mox.ReceivedID(mox.CidFromCtx(ctx))

696 recvHdrFor := func(rcptTo string) string {

697 recvHdr := &message.HeaderWriter{}

698 // For additional Received-header clauses, see:

699 // https://www.iana.org/assignments/mail-parameters/mail-parameters.xhtml#table-mail-parameters-8

700 // Note: we don't have "via" or "with", there is no registered for webmail.

701 recvHdr.Add(" ", "Received:", "from", recvFrom, "by", recvBy, "id", recvID) // ../rfc/5321:3158

702 if reqInfo.Request.TLS != nil {

703 recvHdr.Add(" ", mox.TLSReceivedComment(log, *reqInfo.Request.TLS)...)

704 }

705 recvHdr.Add(" ", "for", "<"+rcptTo+">;", time.Now().Format(message.RFC5322Z))

706 return recvHdr.String()

707 }

708

709 // Outer message headers.

710 xc.HeaderAddrs("From", []message.NameAddress{from})

711 if len(replyTos) > 0 {

712 xc.HeaderAddrs("Reply-To", replyTos)

713 }

714 xc.HeaderAddrs("To", to)

715 xc.HeaderAddrs("Cc", cc)

716 // We prepend Bcc headers to the message when adding to the Sent mailbox.

717 if m.Subject != "" {

718 xcheckcontrol(m.Subject)

719 xc.Subject(m.Subject)

720 }

721

722 var date time.Time

723 if m.Date != nil {

724 date = *m.Date

725 } else {

726 date = time.Now()

727 }

728 xc.Header("Date", date.Format(message.RFC5322Z))

729

730 if m.MessageID == "" {

731 m.MessageID = fmt.Sprintf("<%s>", mox.MessageIDGen(smtputf8))

732 } else if !strings.HasPrefix(m.MessageID, "<") || !strings.HasSuffix(m.MessageID, ">") {

733 return resp, webapi.Error{Code: "malformedMessageID", Message: "missing <> in message-id"}

734 }

735 xcheckcontrol(m.MessageID)

736 xc.Header("Message-Id", m.MessageID)

737

738 if len(m.References) > 0 {

739 for _, ref := range m.References {

740 xcheckcontrol(ref)

741 // We don't check for <>'s. If caller just puts in what they got, we don't want to

742 // reject the message.

743 }

744 xc.Header("References", strings.Join(m.References, "\r\n\t"))

745 xc.Header("In-Reply-To", m.References[len(m.References)-1])

746 }

747 xc.Header("MIME-Version", "1.0")

748

749 var haveUserAgent bool

750 for _, kv := range req.Headers {

751 xcheckcontrol(kv[0])

752 xcheckcontrol(kv[1])

753 xc.Header(kv[0], kv[1])

754 if strings.EqualFold(kv[0], "User-Agent") || strings.EqualFold(kv[0], "X-Mailer") {

755 haveUserAgent = true

756 }

757 }

758 if !haveUserAgent {

759 xc.Header("User-Agent", "mox/"+moxvar.Version)

760 }

761

762 // Whether we have additional separately alternative/inline/attached file(s).

763 mpf := reqInfo.Request.MultipartForm

764 formAlternative := mpf != nil && len(mpf.File["alternativefile"]) > 0

765 formInline := mpf != nil && len(mpf.File["inlinefile"]) > 0

766 formAttachment := mpf != nil && len(mpf.File["attachedfile"]) > 0

767

768 // MIME structure we'll build:

769 // - multipart/mixed (in case of attached files)

770 // - multipart/related (in case of inline files, we assume they are relevant both text and html part if present)

771 // - multipart/alternative (in case we have both text and html bodies)

772 // - text/plain (optional)

773 // - text/html (optional)

774 // - alternative file, ...

775 // - inline file, ...

776 // - attached file, ...

777

778 // We keep track of cur, which is where we add new parts to, whether the text or

779 // html part, or the inline or attached files.

780 var cur, mixed, related, alternative *multipart.Writer

781 xcreateMultipart := func(subtype string) *multipart.Writer {

782 mp := multipart.NewWriter(xc)

783 if cur == nil {

784 xc.Header("Content-Type", fmt.Sprintf(`multipart/%s; boundary="%s"`, subtype, mp.Boundary()))

785 xc.Line()

786 } else {

787 _, err := cur.CreatePart(textproto.MIMEHeader{"Content-Type": []string{fmt.Sprintf(`multipart/%s; boundary="%s"`, subtype, mp.Boundary())}})

788 xcheckf(err, "adding multipart")

789 }

790 return mp

791 }

792 xcreatePart := func(header textproto.MIMEHeader) io.Writer {

793 if cur == nil {

794 for k, vl := range header {

795 for _, v := range vl {

796 xc.Header(k, v)

797 }

798 }

799 xc.Line()

800 return xc

801 }

802 p, err := cur.CreatePart(header)

803 xcheckf(err, "adding part")

804 return p

805 }

806 // We create multiparts from outer structure to inner. Then for each we add its

807 // inner parts and close the multipart.

808 if len(req.AttachedFiles) > 0 || formAttachment {

809 mixed = xcreateMultipart("mixed")

810 cur = mixed

811 }

812 if len(req.InlineFiles) > 0 || formInline {

813 related = xcreateMultipart("related")

814 cur = related

815 }

816 if m.Text != "" && m.HTML != "" || len(req.AlternativeFiles) > 0 || formAlternative {

817 alternative = xcreateMultipart("alternative")

818 cur = alternative

819 }

820 if m.Text != "" {

821 textBody, ct, cte := xc.TextPart("plain", m.Text)

822 tp := xcreatePart(textproto.MIMEHeader{"Content-Type": []string{ct}, "Content-Transfer-Encoding": []string{cte}})

823 _, err := tp.Write([]byte(textBody))

824 xcheckf(err, "write text part")

825 }

826 if m.HTML != "" {

827 htmlBody, ct, cte := xc.TextPart("html", m.HTML)

828 tp := xcreatePart(textproto.MIMEHeader{"Content-Type": []string{ct}, "Content-Transfer-Encoding": []string{cte}})

829 _, err := tp.Write([]byte(htmlBody))

830 xcheckf(err, "write html part")

831 }

832

833 xaddFileBase64 := func(ct string, inline bool, filename string, cid string, base64Data string) {

834 h := textproto.MIMEHeader{}

835 disp := "attachment"

836 if inline {

837 disp = "inline"

838 }

839 cd := mime.FormatMediaType(disp, map[string]string{"filename": filename})

840

841 h.Set("Content-Type", ct)

842 h.Set("Content-Disposition", cd)

843 if cid != "" {

844 h.Set("Content-ID", cid)

845 }

846 h.Set("Content-Transfer-Encoding", "base64")

847 p := xcreatePart(h)

848

849 for len(base64Data) > 0 {

850 line := base64Data

851 n := len(line)

852 if n > 78 {

853 n = 78

854 }

855 line, base64Data = base64Data[:n], base64Data[n:]

856 _, err := p.Write([]byte(line))

857 xcheckf(err, "writing attachment")

858 _, err = p.Write([]byte("\r\n"))

859 xcheckf(err, "writing attachment")

860 }

861 }

862 xaddJSONFiles := func(l []webapi.File, inline bool) {

863 for _, f := range l {

864 if f.ContentType == "" {

865 buf, _ := io.ReadAll(io.LimitReader(base64.NewDecoder(base64.StdEncoding, strings.NewReader(f.Data)), 512))

866 f.ContentType = http.DetectContentType(buf)

867 if f.ContentType == "application/octet-stream" {

868 f.ContentType = ""

869 }

870 }

871

872 // Ensure base64 is valid, then we'll write the original string.

873 _, err := io.Copy(io.Discard, base64.NewDecoder(base64.StdEncoding, strings.NewReader(f.Data)))

874 xcheckuserf(err, "parsing attachment as base64")

875

876 xaddFileBase64(f.ContentType, inline, f.Name, f.ContentID, f.Data)

877 }

878 }

879 xaddFile := func(fh *multipart.FileHeader, inline bool) {

880 f, err := fh.Open()

881 xcheckf(err, "open uploaded file")

882 defer func() {

883 err := f.Close()

884 log.Check(err, "closing uploaded file")

885 }()

886

887 ct := fh.Header.Get("Content-Type")

888 if ct == "" {

889 buf, err := io.ReadAll(io.LimitReader(f, 512))

890 if err == nil {

891 ct = http.DetectContentType(buf)

892 }

893 _, err = f.Seek(0, 0)

894 xcheckf(err, "rewind uploaded file after content-detection")

895 if ct == "application/octet-stream" {

896 ct = ""

897 }

898 }

899

900 h := textproto.MIMEHeader{}

901 disp := "attachment"

902 if inline {

903 disp = "inline"

904 }

905 cd := mime.FormatMediaType(disp, map[string]string{"filename": fh.Filename})

906

907 if ct != "" {

908 h.Set("Content-Type", ct)

909 }

910 h.Set("Content-Disposition", cd)

911 cid := fh.Header.Get("Content-ID")

912 if cid != "" {

913 h.Set("Content-ID", cid)

914 }

915 h.Set("Content-Transfer-Encoding", "base64")

916 p := xcreatePart(h)

917 bw := moxio.Base64Writer(p)

918 _, err = io.Copy(bw, f)

919 xcheckf(err, "adding uploaded file")

920 err = bw.Close()

921 xcheckf(err, "flushing uploaded file")

922 }

923

924 cur = alternative

925 xaddJSONFiles(req.AlternativeFiles, true)

926 if mpf != nil {

927 for _, fh := range mpf.File["alternativefile"] {

928 xaddFile(fh, true)

929 }

930 }

931 if alternative != nil {

932 alternative.Close()

933 alternative = nil

934 }

935

936 cur = related

937 xaddJSONFiles(req.InlineFiles, true)

938 if mpf != nil {

939 for _, fh := range mpf.File["inlinefile"] {

940 xaddFile(fh, true)

941 }

942 }

943 if related != nil {

944 related.Close()

945 related = nil

946 }

947 cur = mixed

948 xaddJSONFiles(req.AttachedFiles, false)

949 if mpf != nil {

950 for _, fh := range mpf.File["attachedfile"] {

951 xaddFile(fh, false)

952 }

953 }

954 if mixed != nil {

955 mixed.Close()

956 mixed = nil

957 }

958 cur = nil

959 xc.Flush()

960

961 // Add DKIM-Signature headers.

962 var msgPrefix string

963 fd := from.Address.Domain

964 confDom, _ := mox.Conf.Domain(fd)

965 selectors := mox.DKIMSelectors(confDom.DKIM)

966 if len(selectors) > 0 {

967 dkimHeaders, err := dkim.Sign(ctx, log.Logger, from.Address.Localpart, fd, selectors, smtputf8, dataFile)

968 if err != nil {

969 metricServerErrors.WithLabelValues("dkimsign").Inc()

970 }

971 xcheckf(err, "sign dkim")

972

973 msgPrefix = dkimHeaders

974 }

975

976 loginAddr, err := smtp.ParseAddress(reqInfo.LoginAddress)

977 xcheckf(err, "parsing login address")

978 useFromID := slices.Contains(accConf.ParsedFromIDLoginAddresses, loginAddr)

979 var localpartBase string

980 if useFromID {

981 if confDom.LocalpartCatchallSeparator == "" {

982 xcheckuserf(errors.New(`localpart catchall separator must be configured for domain`), `composing unique "from" address`)

983 }

984 localpartBase = strings.SplitN(string(fromPath.Localpart), confDom.LocalpartCatchallSeparator, 2)[0]

985 }

986 fromIDs := make([]string, len(recipients))

987 qml := make([]queue.Msg, len(recipients))

988 now := time.Now()

989 for i, rcpt := range recipients {

990 fp := fromPath

991 if useFromID {

992 fromIDs[i] = xrandomID(16)

993 fp.Localpart = smtp.Localpart(localpartBase + confDom.LocalpartCatchallSeparator + fromIDs[i])

994 }

995

996 // Don't use per-recipient unique message prefix when multiple recipients are

997 // present, we want to keep the message identical.

998 var recvRcpt string

999 if len(recipients) == 1 {

1000 recvRcpt = rcpt.XString(smtputf8)

1001 }

1002 rcptMsgPrefix := recvHdrFor(recvRcpt) + msgPrefix

1003 msgSize := int64(len(rcptMsgPrefix)) + xc.Size

1004 qm := queue.MakeMsg(fp, rcpt, xc.Has8bit, xc.SMTPUTF8, msgSize, m.MessageID, []byte(rcptMsgPrefix), req.RequireTLS, now, m.Subject)

1005 qm.FromID = fromIDs[i]

1006 qm.Extra = req.Extra

1007 if req.FutureRelease != nil {

1008 ival := time.Until(*req.FutureRelease)

1009 if ival > queue.FutureReleaseIntervalMax {

1010 xcheckuserf(fmt.Errorf("date/time can not be further than %v in the future", queue.FutureReleaseIntervalMax), "scheduling delivery")

1011 }

1012 qm.NextAttempt = *req.FutureRelease

1013 qm.FutureReleaseRequest = "until;" + req.FutureRelease.Format(time.RFC3339)

1014 // todo: possibly add a header to the message stored in the Sent mailbox to indicate it was scheduled for later delivery.

1015 }

1016 qml[i] = qm

1017 }

1018 err = queue.Add(ctx, log, acc.Name, dataFile, qml...)

1019 if err != nil {

1020 metricSubmission.WithLabelValues("queueerror").Inc()

1021 }

1022 xcheckf(err, "adding messages to the delivery queue")

1023 metricSubmission.WithLabelValues("ok").Inc()

1024

1025 if req.SaveSent {

1026 // Append message to Sent mailbox and mark original messages as answered/forwarded.

1027 acc.WithRLock(func() {

1028 var changes []store.Change

1029

1030 metricked := false

1031 defer func() {

1032 if x := recover(); x != nil {

1033 if !metricked {

1034 metricServerErrors.WithLabelValues("submit").Inc()

1035 }

1036 panic(x)

1037 }

1038 }()

1039 xdbwrite(ctx, reqInfo.Account, func(tx *bstore.Tx) {

1040 sentmb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Sent", true).Get()

1041 if err == bstore.ErrAbsent {

1042 // There is no mailbox designated as Sent mailbox, so we're done.

1043 return

1044 }

1045 xcheckf(err, "message submitted to queue, adding to Sent mailbox")

1046

1047 modseq, err := acc.NextModSeq(tx)

1048 xcheckf(err, "next modseq")

1049

1050 // If there were bcc headers, prepend those to the stored message only, before the

1051 // DKIM signature. The DKIM-signature oversigns the bcc header, so this stored message

1052 // won't validate with DKIM anymore, which is fine.

1053 if len(bcc) > 0 {

1054 var sb strings.Builder

1055 xbcc := message.NewComposer(&sb, 100*1024, smtputf8)

1056 xbcc.HeaderAddrs("Bcc", bcc)

1057 xbcc.Flush()

1058 msgPrefix = sb.String() + msgPrefix

1059 }

1060

1061 sentm := store.Message{

1062 CreateSeq: modseq,

1063 ModSeq: modseq,

1064 MailboxID: sentmb.ID,

1065 MailboxOrigID: sentmb.ID,

1066 Flags: store.Flags{Notjunk: true, Seen: true},

1067 Size: int64(len(msgPrefix)) + xc.Size,

1068 MsgPrefix: []byte(msgPrefix),

1069 }

1070

1071 if ok, maxSize, err := acc.CanAddMessageSize(tx, sentm.Size); err != nil {

1072 xcheckf(err, "checking quota")

1073 } else if !ok {

1074 panic(webapi.Error{Code: "sentOverQuota", Message: fmt.Sprintf("message was sent, but not stored in sent mailbox due to quota of total %d bytes reached", maxSize)})

1075 }

1076

1077 // Update mailbox before delivery, which changes uidnext.

1078 sentmb.Add(sentm.MailboxCounts())

1079 err = tx.Update(&sentmb)

1080 xcheckf(err, "updating sent mailbox for counts")

1081

1082 err = acc.DeliverMessage(log, tx, &sentm, dataFile, true, false, false, true)

1083 if err != nil {

1084 metricSubmission.WithLabelValues("storesenterror").Inc()

1085 metricked = true

1086 }