1// Package webapisrv implements the server-side of the webapi.

2package webapisrv

4// In a separate package from webapi, so webapi.Client can be used and imported

5// without including all mox internals. Documentation for the functions is in

6// ../webapi/client.go.

8import (

9 "bytes"

10 "context"

11 cryptorand "crypto/rand"

12 "encoding/base64"

13 "encoding/json"

14 "errors"

15 "fmt"

16 htmltemplate "html/template"

17 "io"

18 "log/slog"

19 "mime"

20 "mime/multipart"

21 "net/http"

22 "net/textproto"

23 "reflect"

24 "runtime/debug"

25 "slices"

26 "strings"

27 "time"

29 "github.com/prometheus/client_golang/prometheus"

30 "github.com/prometheus/client_golang/prometheus/promauto"

32 "github.com/mjl-/bstore"

34 "github.com/mjl-/mox/dkim"

35 "github.com/mjl-/mox/dns"

36 "github.com/mjl-/mox/message"

37 "github.com/mjl-/mox/metrics"

38 "github.com/mjl-/mox/mlog"

39 "github.com/mjl-/mox/mox-"

40 "github.com/mjl-/mox/moxio"

41 "github.com/mjl-/mox/moxvar"

42 "github.com/mjl-/mox/queue"

43 "github.com/mjl-/mox/smtp"

44 "github.com/mjl-/mox/store"

45 "github.com/mjl-/mox/webapi"

46 "github.com/mjl-/mox/webauth"

47 "github.com/mjl-/mox/webhook"

48 "github.com/mjl-/mox/webops"

49)

51var pkglog = mlog.New("webapi", nil)

53var (

54 // Similar between ../webmail/webmail.go:/metricSubmission and ../smtpserver/server.go:/metricSubmission and ../webapisrv/server.go:/metricSubmission

55 metricSubmission = promauto.NewCounterVec(

56 prometheus.CounterOpts{

57 Name: "mox_webapi_submission_total",

58 Help: "Webapi message submission results, known values (those ending with error are server errors): ok, badfrom, messagelimiterror, recipientlimiterror, queueerror, storesenterror.",

59 },

60 []string{

61 "result",

62 },

63 )

64 metricServerErrors = promauto.NewCounterVec(

65 prometheus.CounterOpts{

66 Name: "mox_webapi_errors_total",

67 Help: "Webapi server errors, known values: dkimsign, submit.",

68 },

69 []string{

70 "error",

71 },

72 )

73 metricResults = promauto.NewCounterVec(

74 prometheus.CounterOpts{

75 Name: "mox_webapi_results_total",

76 Help: "HTTP webapi results by method and result.",

77 },

78 []string{"method", "result"}, // result: "badauth", "ok", or error code

79 )

80 metricDuration = promauto.NewHistogramVec(

81 prometheus.HistogramOpts{

82 Name: "mox_webapi_duration_seconds",

83 Help: "HTTP webhook call duration.",

84 Buckets: []float64{0.01, 0.05, 0.1, 0.5, 1, 5, 10, 20, 30},

85 },

86 []string{"method"},

87 )

88)

90// We pass the request to the handler so the TLS info can be used for

91// the Received header in submitted messages. Most API calls need just the

92// account name.

93type ctxKey string

95var requestInfoCtxKey ctxKey = "requestInfo"

97type requestInfo struct {

98 Log mlog.Log

99 LoginAddress string

100 Account *store.Account

101 Response http.ResponseWriter // For setting headers for non-JSON responses.

102 Request *http.Request // For Proto and TLS connection state during message submit.

103}

104

105// todo: show a curl invocation on the method pages

106

107var docsMethodTemplate = htmltemplate.Must(htmltemplate.New("method").Parse(`<!doctype html>

108 <head>

109 <meta charset="utf-8" />

110 <meta name="robots" content="noindex,nofollow" />

111 <title>Method {{ .Method }} - WebAPI - Mox</title>

112 <style>

113body, html { padding: 1em; font-size: 16px; }

114* { font-size: inherit; font-family: ubuntu, lato, sans-serif; margin: 0; padding: 0; box-sizing: border-box; }

115h1, h2, h3, h4 { margin-bottom: 1ex; }

116h1 { font-size: 1.2rem; }

117h2 { font-size: 1.1rem; }

118h3, h4 { font-size: 1rem; }

119ul { padding-left: 1rem; }

120p { margin-bottom: 1em; max-width: 50em; }

121[title] { text-decoration: underline; text-decoration-style: dotted; }

122fieldset { border: 0; }

123textarea { width: 100%; max-width: 50em; }

124 </style>

125 </head>

126 <body>

127 <h1><a href="../">WebAPI</a> - Method {{ .Method }}</h1>

128 <form id="webapicall" method="POST">

129 <fieldset id="webapifieldset">

130 <h2>Request JSON</h2>

131 <div><textarea id="webapirequest" name="request" required rows="20">{{ .Request }}</textarea></div>

132 <br/>

133 <div>

134 <button type="reset">Reset</button>

135 <button type="submit">Call</button>

136 </div>

137 <br/>

138{{ if .ReturnsBytes }}

139 <p>Method has a non-JSON response.</p>

140{{ else }}

141 <h2>Response JSON</h2>

142 <div><textarea id="webapiresponse" rows="20">{{ .Response }}</textarea></div>

143{{ end }}

144 </fieldset>

145 </form>

146 <script>

147window.addEventListener('load', () => {

148 window.webapicall.addEventListener('submit', async (e) => {

149 const stop = () => {

150 e.stopPropagation()

151 e.preventDefault()

152 }

153

154 let req

155 try {

156 req = JSON.parse(window.webapirequest.value)

157 } catch (err) {

158 window.alert('Error parsing request: ' + err.message)

159 stop()

160 return

161 }

162 if (!req) {

163 window.alert('Empty request')

164 stop()

165 return

166 }

167

168 if ({{ .ReturnsBytes }}) {

169 // Just POST to this URL.

170 return

171 }

172

173 stop()

174 // Do call ourselves, get response and put it in the response textarea.

175 window.webapifieldset.disabled = true

176 let data = new window.FormData()

177 data.append("request", window.webapirequest.value)

178 try {

179 const response = await fetch("{{ .Method }}", {body: data, method: "POST"})

180 const text = await response.text()

181 try {

182 window.webapiresponse.value = JSON.stringify(JSON.parse(text), undefined, '\t')

183 } catch (err) {

184 window.webapiresponse.value = text

185 }

186 } catch (err) {

187 window.alert('Error: ' + err.message)

188 } finally {

189 window.webapifieldset.disabled = false

190 }

191 })

192})

193 </script>

194 </body>

195</html>

196`))

197

198var docsIndex []byte

199

200func init() {

201 var methods []string

202 mt := reflect.TypeOf((*webapi.Methods)(nil)).Elem()

203 n := mt.NumMethod()

204 for i := 0; i < n; i++ {

205 methods = append(methods, mt.Method(i).Name)

206 }

207 docsIndexTmpl := htmltemplate.Must(htmltemplate.New("index").Parse(`<!doctype html>

208<html>

209 <head>

210 <meta charset="utf-8" />

211 <meta name="robots" content="noindex,nofollow" />

212 <title>Webapi - Mox</title>

213 <style>

214body, html { padding: 1em; font-size: 16px; }

215* { font-size: inherit; font-family: ubuntu, lato, sans-serif; margin: 0; padding: 0; box-sizing: border-box; }

216h1, h2, h3, h4 { margin-bottom: 1ex; }

217h1 { font-size: 1.2rem; }

218h2 { font-size: 1.1rem; }

219h3, h4 { font-size: 1rem; }

220ul { padding-left: 1rem; }

221p { margin-bottom: 1em; max-width: 50em; }

222[title] { text-decoration: underline; text-decoration-style: dotted; }

223fieldset { border: 0; }

224 </style>

225 </head>

226 <body>

227 <h1>Webapi and webhooks</h1>

228 <p>The mox webapi is a simple HTTP/JSON-based API for sending messages and processing incoming messages.</p>

229 <p>Configure webhooks in mox to receive notifications about outgoing delivery event, and/or incoming deliveries of messages.</p>

230 <p>Documentation and examples:</p>

231 <p><a href="{{ .WebapiDocsURL }}">{{ .WebapiDocsURL }}</a></p>

232 <h2>Methods</h2>

233 <p>The methods below are available in this version of mox. Follow a link for an example request/response JSON, and a button to make an API call.</p>

234 <ul>

235{{ range $i, $method := .Methods }}

236 <li><a href="{{ $method }}">{{ $method }}</a></li>

237{{ end }}

238 </ul>

239 </body>

240</html>

241`))

242 webapiDocsURL := "https://pkg.go.dev/github.com/mjl-/mox@" + moxvar.VersionBare + "/webapi/"

243 webhookDocsURL := "https://pkg.go.dev/github.com/mjl-/mox@" + moxvar.VersionBare + "/webhook/"

244 indexArgs := struct {

245 WebapiDocsURL string

246 WebhookDocsURL string

247 Methods []string

248 }{webapiDocsURL, webhookDocsURL, methods}

249 var b bytes.Buffer

250 err := docsIndexTmpl.Execute(&b, indexArgs)

251 if err != nil {

252 panic("executing api docs index template: " + err.Error())

253 }

254 docsIndex = b.Bytes()

255}

256

257// NewServer returns a new http.Handler for a webapi server.

258func NewServer(maxMsgSize int64, path string, isForwarded bool) http.Handler {

259 return server{maxMsgSize, path, isForwarded}

260}

261

262// server implements the webapi methods.

263type server struct {

264 maxMsgSize int64 // Of outgoing messages.

265 path string // Path webapi is configured under, typically /webapi/, with methods at /webapi/v0/<method>.

266 isForwarded bool // Whether incoming requests are reverse-proxied. Used for getting remote IPs for rate limiting.

267}

268

269var _ webapi.Methods = server{}

270

271// ServeHTTP implements http.Handler.

272func (s server) ServeHTTP(w http.ResponseWriter, r *http.Request) {

273 log := pkglog.WithContext(r.Context()) // Take cid from webserver.

274

275 // Send requests to /webapi/ to /webapi/v0/.

276 if r.URL.Path == "/" {

277 if r.Method != "GET" {

278 http.Error(w, "405 - method not allow", http.StatusMethodNotAllowed)

279 return

280 }

281 http.Redirect(w, r, s.path+"v0/", http.StatusSeeOther)

282 return

283 }

284 // Serve short introduction and list to methods at /webapi/v0/.

285 if r.URL.Path == "/v0/" {

286 w.Header().Set("Content-Type", "text/html; charset=utf-8")

287 w.Write(docsIndex)

288 return

289 }

290

291 // Anything else must be a method endpoint.

292 if !strings.HasPrefix(r.URL.Path, "/v0/") {

293 http.NotFound(w, r)

294 return

295 }

296 fn := r.URL.Path[len("/v0/"):]

297 log = log.With(slog.String("method", fn))

298 rfn := reflect.ValueOf(s).MethodByName(fn)

299 var zero reflect.Value

300 if rfn == zero || rfn.Type().NumIn() != 2 || rfn.Type().NumOut() != 2 {

301 log.Debug("unknown webapi method")

302 http.NotFound(w, r)

303 return

304 }

305

306 // GET on method returns an example request JSON, a button to call the method,

307 // which either fills a textarea with the response (in case of JSON) or posts to

308 // the URL letting the browser handle the response (e.g. raw message or part).

309 if r.Method == "GET" {

310 formatJSON := func(v any) (string, error) {

311 var b bytes.Buffer

312 enc := json.NewEncoder(&b)

313 enc.SetIndent("", "\t")

314 enc.SetEscapeHTML(false)

315 err := enc.Encode(v)

316 return string(b.String()), err

317 }

318

319 req, err := formatJSON(mox.FillExample(nil, reflect.New(rfn.Type().In(1))).Interface())

320 if err != nil {

321 log.Errorx("formatting request as json", err)

322 http.Error(w, "500 - internal server error - marshal request: "+err.Error(), http.StatusInternalServerError)

323 return

324 }

325 // todo: could check for io.ReadCloser, but we don't return other interfaces than that one.

326 returnsBytes := rfn.Type().Out(0).Kind() == reflect.Interface

327 var resp string

328 if !returnsBytes {

329 resp, err = formatJSON(mox.FillExample(nil, reflect.New(rfn.Type().Out(0))).Interface())

330 if err != nil {

331 log.Errorx("formatting response as json", err)

332 http.Error(w, "500 - internal server error - marshal response: "+err.Error(), http.StatusInternalServerError)

333 return

334 }

335 }

336 args := struct {

337 Method string

338 Request string

339 Response string

340 ReturnsBytes bool

341 }{fn, req, resp, returnsBytes}

342 w.Header().Set("Content-Type", "text/html; charset=utf-8")

343 err = docsMethodTemplate.Execute(w, args)

344 log.Check(err, "executing webapi method template")

345 return

346 } else if r.Method != "POST" {

347 http.Error(w, "405 - method not allowed - use get or post", http.StatusMethodNotAllowed)

348 return

349 }

350

351 // Account is available during call, but we close it before we start writing a

352 // response, to prevent slow readers from holding a reference for a long time.

353 var acc *store.Account

354 closeAccount := func() {

355 if acc != nil {

356 err := acc.Close()

357 log.Check(err, "closing account")

358 acc = nil

359 }

360 }

361 defer closeAccount()

362

363 email, password, aok := r.BasicAuth()

364 if !aok {

365 metricResults.WithLabelValues(fn, "badauth").Inc()

366 log.Debug("missing http basic authentication credentials")

367 w.Header().Set("WWW-Authenticate", "Basic realm=webapi")

368 http.Error(w, "401 - unauthorized - use http basic auth with email address as username", http.StatusUnauthorized)

369 return

370 }

371 log = log.With(slog.String("username", email))

372

373 t0 := time.Now()

374

375 // If remote IP/network resulted in too many authentication failures, refuse to serve.

376 remoteIP := webauth.RemoteIP(log, s.isForwarded, r)

377 if remoteIP == nil {

378 metricResults.WithLabelValues(fn, "internal").Inc()

379 log.Debug("cannot find remote ip for rate limiter")

380 http.Error(w, "500 - internal server error - cannot find remote ip", http.StatusInternalServerError)

381 return

382 }

383 if !mox.LimiterFailedAuth.CanAdd(remoteIP, t0, 1) {

384 metrics.AuthenticationRatelimitedInc("webapi")

385 log.Debug("refusing connection due to many auth failures", slog.Any("remoteip", remoteIP))

386 http.Error(w, "429 - too many auth attempts", http.StatusTooManyRequests)

387 return

388 }

389

390 writeError := func(err webapi.Error) {

391 closeAccount()

392 metricResults.WithLabelValues(fn, err.Code).Inc()

393

394 if err.Code == "server" {

395 log.Errorx("webapi call result", err, slog.String("resultcode", err.Code))

396 } else {

397 log.Infox("webapi call result", err, slog.String("resultcode", err.Code))

398 }

399

400 w.Header().Set("Content-Type", "application/json; charset=utf-8")

401 w.WriteHeader(http.StatusBadRequest)

402 enc := json.NewEncoder(w)

403 enc.SetEscapeHTML(false)

404 werr := enc.Encode(err)

405 if werr != nil && !moxio.IsClosed(werr) {

406 log.Infox("writing error response", werr)

407 }

408 }

409

410 // Called for all successful JSON responses, not non-JSON responses.

411 writeResponse := func(resp any) {

412 closeAccount()

413 metricResults.WithLabelValues(fn, "ok").Inc()

414 log.Debug("webapi call result", slog.String("resultcode", "ok"))

415 w.Header().Set("Content-Type", "application/json; charset=utf-8")

416 enc := json.NewEncoder(w)

417 enc.SetEscapeHTML(false)

418 werr := enc.Encode(resp)

419 if werr != nil && !moxio.IsClosed(werr) {

420 log.Infox("writing error response", werr)

421 }

422 }

423

424 authResult := "error"

425 defer func() {

426 metricDuration.WithLabelValues(fn).Observe(float64(time.Since(t0)) / float64(time.Second))

427 metrics.AuthenticationInc("webapi", "httpbasic", authResult)

428 }()

429

430 var err error

431 acc, err = store.OpenEmailAuth(log, email, password)

432 if err != nil {

433 mox.LimiterFailedAuth.Add(remoteIP, t0, 1)

434 if errors.Is(err, mox.ErrDomainNotFound) || errors.Is(err, mox.ErrAddressNotFound) || errors.Is(err, store.ErrUnknownCredentials) {

435 log.Debug("bad http basic authentication credentials")

436 metricResults.WithLabelValues(fn, "badauth").Inc()

437 authResult = "badcreds"

438 w.Header().Set("WWW-Authenticate", "Basic realm=webapi")

439 http.Error(w, "401 - unauthorized - use http basic auth with email address as username", http.StatusUnauthorized)

440 return

441 }

442 writeError(webapi.Error{Code: "server", Message: "error verifying credentials"})

443 return

444 }

445 authResult = "ok"

446 mox.LimiterFailedAuth.Reset(remoteIP, t0)

447

448 ct := r.Header.Get("Content-Type")

449 ct, _, err = mime.ParseMediaType(ct)

450 if err != nil {

451 writeError(webapi.Error{Code: "protocol", Message: "unknown content-type " + r.Header.Get("Content-Type")})

452 return

453 }

454 if ct == "multipart/form-data" {

455 err = r.ParseMultipartForm(200 * 1024)

456 } else {

457 err = r.ParseForm()

458 }

459 if err != nil {

460 writeError(webapi.Error{Code: "protocol", Message: "parsing form: " + err.Error()})

461 return

462 }

463

464 reqstr := r.PostFormValue("request")

465 if reqstr == "" {

466 writeError(webapi.Error{Code: "protocol", Message: "missing/empty request"})

467 return

468 }

469

470 defer func() {

471 x := recover()

472 if x == nil {

473 return

474 }

475 if err, eok := x.(webapi.Error); eok {

476 writeError(err)

477 return

478 }

479 log.Error("unhandled panic in webapi call", slog.Any("x", x), slog.String("resultcode", "server"))

480 metrics.PanicInc(metrics.Webapi)

481 debug.PrintStack()

482 writeError(webapi.Error{Code: "server", Message: "unhandled error"})

483 }()

484 req := reflect.New(rfn.Type().In(1))

485 dec := json.NewDecoder(strings.NewReader(reqstr))

486 dec.DisallowUnknownFields()

487 if err := dec.Decode(req.Interface()); err != nil {

488 writeError(webapi.Error{Code: "protocol", Message: fmt.Sprintf("parsing request: %s", err)})

489 return

490 }

491

492 reqInfo := requestInfo{log, email, acc, w, r}

493 nctx := context.WithValue(r.Context(), requestInfoCtxKey, reqInfo)

494 resp := rfn.Call([]reflect.Value{reflect.ValueOf(nctx), req.Elem()})

495 if !resp[1].IsZero() {

496 var e webapi.Error

497 err := resp[1].Interface().(error)

498 if x, eok := err.(webapi.Error); eok {

499 e = x

500 } else {

501 e = webapi.Error{Code: "error", Message: err.Error()}

502 }

503 writeError(e)

504 return

505 }

506 rc, ok := resp[0].Interface().(io.ReadCloser)

507 if !ok {

508 rv, _ := mox.FillNil(resp[0])

509 writeResponse(rv.Interface())

510 return

511 }

512 closeAccount()

513 log.Debug("webapi call result", slog.String("resultcode", "ok"))

514 metricResults.WithLabelValues(fn, "ok").Inc()

515 defer rc.Close()

516 if _, err := io.Copy(w, rc); err != nil && !moxio.IsClosed(err) {

517 log.Errorx("writing response to client", err)

518 }

519}

520

521func xcheckf(err error, format string, args ...any) {

522 if err != nil {

523 msg := fmt.Sprintf(format, args...)

524 panic(webapi.Error{Code: "server", Message: fmt.Sprintf("%s: %s", msg, err)})

525 }

526}

527

528func xcheckuserf(err error, format string, args ...any) {

529 if err != nil {

530 msg := fmt.Sprintf(format, args...)

531 panic(webapi.Error{Code: "user", Message: fmt.Sprintf("%s: %s", msg, err)})

532 }

533}

534

535func xdbwrite(ctx context.Context, acc *store.Account, fn func(tx *bstore.Tx)) {

536 err := acc.DB.Write(ctx, func(tx *bstore.Tx) error {

537 fn(tx)

538 return nil

539 })

540 xcheckf(err, "transaction")

541}

542

543func xdbread(ctx context.Context, acc *store.Account, fn func(tx *bstore.Tx)) {

544 err := acc.DB.Read(ctx, func(tx *bstore.Tx) error {

545 fn(tx)

546 return nil

547 })

548 xcheckf(err, "transaction")

549}

550

551func xcheckcontrol(s string) {

552 for _, c := range s {

553 if c < 0x20 {

554 xcheckuserf(errors.New("control characters not allowed"), "checking header values")

555 }

556 }

557}

558

559func xparseAddress(addr string) smtp.Address {

560 a, err := smtp.ParseAddress(addr)

561 if err != nil {

562 panic(webapi.Error{Code: "badAddress", Message: fmt.Sprintf("parsing address %q: %s", addr, err)})

563 }

564 return a

565}

566

567func xparseAddresses(l []webapi.NameAddress) ([]message.NameAddress, []smtp.Path) {

568 r := make([]message.NameAddress, len(l))

569 paths := make([]smtp.Path, len(l))

570 for i, a := range l {

571 xcheckcontrol(a.Name)

572 addr := xparseAddress(a.Address)

573 r[i] = message.NameAddress{DisplayName: a.Name, Address: addr}

574 paths[i] = addr.Path()

575 }

576 return r, paths

577}

578

579func xrandomID(n int) string {

580 return base64.RawURLEncoding.EncodeToString(xrandom(n))

581}

582

583func xrandom(n int) []byte {

584 buf := make([]byte, n)

585 x, err := cryptorand.Read(buf)

586 if err != nil {

587 panic("read random")

588 } else if x != n {

589 panic("short random read")

590 }

591 return buf

592}

593

594func (s server) Send(ctx context.Context, req webapi.SendRequest) (resp webapi.SendResult, err error) {

595 // Similar between ../smtpserver/server.go:/submit\( and ../webmail/api.go:/MessageSubmit\( and ../webapisrv/server.go:/Send\(

596

597 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

598 log := reqInfo.Log

599 acc := reqInfo.Account

600

601 m := req.Message

602

603 accConf, _ := acc.Conf()

604

605 if m.Text == "" && m.HTML == "" {

606 return resp, webapi.Error{Code: "missingBody", Message: "at least text or html body required"}

607 }

608

609 if len(m.From) == 0 {

610 m.From = []webapi.NameAddress{{Name: accConf.FullName, Address: reqInfo.LoginAddress}}

611 } else if len(m.From) > 1 {

612 return resp, webapi.Error{Code: "multipleFrom", Message: "multiple from-addresses not allowed"}

613 }

614 froms, fromPaths := xparseAddresses(m.From)

615 from, fromPath := froms[0], fromPaths[0]

616 to, toPaths := xparseAddresses(m.To)

617 cc, ccPaths := xparseAddresses(m.CC)

618 bcc, bccPaths := xparseAddresses(m.BCC)

619

620 recipients := append(append(toPaths, ccPaths...), bccPaths...)

621 addresses := append(append(m.To, m.CC...), m.BCC...)

622

623 // Check if from address is allowed for account.

624 if !mox.AllowMsgFrom(acc.Name, from.Address) {

625 metricSubmission.WithLabelValues("badfrom").Inc()

626 return resp, webapi.Error{Code: "badFrom", Message: "from-address not configured for account"}

627 }

628

629 if len(recipients) == 0 {

630 return resp, webapi.Error{Code: "noRecipients", Message: "no recipients"}

631 }

632

633 // Check outgoing message rate limit.

634 xdbread(ctx, acc, func(tx *bstore.Tx) {

635 msglimit, rcptlimit, err := acc.SendLimitReached(tx, recipients)

636 if msglimit >= 0 {

637 metricSubmission.WithLabelValues("messagelimiterror").Inc()

638 panic(webapi.Error{Code: "messageLimitReached", Message: "outgoing message rate limit reached"})

639 } else if rcptlimit >= 0 {

640 metricSubmission.WithLabelValues("recipientlimiterror").Inc()

641 panic(webapi.Error{Code: "recipientLimitReached", Message: "outgoing new recipient rate limit reached"})

642 }

643 xcheckf(err, "checking send limit")

644 })

645

646 // If we have a non-ascii localpart, we will be sending with smtputf8. We'll go

647 // full utf-8 then.

648 intl := func(l []smtp.Path) bool {

649 for _, p := range l {

650 if p.Localpart.IsInternational() {

651 return true

652 }

653 }

654 return false

655 }

656 smtputf8 := intl([]smtp.Path{fromPath}) || intl(toPaths) || intl(ccPaths) || intl(bccPaths)

657

658 replyTos, replyToPaths := xparseAddresses(m.ReplyTo)

659 for _, rt := range replyToPaths {

660 if rt.Localpart.IsInternational() {

661 smtputf8 = true

662 }

663 }

664

665 // Create file to compose message into.

666 dataFile, err := store.CreateMessageTemp(log, "webapi-submit")

667 xcheckf(err, "creating temporary file for message")

668 defer store.CloseRemoveTempFile(log, dataFile, "message to submit")

669

670 // If writing to the message file fails, we abort immediately.

671 xc := message.NewComposer(dataFile, s.maxMsgSize, smtputf8)

672 defer func() {

673 x := recover()

674 if x == nil {

675 return

676 }

677 if err, ok := x.(error); ok && errors.Is(err, message.ErrMessageSize) {

678 panic(webapi.Error{Code: "messageTooLarge", Message: "message too large"})

679 } else if ok && errors.Is(err, message.ErrCompose) {

680 xcheckf(err, "making message")

681 }

682 panic(x)

683 }()

684

685 // Each queued message gets a Received header.

686 // We cannot use VIA, because there is no registered method. We would like to use

687 // it to add the ascii domain name in case of smtputf8 and IDNA host name.

688 // We don't add the IP address of the submitter. Exposing likely not desirable.

689 recvFrom := message.HeaderCommentDomain(mox.Conf.Static.HostnameDomain, smtputf8)

690 recvBy := mox.Conf.Static.HostnameDomain.XName(smtputf8)

691 recvID := mox.ReceivedID(mox.CidFromCtx(ctx))

692 recvHdrFor := func(rcptTo string) string {

693 recvHdr := &message.HeaderWriter{}

694 // For additional Received-header clauses, see:

695 // https://www.iana.org/assignments/mail-parameters/mail-parameters.xhtml#table-mail-parameters-8

696 // Note: we don't have "via" or "with", there is no registered for webmail.

697 recvHdr.Add(" ", "Received:", "from", recvFrom, "by", recvBy, "id", recvID) // ../rfc/5321:3158

698 if reqInfo.Request.TLS != nil {

699 recvHdr.Add(" ", mox.TLSReceivedComment(log, *reqInfo.Request.TLS)...)

700 }

701 recvHdr.Add(" ", "for", "<"+rcptTo+">;", time.Now().Format(message.RFC5322Z))

702 return recvHdr.String()

703 }

704

705 // Outer message headers.

706 xc.HeaderAddrs("From", []message.NameAddress{from})

707 if len(replyTos) > 0 {

708 xc.HeaderAddrs("Reply-To", replyTos)

709 }

710 xc.HeaderAddrs("To", to)

711 xc.HeaderAddrs("Cc", cc)

712 // We prepend Bcc headers to the message when adding to the Sent mailbox.

713 if m.Subject != "" {

714 xcheckcontrol(m.Subject)

715 xc.Subject(m.Subject)

716 }

717

718 var date time.Time

719 if m.Date != nil {

720 date = *m.Date

721 } else {

722 date = time.Now()

723 }

724 xc.Header("Date", date.Format(message.RFC5322Z))

725

726 if m.MessageID == "" {

727 m.MessageID = fmt.Sprintf("<%s>", mox.MessageIDGen(smtputf8))

728 } else if !strings.HasPrefix(m.MessageID, "<") || !strings.HasSuffix(m.MessageID, ">") {

729 return resp, webapi.Error{Code: "malformedMessageID", Message: "missing <> in message-id"}

730 }

731 xcheckcontrol(m.MessageID)

732 xc.Header("Message-Id", m.MessageID)

733

734 if len(m.References) > 0 {

735 for _, ref := range m.References {

736 xcheckcontrol(ref)

737 // We don't check for <>'s. If caller just puts in what they got, we don't want to

738 // reject the message.

739 }

740 xc.Header("References", strings.Join(m.References, "\r\n\t"))

741 xc.Header("In-Reply-To", m.References[len(m.References)-1])

742 }

743 xc.Header("MIME-Version", "1.0")

744

745 var haveUserAgent bool

746 for _, kv := range req.Headers {

747 xcheckcontrol(kv[0])

748 xcheckcontrol(kv[1])

749 xc.Header(kv[0], kv[1])

750 if strings.EqualFold(kv[0], "User-Agent") || strings.EqualFold(kv[0], "X-Mailer") {

751 haveUserAgent = true

752 }

753 }

754 if !haveUserAgent {

755 xc.Header("User-Agent", "mox/"+moxvar.Version)

756 }

757

758 // Whether we have additional separately inline/attached file(s).

759 mpf := reqInfo.Request.MultipartForm

760 formInline := mpf != nil && len(mpf.File["inlinefile"]) > 0

761 formAttachment := mpf != nil && len(mpf.File["attachedfile"]) > 0

762

763 // MIME structure we'll build:

764 // - multipart/mixed (in case of attached files)

765 // - multipart/related (in case of inline files, we assume they are relevant both text and html part if present)

766 // - multipart/alternative (in case we have both text and html bodies)

767 // - text/plain (optional)

768 // - text/html (optional)

769 // - inline file, ...

770 // - attached file, ...

771

772 // We keep track of cur, which is where we add new parts to, whether the text or

773 // html part, or the inline or attached files.

774 var cur, mixed, related, alternative *multipart.Writer

775 xcreateMultipart := func(subtype string) *multipart.Writer {

776 mp := multipart.NewWriter(xc)

777 if cur == nil {

778 xc.Header("Content-Type", fmt.Sprintf(`multipart/%s; boundary="%s"`, subtype, mp.Boundary()))

779 xc.Line()

780 } else {

781 _, err := cur.CreatePart(textproto.MIMEHeader{"Content-Type": []string{fmt.Sprintf(`multipart/%s; boundary="%s"`, subtype, mp.Boundary())}})

782 xcheckf(err, "adding multipart")

783 }

784 return mp

785 }

786 xcreatePart := func(header textproto.MIMEHeader) io.Writer {

787 if cur == nil {

788 for k, vl := range header {

789 for _, v := range vl {

790 xc.Header(k, v)

791 }

792 }

793 xc.Line()

794 return xc

795 }

796 p, err := cur.CreatePart(header)

797 xcheckf(err, "adding part")

798 return p

799 }

800 // We create multiparts from outer structure to inner. Then for each we add its

801 // inner parts and close the multipart.

802 if len(req.AttachedFiles) > 0 || formAttachment {

803 mixed = xcreateMultipart("mixed")

804 cur = mixed

805 }

806 if len(req.InlineFiles) > 0 || formInline {

807 related = xcreateMultipart("related")

808 cur = related

809 }

810 if m.Text != "" && m.HTML != "" {

811 alternative = xcreateMultipart("alternative")

812 cur = alternative

813 }

814 if m.Text != "" {

815 textBody, ct, cte := xc.TextPart("plain", m.Text)

816 tp := xcreatePart(textproto.MIMEHeader{"Content-Type": []string{ct}, "Content-Transfer-Encoding": []string{cte}})

817 _, err := tp.Write([]byte(textBody))

818 xcheckf(err, "write text part")

819 }

820 if m.HTML != "" {

821 htmlBody, ct, cte := xc.TextPart("html", m.HTML)

822 tp := xcreatePart(textproto.MIMEHeader{"Content-Type": []string{ct}, "Content-Transfer-Encoding": []string{cte}})

823 _, err := tp.Write([]byte(htmlBody))

824 xcheckf(err, "write html part")

825 }

826 if alternative != nil {

827 alternative.Close()

828 alternative = nil

829 }

830

831 xaddFileBase64 := func(ct string, inline bool, filename string, cid string, base64Data string) {

832 h := textproto.MIMEHeader{}

833 disp := "attachment"

834 if inline {

835 disp = "inline"

836 }

837 cd := mime.FormatMediaType(disp, map[string]string{"filename": filename})

838

839 h.Set("Content-Type", ct)

840 h.Set("Content-Disposition", cd)

841 if cid != "" {

842 h.Set("Content-ID", cid)

843 }

844 h.Set("Content-Transfer-Encoding", "base64")

845 p := xcreatePart(h)

846

847 for len(base64Data) > 0 {

848 line := base64Data

849 n := len(line)

850 if n > 78 {

851 n = 78

852 }

853 line, base64Data = base64Data[:n], base64Data[n:]

854 _, err := p.Write([]byte(line))

855 xcheckf(err, "writing attachment")

856 _, err = p.Write([]byte("\r\n"))

857 xcheckf(err, "writing attachment")

858 }

859 }

860 xaddJSONFiles := func(l []webapi.File, inline bool) {

861 for _, f := range l {

862 if f.ContentType == "" {

863 buf, _ := io.ReadAll(io.LimitReader(base64.NewDecoder(base64.StdEncoding, strings.NewReader(f.Data)), 512))

864 f.ContentType = http.DetectContentType(buf)

865 if f.ContentType == "application/octet-stream" {

866 f.ContentType = ""

867 }

868 }

869

870 // Ensure base64 is valid, then we'll write the original string.

871 _, err := io.Copy(io.Discard, base64.NewDecoder(base64.StdEncoding, strings.NewReader(f.Data)))

872 xcheckuserf(err, "parsing attachment as base64")

873

874 xaddFileBase64(f.ContentType, inline, f.Name, f.ContentID, f.Data)

875 }

876 }

877 xaddFile := func(fh *multipart.FileHeader, inline bool) {

878 f, err := fh.Open()

879 xcheckf(err, "open uploaded file")

880 defer func() {

881 err := f.Close()

882 log.Check(err, "closing uploaded file")

883 }()

884

885 ct := fh.Header.Get("Content-Type")

886 if ct == "" {

887 buf, err := io.ReadAll(io.LimitReader(f, 512))

888 if err == nil {

889 ct = http.DetectContentType(buf)

890 }

891 _, err = f.Seek(0, 0)

892 xcheckf(err, "rewind uploaded file after content-detection")

893 if ct == "application/octet-stream" {

894 ct = ""

895 }

896 }

897

898 h := textproto.MIMEHeader{}

899 disp := "attachment"

900 if inline {

901 disp = "inline"

902 }

903 cd := mime.FormatMediaType(disp, map[string]string{"filename": fh.Filename})

904

905 if ct != "" {

906 h.Set("Content-Type", ct)

907 }

908 h.Set("Content-Disposition", cd)

909 cid := fh.Header.Get("Content-ID")

910 if cid != "" {

911 h.Set("Content-ID", cid)

912 }

913 h.Set("Content-Transfer-Encoding", "base64")

914 p := xcreatePart(h)

915 bw := moxio.Base64Writer(p)

916 _, err = io.Copy(bw, f)

917 xcheckf(err, "adding uploaded file")

918 err = bw.Close()

919 xcheckf(err, "flushing uploaded file")

920 }

921

922 cur = related

923 xaddJSONFiles(req.InlineFiles, true)

924 if mpf != nil {

925 for _, fh := range mpf.File["inlinefile"] {

926 xaddFile(fh, true)

927 }

928 }

929 if related != nil {

930 related.Close()

931 related = nil

932 }

933 cur = mixed

934 xaddJSONFiles(req.AttachedFiles, false)

935 if mpf != nil {

936 for _, fh := range mpf.File["attachedfile"] {

937 xaddFile(fh, false)

938 }

939 }

940 if mixed != nil {

941 mixed.Close()

942 mixed = nil

943 }

944 cur = nil

945 xc.Flush()

946

947 // Add DKIM-Signature headers.

948 var msgPrefix string

949 fd := from.Address.Domain

950 confDom, _ := mox.Conf.Domain(fd)

951 selectors := mox.DKIMSelectors(confDom.DKIM)

952 if len(selectors) > 0 {

953 dkimHeaders, err := dkim.Sign(ctx, log.Logger, from.Address.Localpart, fd, selectors, smtputf8, dataFile)

954 if err != nil {

955 metricServerErrors.WithLabelValues("dkimsign").Inc()

956 }

957 xcheckf(err, "sign dkim")

958

959 msgPrefix = dkimHeaders

960 }

961

962 loginAddr, err := smtp.ParseAddress(reqInfo.LoginAddress)

963 xcheckf(err, "parsing login address")

964 useFromID := slices.Contains(accConf.ParsedFromIDLoginAddresses, loginAddr)

965 var localpartBase string

966 if useFromID {

967 if confDom.LocalpartCatchallSeparator == "" {

968 xcheckuserf(errors.New(`localpart catchall separator must be configured for domain`), `composing unique "from" address`)

969 }

970 localpartBase = strings.SplitN(string(fromPath.Localpart), confDom.LocalpartCatchallSeparator, 2)[0]

971 }

972 fromIDs := make([]string, len(recipients))

973 qml := make([]queue.Msg, len(recipients))

974 now := time.Now()

975 for i, rcpt := range recipients {

976 fp := fromPath

977 if useFromID {

978 fromIDs[i] = xrandomID(16)

979 fp.Localpart = smtp.Localpart(localpartBase + confDom.LocalpartCatchallSeparator + fromIDs[i])

980 }

981

982 // Don't use per-recipient unique message prefix when multiple recipients are

983 // present, we want to keep the message identical.

984 var recvRcpt string

985 if len(recipients) == 1 {

986 recvRcpt = rcpt.XString(smtputf8)

987 }

988 rcptMsgPrefix := recvHdrFor(recvRcpt) + msgPrefix

989 msgSize := int64(len(rcptMsgPrefix)) + xc.Size

990 qm := queue.MakeMsg(fp, rcpt, xc.Has8bit, xc.SMTPUTF8, msgSize, m.MessageID, []byte(rcptMsgPrefix), req.RequireTLS, now, m.Subject)

991 qm.FromID = fromIDs[i]

992 qm.Extra = req.Extra

993 if req.FutureRelease != nil {

994 ival := time.Until(*req.FutureRelease)

995 if ival > queue.FutureReleaseIntervalMax {

996 xcheckuserf(fmt.Errorf("date/time can not be further than %v in the future", queue.FutureReleaseIntervalMax), "scheduling delivery")

997 }

998 qm.NextAttempt = *req.FutureRelease

999 qm.FutureReleaseRequest = "until;" + req.FutureRelease.Format(time.RFC3339)

1000 // todo: possibly add a header to the message stored in the Sent mailbox to indicate it was scheduled for later delivery.

1001 }

1002 qml[i] = qm

1003 }

1004 err = queue.Add(ctx, log, acc.Name, dataFile, qml...)

1005 if err != nil {

1006 metricSubmission.WithLabelValues("queueerror").Inc()

1007 }

1008 xcheckf(err, "adding messages to the delivery queue")

1009 metricSubmission.WithLabelValues("ok").Inc()

1010

1011 if req.SaveSent {

1012 // Append message to Sent mailbox and mark original messages as answered/forwarded.

1013 acc.WithRLock(func() {

1014 var changes []store.Change

1015

1016 metricked := false

1017 defer func() {

1018 if x := recover(); x != nil {

1019 if !metricked {

1020 metricServerErrors.WithLabelValues("submit").Inc()

1021 }

1022 panic(x)

1023 }

1024 }()

1025 xdbwrite(ctx, reqInfo.Account, func(tx *bstore.Tx) {

1026 sentmb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Sent", true).Get()

1027 if err == bstore.ErrAbsent {

1028 // There is no mailbox designated as Sent mailbox, so we're done.

1029 return

1030 }

1031 xcheckf(err, "message submitted to queue, adding to Sent mailbox")

1032

1033 modseq, err := acc.NextModSeq(tx)

1034 xcheckf(err, "next modseq")

1035

1036 // If there were bcc headers, prepend those to the stored message only, before the

1037 // DKIM signature. The DKIM-signature oversigns the bcc header, so this stored message

1038 // won't validate with DKIM anymore, which is fine.

1039 if len(bcc) > 0 {

1040 var sb strings.Builder

1041 xbcc := message.NewComposer(&sb, 100*1024, smtputf8)

1042 xbcc.HeaderAddrs("Bcc", bcc)

1043 xbcc.Flush()

1044 msgPrefix = sb.String() + msgPrefix

1045 }

1046

1047 sentm := store.Message{

1048 CreateSeq: modseq,

1049 ModSeq: modseq,

1050 MailboxID: sentmb.ID,

1051 MailboxOrigID: sentmb.ID,

1052 Flags: store.Flags{Notjunk: true, Seen: true},

1053 Size: int64(len(msgPrefix)) + xc.Size,

1054 MsgPrefix: []byte(msgPrefix),

1055 }

1056

1057 if ok, maxSize, err := acc.CanAddMessageSize(tx, sentm.Size); err != nil {

1058 xcheckf(err, "checking quota")

1059 } else if !ok {

1060 panic(webapi.Error{Code: "sentOverQuota", Message: fmt.Sprintf("message was sent, but not stored in sent mailbox due to quota of total %d bytes reached", maxSize)})

1061 }

1062

1063 // Update mailbox before delivery, which changes uidnext.

1064 sentmb.Add(sentm.MailboxCounts())

1065 err = tx.Update(&sentmb)

1066 xcheckf(err, "updating sent mailbox for counts")

1067

1068 err = acc.DeliverMessage(log, tx, &sentm, dataFile, true, false, false, true)

1069 if err != nil {

1070 metricSubmission.WithLabelValues("storesenterror").Inc()

1071 metricked = true

1072 }