1// Package webapisrv implements the server-side of the webapi.

2package webapisrv

4// In a separate package from webapi, so webapi.Client can be used and imported

5// without including all mox internals. Documentation for the functions is in

6// ../webapi/client.go.

8import (

9 "bytes"

10 "context"

11 cryptorand "crypto/rand"

12 "encoding/base64"

13 "encoding/json"

14 "errors"

15 "fmt"

16 htmltemplate "html/template"

17 "io"

18 "log/slog"

19 "mime"

20 "mime/multipart"

21 "net"

22 "net/http"

23 "net/textproto"

24 "reflect"

25 "runtime/debug"

26 "slices"

27 "strings"

28 "time"

30 "github.com/prometheus/client_golang/prometheus"

31 "github.com/prometheus/client_golang/prometheus/promauto"

33 "github.com/mjl-/bstore"

35 "github.com/mjl-/mox/dkim"

36 "github.com/mjl-/mox/dns"

37 "github.com/mjl-/mox/message"

38 "github.com/mjl-/mox/metrics"

39 "github.com/mjl-/mox/mlog"

40 "github.com/mjl-/mox/mox-"

41 "github.com/mjl-/mox/moxio"

42 "github.com/mjl-/mox/moxvar"

43 "github.com/mjl-/mox/queue"

44 "github.com/mjl-/mox/smtp"

45 "github.com/mjl-/mox/store"

46 "github.com/mjl-/mox/webapi"

47 "github.com/mjl-/mox/webauth"

48 "github.com/mjl-/mox/webops"

49)

51var pkglog = mlog.New("webapi", nil)

53var (

54 // Similar between ../webmail/webmail.go:/metricSubmission and ../smtpserver/server.go:/metricSubmission and ../webapisrv/server.go:/metricSubmission

55 metricSubmission = promauto.NewCounterVec(

56 prometheus.CounterOpts{

57 Name: "mox_webapi_submission_total",

58 Help: "Webapi message submission results, known values (those ending with error are server errors): ok, badfrom, messagelimiterror, recipientlimiterror, queueerror, storesenterror, domaindisabled.",

59 },

60 []string{

61 "result",

62 },

63 )

64 metricServerErrors = promauto.NewCounterVec(

65 prometheus.CounterOpts{

66 Name: "mox_webapi_errors_total",

67 Help: "Webapi server errors, known values: dkimsign, submit.",

68 },

69 []string{

70 "error",

71 },

72 )

73 metricResults = promauto.NewCounterVec(

74 prometheus.CounterOpts{

75 Name: "mox_webapi_results_total",

76 Help: "HTTP webapi results by method and result.",

77 },

78 []string{"method", "result"}, // result: "badauth", "ok", or error code

79 )

80 metricDuration = promauto.NewHistogramVec(

81 prometheus.HistogramOpts{

82 Name: "mox_webapi_duration_seconds",

83 Help: "HTTP webhook call duration.",

84 Buckets: []float64{0.01, 0.05, 0.1, 0.5, 1, 5, 10, 20, 30},

85 },

86 []string{"method"},

87 )

88)

90// We pass the request to the handler so the TLS info can be used for

91// the Received header in submitted messages. Most API calls need just the

92// account name.

93type ctxKey string

95var requestInfoCtxKey ctxKey = "requestInfo"

97type requestInfo struct {

98 Log mlog.Log

99 LoginAddress string

100 Account *store.Account

101 Response http.ResponseWriter // For setting headers for non-JSON responses.

102 Request *http.Request // For Proto and TLS connection state during message submit.

103}

104

105// todo: show a curl invocation on the method pages

106

107var docsMethodTemplate = htmltemplate.Must(htmltemplate.New("method").Parse(`<!doctype html>

108 <head>

109 <meta charset="utf-8" />

110 <meta name="robots" content="noindex,nofollow" />

111 <title>Method {{ .Method }} - WebAPI - Mox</title>

112 <style>

113body, html { padding: 1em; font-size: 16px; }

114* { font-size: inherit; font-family: ubuntu, lato, sans-serif; margin: 0; padding: 0; box-sizing: border-box; }

115h1, h2, h3, h4 { margin-bottom: 1ex; }

116h1 { font-size: 1.2rem; }

117h2 { font-size: 1.1rem; }

118h3, h4 { font-size: 1rem; }

119ul { padding-left: 1rem; }

120p { margin-bottom: 1em; max-width: 50em; }

121[title] { text-decoration: underline; text-decoration-style: dotted; }

122fieldset { border: 0; }

123textarea { width: 100%; max-width: 50em; }

124 </style>

125 </head>

126 <body>

127 <h1><a href="../">WebAPI</a> - Method {{ .Method }}</h1>

128 <form id="webapicall" method="POST">

129 <fieldset id="webapifieldset">

130 <h2>Request JSON</h2>

131 <div><textarea id="webapirequest" name="request" required rows="20">{{ .Request }}</textarea></div>

132 <br/>

133 <div>

134 <button type="reset">Reset</button>

135 <button type="submit">Call</button>

136 </div>

137 <br/>

138{{ if .ReturnsBytes }}

139 <p>Method has a non-JSON response.</p>

140{{ else }}

141 <h2>Response JSON</h2>

142 <div><textarea id="webapiresponse" rows="20">{{ .Response }}</textarea></div>

143{{ end }}

144 </fieldset>

145 </form>

146 <script>

147window.addEventListener('load', () => {

148 window.webapicall.addEventListener('submit', async (e) => {

149 const stop = () => {

150 e.stopPropagation()

151 e.preventDefault()

152 }

153

154 let req

155 try {

156 req = JSON.parse(window.webapirequest.value)

157 } catch (err) {

158 window.alert('Error parsing request: ' + err.message)

159 stop()

160 return

161 }

162 if (!req) {

163 window.alert('Empty request')

164 stop()

165 return

166 }

167

168 if ({{ .ReturnsBytes }}) {

169 // Just POST to this URL.

170 return

171 }

172

173 stop()

174 // Do call ourselves, get response and put it in the response textarea.

175 window.webapifieldset.disabled = true

176 let data = new window.FormData()

177 data.append("request", window.webapirequest.value)

178 try {

179 const response = await fetch("{{ .Method }}", {body: data, method: "POST"})

180 const text = await response.text()

181 try {

182 window.webapiresponse.value = JSON.stringify(JSON.parse(text), undefined, '\t')

183 } catch (err) {

184 window.webapiresponse.value = text

185 }

186 } catch (err) {

187 window.alert('Error: ' + err.message)

188 } finally {

189 window.webapifieldset.disabled = false

190 }

191 })

192})

193 </script>

194 </body>

195</html>

196`))

197

198var docsIndex []byte

199

200func init() {

201 var methods []string

202 mt := reflect.TypeFor[webapi.Methods]()

203 n := mt.NumMethod()

204 for i := 0; i < n; i++ {

205 methods = append(methods, mt.Method(i).Name)

206 }

207 docsIndexTmpl := htmltemplate.Must(htmltemplate.New("index").Parse(`<!doctype html>

208<html>

209 <head>

210 <meta charset="utf-8" />

211 <meta name="robots" content="noindex,nofollow" />

212 <title>Webapi - Mox</title>

213 <style>

214body, html { padding: 1em; font-size: 16px; }

215* { font-size: inherit; font-family: ubuntu, lato, sans-serif; margin: 0; padding: 0; box-sizing: border-box; }

216h1, h2, h3, h4 { margin-bottom: 1ex; }

217h1 { font-size: 1.2rem; }

218h2 { font-size: 1.1rem; }

219h3, h4 { font-size: 1rem; }

220ul { padding-left: 1rem; }

221p { margin-bottom: 1em; max-width: 50em; }

222[title] { text-decoration: underline; text-decoration-style: dotted; }

223fieldset { border: 0; }

224 </style>

225 </head>

226 <body>

227 <h1>Webapi and webhooks</h1>

228 <p>The mox webapi is a simple HTTP/JSON-based API for sending messages and processing incoming messages.</p>

229 <p>Configure webhooks in mox to receive notifications about outgoing delivery event, and/or incoming deliveries of messages.</p>

230 <p>Documentation and examples:</p>

231 <p><a href="{{ .WebapiDocsURL }}">{{ .WebapiDocsURL }}</a></p>

232 <h2>Methods</h2>

233 <p>The methods below are available in this version of mox. Follow a link for an example request/response JSON, and a button to make an API call.</p>

234 <ul>

235{{ range $i, $method := .Methods }}

236 <li><a href="{{ $method }}">{{ $method }}</a></li>

237{{ end }}

238 </ul>

239 </body>

240</html>

241`))

242 webapiDocsURL := "https://pkg.go.dev/github.com/mjl-/mox@" + moxvar.VersionBare + "/webapi/"

243 webhookDocsURL := "https://pkg.go.dev/github.com/mjl-/mox@" + moxvar.VersionBare + "/webhook/"

244 indexArgs := struct {

245 WebapiDocsURL string

246 WebhookDocsURL string

247 Methods []string

248 }{webapiDocsURL, webhookDocsURL, methods}

249 var b bytes.Buffer

250 err := docsIndexTmpl.Execute(&b, indexArgs)

251 if err != nil {

252 panic("executing api docs index template: " + err.Error())

253 }

254 docsIndex = b.Bytes()

255

256 mox.NewWebapiHandler = func(maxMsgSize int64, basePath string, isForwarded bool) http.Handler {

257 return NewServer(maxMsgSize, basePath, isForwarded)

258 }

259}

260

261// NewServer returns a new http.Handler for a webapi server.

262func NewServer(maxMsgSize int64, path string, isForwarded bool) http.Handler {

263 return server{maxMsgSize, path, isForwarded}

264}

265

266// server implements the webapi methods.

267type server struct {

268 maxMsgSize int64 // Of outgoing messages.

269 path string // Path webapi is configured under, typically /webapi/, with methods at /webapi/v0/<method>.

270 isForwarded bool // Whether incoming requests are reverse-proxied. Used for getting remote IPs for rate limiting.

271}

272

273var _ webapi.Methods = server{}

274

275// ServeHTTP implements http.Handler.

276func (s server) ServeHTTP(w http.ResponseWriter, r *http.Request) {

277 log := pkglog.WithContext(r.Context()) // Take cid from webserver.

278

279 // Send requests to /webapi/ to /webapi/v0/.

280 if r.URL.Path == "/" {

281 if r.Method != "GET" {

282 http.Error(w, "405 - method not allow", http.StatusMethodNotAllowed)

283 return

284 }

285 http.Redirect(w, r, s.path+"v0/", http.StatusSeeOther)

286 return

287 }

288 // Serve short introduction and list to methods at /webapi/v0/.

289 if r.URL.Path == "/v0/" {

290 w.Header().Set("Content-Type", "text/html; charset=utf-8")

291 w.Write(docsIndex)

292 return

293 }

294

295 // Anything else must be a method endpoint.

296 if !strings.HasPrefix(r.URL.Path, "/v0/") {

297 http.NotFound(w, r)

298 return

299 }

300 fn := r.URL.Path[len("/v0/"):]

301 log = log.With(slog.String("method", fn))

302 rfn := reflect.ValueOf(s).MethodByName(fn)

303 var zero reflect.Value

304 if rfn == zero || rfn.Type().NumIn() != 2 || rfn.Type().NumOut() != 2 {

305 log.Debug("unknown webapi method")

306 http.NotFound(w, r)

307 return

308 }

309

310 // GET on method returns an example request JSON, a button to call the method,

311 // which either fills a textarea with the response (in case of JSON) or posts to

312 // the URL letting the browser handle the response (e.g. raw message or part).

313 if r.Method == "GET" {

314 formatJSON := func(v any) (string, error) {

315 var b bytes.Buffer

316 enc := json.NewEncoder(&b)

317 enc.SetIndent("", "\t")

318 enc.SetEscapeHTML(false)

319 err := enc.Encode(v)

320 return string(b.String()), err

321 }

322

323 req, err := formatJSON(mox.FillExample(nil, reflect.New(rfn.Type().In(1))).Interface())

324 if err != nil {

325 log.Errorx("formatting request as json", err)

326 http.Error(w, "500 - internal server error - marshal request: "+err.Error(), http.StatusInternalServerError)

327 return

328 }

329 // todo: could check for io.ReadCloser, but we don't return other interfaces than that one.

330 returnsBytes := rfn.Type().Out(0).Kind() == reflect.Interface

331 var resp string

332 if !returnsBytes {

333 resp, err = formatJSON(mox.FillExample(nil, reflect.New(rfn.Type().Out(0))).Interface())

334 if err != nil {

335 log.Errorx("formatting response as json", err)

336 http.Error(w, "500 - internal server error - marshal response: "+err.Error(), http.StatusInternalServerError)

337 return

338 }

339 }

340 args := struct {

341 Method string

342 Request string

343 Response string

344 ReturnsBytes bool

345 }{fn, req, resp, returnsBytes}

346 w.Header().Set("Content-Type", "text/html; charset=utf-8")

347 err = docsMethodTemplate.Execute(w, args)

348 log.Check(err, "executing webapi method template")

349 return

350 } else if r.Method != "POST" {

351 http.Error(w, "405 - method not allowed - use get or post", http.StatusMethodNotAllowed)

352 return

353 }

354

355 // Account is available during call, but we close it before we start writing a

356 // response, to prevent slow readers from holding a reference for a long time.

357 var acc *store.Account

358 closeAccount := func() {

359 if acc != nil {

360 err := acc.Close()

361 log.Check(err, "closing account")

362 acc = nil

363 }

364 }

365 defer closeAccount()

366

367 email, password, aok := r.BasicAuth()

368 if !aok {

369 metricResults.WithLabelValues(fn, "badauth").Inc()

370 log.Debug("missing http basic authentication credentials")

371 w.Header().Set("WWW-Authenticate", "Basic realm=webapi")

372 http.Error(w, "401 - unauthorized - use http basic auth with email address as username", http.StatusUnauthorized)

373 return

374 }

375 log = log.With(slog.String("username", email))

376

377 t0 := time.Now()

378

379 // If remote IP/network resulted in too many authentication failures, refuse to serve.

380 remoteIP := webauth.RemoteIP(log, s.isForwarded, r)

381 if remoteIP == nil {

382 metricResults.WithLabelValues(fn, "internal").Inc()

383 log.Debug("cannot find remote ip for rate limiter")

384 http.Error(w, "500 - internal server error - cannot find remote ip", http.StatusInternalServerError)

385 return

386 }

387 if !mox.LimiterFailedAuth.CanAdd(remoteIP, t0, 1) {

388 metrics.AuthenticationRatelimitedInc("webapi")

389 log.Debug("refusing connection due to many auth failures", slog.Any("remoteip", remoteIP))

390 http.Error(w, "429 - too many auth attempts", http.StatusTooManyRequests)

391 return

392 }

393

394 writeError := func(err webapi.Error) {

395 closeAccount()

396 metricResults.WithLabelValues(fn, err.Code).Inc()

397

398 if err.Code == "server" {

399 log.Errorx("webapi call result", err, slog.String("resultcode", err.Code))

400 } else {

401 log.Infox("webapi call result", err, slog.String("resultcode", err.Code))

402 }

403

404 w.Header().Set("Content-Type", "application/json; charset=utf-8")

405 w.WriteHeader(http.StatusBadRequest)

406 enc := json.NewEncoder(w)

407 enc.SetEscapeHTML(false)

408 werr := enc.Encode(err)

409 if werr != nil && !moxio.IsClosed(werr) {

410 log.Infox("writing error response", werr)

411 }

412 }

413

414 // Called for all successful JSON responses, not non-JSON responses.

415 writeResponse := func(resp any) {

416 closeAccount()

417 metricResults.WithLabelValues(fn, "ok").Inc()

418 log.Debug("webapi call result", slog.String("resultcode", "ok"))

419 w.Header().Set("Content-Type", "application/json; charset=utf-8")

420 enc := json.NewEncoder(w)

421 enc.SetEscapeHTML(false)

422 werr := enc.Encode(resp)

423 if werr != nil && !moxio.IsClosed(werr) {

424 log.Infox("writing error response", werr)

425 }

426 }

427

428 la := loginAttempt(r, "webapi", "httpbasic")

429 la.LoginAddress = email

430 defer func() {

431 store.LoginAttemptAdd(context.Background(), log, la)

432 metricDuration.WithLabelValues(fn).Observe(float64(time.Since(t0)) / float64(time.Second))

433 }()

434

435 var err error

436 acc, la.AccountName, err = store.OpenEmailAuth(log, email, password, true)

437 if err != nil {

438 mox.LimiterFailedAuth.Add(remoteIP, t0, 1)

439 if errors.Is(err, mox.ErrDomainNotFound) || errors.Is(err, mox.ErrAddressNotFound) || errors.Is(err, store.ErrUnknownCredentials) || errors.Is(err, store.ErrLoginDisabled) {

440 log.Debug("bad http basic authentication credentials")

441 metricResults.WithLabelValues(fn, "badauth").Inc()

442 la.Result = store.AuthBadCredentials

443 msg := "use http basic auth with email address as username"

444 if errors.Is(err, store.ErrLoginDisabled) {

445 la.Result = store.AuthLoginDisabled

446 msg = "login is disabled for this account"

447 }

448 w.Header().Set("WWW-Authenticate", "Basic realm=webapi")

449 http.Error(w, "401 - unauthorized - "+msg, http.StatusUnauthorized)

450 return

451 }

452 writeError(webapi.Error{Code: "server", Message: "error verifying credentials"})

453 return

454 }

455 la.AccountName = acc.Name

456 la.Result = store.AuthSuccess

457 mox.LimiterFailedAuth.Reset(remoteIP, t0)

458

459 ct := r.Header.Get("Content-Type")

460 ct, _, err = mime.ParseMediaType(ct)

461 if err != nil {

462 writeError(webapi.Error{Code: "protocol", Message: "unknown content-type " + r.Header.Get("Content-Type")})

463 return

464 }

465 if ct == "multipart/form-data" {

466 err = r.ParseMultipartForm(200 * 1024)

467 } else {

468 err = r.ParseForm()

469 }

470 if err != nil {

471 writeError(webapi.Error{Code: "protocol", Message: "parsing form: " + err.Error()})

472 return

473 }

474

475 reqstr := r.PostFormValue("request")

476 if reqstr == "" {

477 writeError(webapi.Error{Code: "protocol", Message: "missing/empty request"})

478 return

479 }

480

481 defer func() {

482 x := recover()

483 if x == nil {

484 return

485 }

486 if err, eok := x.(webapi.Error); eok {

487 writeError(err)

488 return

489 }

490 log.Error("unhandled panic in webapi call", slog.Any("x", x), slog.String("resultcode", "server"))

491 metrics.PanicInc(metrics.Webapi)

492 debug.PrintStack()

493 writeError(webapi.Error{Code: "server", Message: "unhandled error"})

494 }()

495 req := reflect.New(rfn.Type().In(1))

496 dec := json.NewDecoder(strings.NewReader(reqstr))

497 dec.DisallowUnknownFields()

498 if err := dec.Decode(req.Interface()); err != nil {

499 writeError(webapi.Error{Code: "protocol", Message: fmt.Sprintf("parsing request: %s", err)})

500 return

501 }

502

503 reqInfo := requestInfo{log, email, acc, w, r}

504 nctx := context.WithValue(r.Context(), requestInfoCtxKey, reqInfo)

505 resp := rfn.Call([]reflect.Value{reflect.ValueOf(nctx), req.Elem()})

506 if !resp[1].IsZero() {

507 var e webapi.Error

508 err := resp[1].Interface().(error)

509 if x, eok := err.(webapi.Error); eok {

510 e = x

511 } else {

512 e = webapi.Error{Code: "error", Message: err.Error()}

513 }

514 writeError(e)

515 return

516 }

517 rc, ok := resp[0].Interface().(io.ReadCloser)

518 if !ok {

519 rv, _ := mox.FillNil(resp[0])

520 writeResponse(rv.Interface())

521 return

522 }

523 closeAccount()

524 log.Debug("webapi call result", slog.String("resultcode", "ok"))

525 metricResults.WithLabelValues(fn, "ok").Inc()

526 defer rc.Close()

527 if _, err := io.Copy(w, rc); err != nil && !moxio.IsClosed(err) {

528 log.Errorx("writing response to client", err)

529 }

530}

531

532// loginAttempt initializes a store.LoginAttempt, for adding to the store after

533// filling in the results and other details.

534func loginAttempt(r *http.Request, protocol, authMech string) store.LoginAttempt {

535 remoteIP, _, _ := net.SplitHostPort(r.RemoteAddr)

536 if remoteIP == "" {

537 remoteIP = r.RemoteAddr

538 }

539

540 return store.LoginAttempt{

541 RemoteIP: remoteIP,

542 TLS: store.LoginAttemptTLS(r.TLS),

543 Protocol: protocol,

544 AuthMech: authMech,

545 UserAgent: r.UserAgent(),

546 Result: store.AuthError, // Replaced by caller.

547 }

548}

549

550func xcheckf(err error, format string, args ...any) {

551 if err != nil {

552 msg := fmt.Sprintf(format, args...)

553 panic(webapi.Error{Code: "server", Message: fmt.Sprintf("%s: %s", msg, err)})

554 }

555}

556

557func xcheckuserf(err error, format string, args ...any) {

558 if err != nil {

559 msg := fmt.Sprintf(format, args...)

560 panic(webapi.Error{Code: "user", Message: fmt.Sprintf("%s: %s", msg, err)})

561 }

562}

563

564func xdbwrite(ctx context.Context, acc *store.Account, fn func(tx *bstore.Tx)) {

565 err := acc.DB.Write(ctx, func(tx *bstore.Tx) error {

566 fn(tx)

567 return nil

568 })

569 xcheckf(err, "transaction")

570}

571

572func xdbread(ctx context.Context, acc *store.Account, fn func(tx *bstore.Tx)) {

573 err := acc.DB.Read(ctx, func(tx *bstore.Tx) error {

574 fn(tx)

575 return nil

576 })

577 xcheckf(err, "transaction")

578}

579

580func xcheckcontrol(s string) {

581 for _, c := range s {

582 if c < 0x20 {

583 xcheckuserf(errors.New("control characters not allowed"), "checking header values")

584 }

585 }

586}

587

588func xparseAddress(addr string) smtp.Address {

589 a, err := smtp.ParseAddress(addr)

590 if err != nil {

591 panic(webapi.Error{Code: "badAddress", Message: fmt.Sprintf("parsing address %q: %s", addr, err)})

592 }

593 return a

594}

595

596func xparseAddresses(l []webapi.NameAddress) ([]message.NameAddress, []smtp.Path) {

597 r := make([]message.NameAddress, len(l))

598 paths := make([]smtp.Path, len(l))

599 for i, a := range l {

600 xcheckcontrol(a.Name)

601 addr := xparseAddress(a.Address)

602 r[i] = message.NameAddress{DisplayName: a.Name, Address: addr}

603 paths[i] = addr.Path()

604 }

605 return r, paths

606}

607

608func xrandomID(n int) string {

609 return base64.RawURLEncoding.EncodeToString(xrandom(n))

610}

611

612func xrandom(n int) []byte {

613 buf := make([]byte, n)

614 x, err := cryptorand.Read(buf)

615 if err != nil {

616 panic("read random")

617 } else if x != n {

618 panic("short random read")

619 }

620 return buf

621}

622

623func (s server) Send(ctx context.Context, req webapi.SendRequest) (resp webapi.SendResult, err error) {

624 // Similar between ../smtpserver/server.go:/submit\( and ../webmail/api.go:/MessageSubmit\( and ../webapisrv/server.go:/Send\(

625

626 reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)

627 log := reqInfo.Log

628 acc := reqInfo.Account

629

630 m := req.Message

631

632 accConf, _ := acc.Conf()

633

634 if m.Text == "" && m.HTML == "" {

635 return resp, webapi.Error{Code: "missingBody", Message: "at least text or html body required"}

636 }

637

638 if len(m.From) == 0 {

639 m.From = []webapi.NameAddress{{Name: accConf.FullName, Address: reqInfo.LoginAddress}}

640 } else if len(m.From) > 1 {

641 return resp, webapi.Error{Code: "multipleFrom", Message: "multiple from-addresses not allowed"}

642 }

643 froms, fromPaths := xparseAddresses(m.From)

644 from, fromPath := froms[0], fromPaths[0]

645 to, toPaths := xparseAddresses(m.To)

646 cc, ccPaths := xparseAddresses(m.CC)

647 bcc, bccPaths := xparseAddresses(m.BCC)

648

649 recipients := append(append(toPaths, ccPaths...), bccPaths...)

650 addresses := append(append(m.To, m.CC...), m.BCC...)

651

652 // Check if from address is allowed for account.

653 if ok, disabled := mox.AllowMsgFrom(acc.Name, from.Address); disabled {

654 metricSubmission.WithLabelValues("domaindisabled").Inc()

655 return resp, webapi.Error{Code: "domainDisabled", Message: "domain of from-address is temporarily disabled"}

656 } else if !ok {

657 metricSubmission.WithLabelValues("badfrom").Inc()

658 return resp, webapi.Error{Code: "badFrom", Message: "from-address not configured for account"}

659 }

660

661 if len(recipients) == 0 {

662 return resp, webapi.Error{Code: "noRecipients", Message: "no recipients"}

663 }

664

665 // Check outgoing message rate limit.

666 xdbread(ctx, acc, func(tx *bstore.Tx) {

667 msglimit, rcptlimit, err := acc.SendLimitReached(tx, recipients)

668 if msglimit >= 0 {

669 metricSubmission.WithLabelValues("messagelimiterror").Inc()

670 panic(webapi.Error{Code: "messageLimitReached", Message: "outgoing message rate limit reached"})

671 } else if rcptlimit >= 0 {

672 metricSubmission.WithLabelValues("recipientlimiterror").Inc()

673 panic(webapi.Error{Code: "recipientLimitReached", Message: "outgoing new recipient rate limit reached"})

674 }

675 xcheckf(err, "checking send limit")

676 })

677

678 // If we have a non-ascii localpart, we will be sending with smtputf8. We'll go

679 // full utf-8 then.

680 intl := func(l []smtp.Path) bool {

681 for _, p := range l {

682 if p.Localpart.IsInternational() {

683 return true

684 }

685 }

686 return false

687 }

688 smtputf8 := intl([]smtp.Path{fromPath}) || intl(toPaths) || intl(ccPaths) || intl(bccPaths)

689

690 replyTos, replyToPaths := xparseAddresses(m.ReplyTo)

691 for _, rt := range replyToPaths {

692 if rt.Localpart.IsInternational() {

693 smtputf8 = true

694 }

695 }

696

697 // Create file to compose message into.

698 dataFile, err := store.CreateMessageTemp(log, "webapi-submit")

699 xcheckf(err, "creating temporary file for message")

700 defer store.CloseRemoveTempFile(log, dataFile, "message to submit")

701

702 // If writing to the message file fails, we abort immediately.

703 xc := message.NewComposer(dataFile, s.maxMsgSize, smtputf8)

704 defer func() {

705 x := recover()

706 if x == nil {

707 return

708 }

709 if err, ok := x.(error); ok && errors.Is(err, message.ErrMessageSize) {

710 panic(webapi.Error{Code: "messageTooLarge", Message: "message too large"})

711 } else if ok && errors.Is(err, message.ErrCompose) {

712 xcheckf(err, "making message")

713 }

714 panic(x)

715 }()

716

717 // Each queued message gets a Received header.

718 // We cannot use VIA, because there is no registered method. We would like to use

719 // it to add the ascii domain name in case of smtputf8 and IDNA host name.

720 // We don't add the IP address of the submitter. Exposing likely not desirable.

721 recvFrom := message.HeaderCommentDomain(mox.Conf.Static.HostnameDomain, smtputf8)

722 recvBy := mox.Conf.Static.HostnameDomain.XName(smtputf8)

723 recvID := mox.ReceivedID(mox.CidFromCtx(ctx))

724 recvHdrFor := func(rcptTo string) string {

725 recvHdr := &message.HeaderWriter{}

726 // For additional Received-header clauses, see:

727 // https://www.iana.org/assignments/mail-parameters/mail-parameters.xhtml#table-mail-parameters-8

728 // Note: we don't have "via" or "with", there is no registered for webmail.

729 recvHdr.Add(" ", "Received:", "from", recvFrom, "by", recvBy, "id", recvID) // ../rfc/5321:3158

730 if reqInfo.Request.TLS != nil {

731 recvHdr.Add(" ", mox.TLSReceivedComment(log, *reqInfo.Request.TLS)...)

732 }

733 recvHdr.Add(" ", "for", "<"+rcptTo+">;", time.Now().Format(message.RFC5322Z))

734 return recvHdr.String()

735 }

736

737 // Outer message headers.

738 xc.HeaderAddrs("From", []message.NameAddress{from})

739 if len(replyTos) > 0 {

740 xc.HeaderAddrs("Reply-To", replyTos)

741 }

742 xc.HeaderAddrs("To", to)

743 xc.HeaderAddrs("Cc", cc)

744 // We prepend Bcc headers to the message when adding to the Sent mailbox.

745 if m.Subject != "" {

746 xcheckcontrol(m.Subject)

747 xc.Subject(m.Subject)

748 }

749

750 var date time.Time

751 if m.Date != nil {

752 date = *m.Date

753 } else {

754 date = time.Now()

755 }

756 xc.Header("Date", date.Format(message.RFC5322Z))

757

758 if m.MessageID == "" {

759 m.MessageID = fmt.Sprintf("<%s>", mox.MessageIDGen(smtputf8))

760 } else if !strings.HasPrefix(m.MessageID, "<") || !strings.HasSuffix(m.MessageID, ">") {

761 return resp, webapi.Error{Code: "malformedMessageID", Message: "missing <> in message-id"}

762 }

763 xcheckcontrol(m.MessageID)

764 xc.Header("Message-Id", m.MessageID)

765

766 if len(m.References) > 0 {

767 for _, ref := range m.References {

768 xcheckcontrol(ref)

769 // We don't check for <>'s. If caller just puts in what they got, we don't want to

770 // reject the message.

771 }

772 xc.Header("References", strings.Join(m.References, "\r\n\t"))

773 xc.Header("In-Reply-To", m.References[len(m.References)-1])

774 }

775 xc.Header("MIME-Version", "1.0")

776

777 var haveUserAgent bool

778 for _, kv := range req.Headers {

779 xcheckcontrol(kv[0])

780 xcheckcontrol(kv[1])

781 xc.Header(kv[0], kv[1])

782 if strings.EqualFold(kv[0], "User-Agent") || strings.EqualFold(kv[0], "X-Mailer") {

783 haveUserAgent = true

784 }

785 }

786 if !haveUserAgent {

787 xc.Header("User-Agent", "mox/"+moxvar.Version)

788 }

789

790 // Whether we have additional separately alternative/inline/attached file(s).

791 mpf := reqInfo.Request.MultipartForm

792 formAlternative := mpf != nil && len(mpf.File["alternativefile"]) > 0

793 formInline := mpf != nil && len(mpf.File["inlinefile"]) > 0

794 formAttachment := mpf != nil && len(mpf.File["attachedfile"]) > 0

795

796 // MIME structure we'll build:

797 // - multipart/mixed (in case of attached files)

798 // - multipart/related (in case of inline files, we assume they are relevant both text and html part if present)

799 // - multipart/alternative (in case we have both text and html bodies)

800 // - text/plain (optional)

801 // - text/html (optional)

802 // - alternative file, ...

803 // - inline file, ...

804 // - attached file, ...

805

806 // We keep track of cur, which is where we add new parts to, whether the text or

807 // html part, or the inline or attached files.

808 var cur, mixed, related, alternative *multipart.Writer

809 xcreateMultipart := func(subtype string) *multipart.Writer {

810 mp := multipart.NewWriter(xc)

811 if cur == nil {

812 xc.Header("Content-Type", fmt.Sprintf(`multipart/%s; boundary="%s"`, subtype, mp.Boundary()))

813 xc.Line()

814 } else {

815 _, err := cur.CreatePart(textproto.MIMEHeader{"Content-Type": []string{fmt.Sprintf(`multipart/%s; boundary="%s"`, subtype, mp.Boundary())}})

816 xcheckf(err, "adding multipart")

817 }

818 return mp

819 }

820 xcreatePart := func(header textproto.MIMEHeader) io.Writer {

821 if cur == nil {

822 for k, vl := range header {

823 for _, v := range vl {

824 xc.Header(k, v)

825 }

826 }

827 xc.Line()

828 return xc

829 }

830 p, err := cur.CreatePart(header)

831 xcheckf(err, "adding part")

832 return p

833 }

834 // We create multiparts from outer structure to inner. Then for each we add its

835 // inner parts and close the multipart.

836 if len(req.AttachedFiles) > 0 || formAttachment {

837 mixed = xcreateMultipart("mixed")

838 cur = mixed

839 }

840 if len(req.InlineFiles) > 0 || formInline {

841 related = xcreateMultipart("related")

842 cur = related

843 }

844 if m.Text != "" && m.HTML != "" || len(req.AlternativeFiles) > 0 || formAlternative {

845 alternative = xcreateMultipart("alternative")

846 cur = alternative

847 }

848 if m.Text != "" {

849 textBody, ct, cte := xc.TextPart("plain", m.Text)

850 tp := xcreatePart(textproto.MIMEHeader{"Content-Type": []string{ct}, "Content-Transfer-Encoding": []string{cte}})

851 _, err := tp.Write([]byte(textBody))

852 xcheckf(err, "write text part")

853 }

854 if m.HTML != "" {

855 htmlBody, ct, cte := xc.TextPart("html", m.HTML)

856 tp := xcreatePart(textproto.MIMEHeader{"Content-Type": []string{ct}, "Content-Transfer-Encoding": []string{cte}})

857 _, err := tp.Write([]byte(htmlBody))

858 xcheckf(err, "write html part")

859 }

860

861 xaddFileBase64 := func(ct string, inline bool, filename string, cid string, base64Data string) {

862 h := textproto.MIMEHeader{}

863 disp := "attachment"

864 if inline {

865 disp = "inline"

866 }

867 cd := mime.FormatMediaType(disp, map[string]string{"filename": filename})

868

869 h.Set("Content-Type", ct)

870 h.Set("Content-Disposition", cd)

871 if cid != "" {

872 h.Set("Content-ID", cid)

873 }

874 h.Set("Content-Transfer-Encoding", "base64")

875 p := xcreatePart(h)

876

877 for len(base64Data) > 0 {

878 line := base64Data

879 n := len(line)

880 if n > 78 {

881 n = 78

882 }

883 line, base64Data = base64Data[:n], base64Data[n:]

884 _, err := p.Write([]byte(line))

885 xcheckf(err, "writing attachment")

886 _, err = p.Write([]byte("\r\n"))

887 xcheckf(err, "writing attachment")

888 }

889 }

890 xaddJSONFiles := func(l []webapi.File, inline bool) {

891 for _, f := range l {

892 if f.ContentType == "" {

893 buf, _ := io.ReadAll(io.LimitReader(base64.NewDecoder(base64.StdEncoding, strings.NewReader(f.Data)), 512))

894 f.ContentType = http.DetectContentType(buf)

895 if f.ContentType == "application/octet-stream" {

896 f.ContentType = ""

897 }

898 }

899

900 // Ensure base64 is valid, then we'll write the original string.

901 _, err := io.Copy(io.Discard, base64.NewDecoder(base64.StdEncoding, strings.NewReader(f.Data)))

902 xcheckuserf(err, "parsing attachment as base64")

903

904 xaddFileBase64(f.ContentType, inline, f.Name, f.ContentID, f.Data)

905 }

906 }

907 xaddFile := func(fh *multipart.FileHeader, inline bool) {

908 f, err := fh.Open()

909 xcheckf(err, "open uploaded file")

910 defer func() {

911 err := f.Close()

912 log.Check(err, "closing uploaded file")

913 }()

914

915 ct := fh.Header.Get("Content-Type")

916 if ct == "" {

917 buf, err := io.ReadAll(io.LimitReader(f, 512))

918 if err == nil {

919 ct = http.DetectContentType(buf)

920 }

921 _, err = f.Seek(0, 0)

922 xcheckf(err, "rewind uploaded file after content-detection")

923 if ct == "application/octet-stream" {

924 ct = ""

925 }

926 }

927

928 h := textproto.MIMEHeader{}

929 disp := "attachment"

930 if inline {

931 disp = "inline"

932 }

933 cd := mime.FormatMediaType(disp, map[string]string{"filename": fh.Filename})

934

935 if ct != "" {

936 h.Set("Content-Type", ct)

937 }

938 h.Set("Content-Disposition", cd)

939 cid := fh.Header.Get("Content-ID")

940 if cid != "" {

941 h.Set("Content-ID", cid)

942 }

943 h.Set("Content-Transfer-Encoding", "base64")

944 p := xcreatePart(h)

945 bw := moxio.Base64Writer(p)

946 _, err = io.Copy(bw, f)

947 xcheckf(err, "adding uploaded file")

948 err = bw.Close()

949 xcheckf(err, "flushing uploaded file")

950 }

951

952 cur = alternative

953 xaddJSONFiles(req.AlternativeFiles, true)

954 if mpf != nil {

955 for _, fh := range mpf.File["alternativefile"] {

956 xaddFile(fh, true)

957 }

958 }

959 if alternative != nil {

960 alternative.Close()

961 alternative = nil

962 }

963

964 cur = related

965 xaddJSONFiles(req.InlineFiles, true)

966 if mpf != nil {

967 for _, fh := range mpf.File["inlinefile"] {

968 xaddFile(fh, true)

969 }

970 }

971 if related != nil {

972 related.Close()

973 related = nil

974 }

975 cur = mixed

976 xaddJSONFiles(req.AttachedFiles, false)

977 if mpf != nil {

978 for _, fh := range mpf.File["attachedfile"] {

979 xaddFile(fh, false)

980 }

981 }

982 if mixed != nil {

983 mixed.Close()

984 mixed = nil

985 }

986 cur = nil

987 xc.Flush()

988

989 // Add DKIM-Signature headers.

990 var msgPrefix string

991 fd := from.Address.Domain

992 confDom, _ := mox.Conf.Domain(fd)

993 if confDom.Disabled {

994 xcheckuserf(mox.ErrDomainDisabled, "checking domain")

995 }

996 selectors := mox.DKIMSelectors(confDom.DKIM)

997 if len(selectors) > 0 {

998 dkimHeaders, err := dkim.Sign(ctx, log.Logger, from.Address.Localpart, fd, selectors, smtputf8, dataFile)

999 if err != nil {

1000 metricServerErrors.WithLabelValues("dkimsign").Inc()

1001 }

1002 xcheckf(err, "sign dkim")

1003

1004 msgPrefix = dkimHeaders

1005 }

1006

1007 loginAddr, err := smtp.ParseAddress(reqInfo.LoginAddress)

1008 xcheckf(err, "parsing login address")

1009 useFromID := slices.Contains(accConf.ParsedFromIDLoginAddresses, loginAddr)

1010 var localpartBase string

1011 if useFromID {

1012 if confDom.LocalpartCatchallSeparator == "" {

1013 xcheckuserf(errors.New(`localpart catchall separator must be configured for domain`), `composing unique "from" address`)

1014 }

1015 localpartBase = strings.SplitN(string(fromPath.Localpart), confDom.LocalpartCatchallSeparator, 2)[0]

1016 }

1017 fromIDs := make([]string, len(recipients))

1018 qml := make([]queue.Msg, len(recipients))

1019 now := time.Now()

1020 for i, rcpt := range recipients {

1021 fp := fromPath

1022 if useFromID {

1023 fromIDs[i] = xrandomID(16)

1024 fp.Localpart = smtp.Localpart(localpartBase + confDom.LocalpartCatchallSeparator + fromIDs[i])

1025 }

1026

1027 // Don't use per-recipient unique message prefix when multiple recipients are

1028 // present, we want to keep the message identical.

1029 var recvRcpt string

1030 if len(recipients) == 1 {

1031 recvRcpt = rcpt.XString(smtputf8)

1032 }

1033 rcptMsgPrefix := recvHdrFor(recvRcpt) + msgPrefix

1034 msgSize := int64(len(rcptMsgPrefix)) + xc.Size

1035 qm := queue.MakeMsg(fp, rcpt, xc.Has8bit, xc.SMTPUTF8, msgSize, m.MessageID, []byte(rcptMsgPrefix), req.RequireTLS, now, m.Subject)

1036 qm.FromID = fromIDs[i]

1037 qm.Extra = req.Extra

1038 if req.FutureRelease != nil {

1039 ival := time.Until(*req.FutureRelease)

1040 if ival > queue.FutureReleaseIntervalMax {

1041 xcheckuserf(fmt.Errorf("date/time can not be further than %v in the future", queue.FutureReleaseIntervalMax), "scheduling delivery")

1042 }

1043 qm.NextAttempt = *req.FutureRelease

1044 qm.FutureReleaseRequest = "until;" + req.FutureRelease.Format(time.RFC3339)

1045 // todo: possibly add a header to the message stored in the Sent mailbox to indicate it was scheduled for later delivery.

1046 }

1047 qml[i] = qm

1048 }

1049 err = queue.Add(ctx, log, acc.Name, dataFile, qml...)

1050 if err != nil {

1051 metricSubmission.WithLabelValues("queueerror").Inc()

1052 }

1053 xcheckf(err, "adding messages to the delivery queue")

1054 metricSubmission.WithLabelValues("ok").Inc()

1055

1056 if req.SaveSent {

1057 // Append message to Sent mailbox and mark original messages as answered/forwarded.

1058 acc.WithRLock(func() {

1059 var changes []store.Change

1060

1061 metricked := false

1062 defer func() {

1063 if x := recover(); x != nil {

1064 if !metricked {

1065 metricServerErrors.WithLabelValues("submit").Inc()

1066 }

1067 panic(x)

1068 }

1069 }()

1070 xdbwrite(ctx, reqInfo.Account, func(tx *bstore.Tx) {

1071 sentmb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Sent", true).Get()

1072 if err == bstore.ErrAbsent {

1073 // There is no mailbox designated as Sent mailbox, so we're done.

1074 return

1075 }

1076 xcheckf(err, "message submitted to queue, adding to Sent mailbox")

1077

1078 modseq, err := acc.NextModSeq(tx)

1079 xcheckf(err, "next modseq")

1080

1081 // If there were bcc headers, prepend those to the stored message only, before the

1082 // DKIM signature. The DKIM-signature oversigns the bcc header, so this stored message

1083 // won't validate with DKIM anymore, which is fine.

1084 if len(bcc) > 0 {

1085 var sb strings.Builder

1086 xbcc := message.NewComposer(&sb, 100*1024, smtputf8)

1087 xbcc.HeaderAddrs("Bcc", bcc)

1088 xbcc.Flush()

1089 msgPrefix = sb.String() + msgPrefix

1090 }

1091

1092 sentm := store.Message{

1093 CreateSeq: modseq,

1094 ModSeq: modseq,

1095 MailboxID: sentmb.ID,

1096 MailboxOrigID: sentmb.ID,

1097 Flags: store.Flags{Notjunk: true, Seen: true},

1098 Size: int64(len(msgPrefix)) + xc.Size,

1099 MsgPrefix: []byte(msgPrefix),

1100 }

1101

1102 if ok, maxSize, err := acc.CanAddMessageSize(tx, sentm.Size); err != nil {

1103 xcheckf(err, "checking quota")

1104 } else if !ok {

1105 panic(webapi.Error{Code: "sentOverQuota", Message: fmt.Sprintf("message was sent, but not stored in sent mailbox due to quota of total %d bytes reached", maxSize)})

1106 }

1107

1108 // Update mailbox before delivery, which changes uidnext.

1109 sentmb.Add(sentm.MailboxCounts())

1110 err = tx.Update(&sentmb)

1111 xcheckf(err, "updating sent mailbox for counts")

1112

1113 err = acc.DeliverMessage(log, tx, &sentm, dataFile, true, false, false, true)

1114 if err != nil {

1115 metricSubmission.WithLabelValues("storesenterror").Inc()

1116 metricked = true

1117 }