1// Package imapserver implements an IMAPv4 server, rev2 (RFC 9051) and rev1 with extensions (RFC 3501 and more).
7IMAP4rev2 includes functionality that was in extensions for IMAP4rev1. The
8extensions sometimes include features not in IMAP4rev2. We want IMAP4rev1-only
9implementations to use extensions, so we implement the full feature set of the
10extension and announce it as capability. The extensions: LITERAL+, IDLE,
11NAMESPACE, BINARY, UNSELECT, UIDPLUS, ESEARCH, SEARCHRES, SASL-IR, ENABLE,
12LIST-EXTENDED, SPECIAL-USE, MOVE, UTF8=ONLY.
14We take a liberty with UTF8=ONLY. We are supposed to wait for ENABLE of
15UTF8=ACCEPT or IMAP4rev2 before we respond with quoted strings that contain
16non-ASCII UTF-8. Until that's enabled, we do use UTF-7 for mailbox names. See
19- We never execute multiple commands at the same time for a connection. We expect a client to open multiple connections instead.
../rfc/9051:1110
20- Do not write output on a connection with an account lock held. Writing can block, a slow client could block account operations.
21- When handling commands that modify the selected mailbox, always check that the mailbox is not opened readonly. And always revalidate the selected mailbox, another session may have deleted the mailbox.
22- After making changes to an account/mailbox/message, you must broadcast changes. You must do this with the account lock held. Otherwise, other later changes (e.g. message deliveries) may be made and broadcast before changes that were made earlier. Make sure to commit changes in the database first, because the commit may fail.
23- Mailbox hierarchies are slash separated, no leading slash. We keep the case, except INBOX is renamed to Inbox, also for submailboxes in INBOX. We don't allow existence of a child where its parent does not exist. We have no \NoInferiors or \NoSelect. Newly created mailboxes are automatically subscribed.
24- For CONDSTORE and QRESYNC support, we set "modseq" for each change/expunge. Once expunged, a modseq doesn't change anymore. We don't yet remove old expunged records. The records aren't too big. Next step may be to let an admin reclaim space manually.
28- todo: do not return binary data for a fetch body. at least not for imap4rev1. we should be encoding it as base64?
29- todo: on expunge we currently remove the message even if other sessions still have a reference to the uid. if they try to query the uid, they'll get an error. we could be nicer and only actually remove the message when the last reference has gone. we could add a new flag to store.Message marking the message as expunged, not give new session access to such messages, and make store remove them at startup, and clean them when the last session referencing the session goes. however, it will get much more complicated. renaming messages would need special handling. and should we do the same for removed mailboxes?
30- todo: try to recover from syntax errors when the last command line ends with a }, i.e. a literal. we currently abort the entire connection. we may want to read some amount of literal data and continue with a next command.
31- todo future: more extensions: OBJECTID, MULTISEARCH, REPLACE, NOTIFY, CATENATE, MULTIAPPEND, SORT, THREAD, CREATE-SPECIAL-USE.
62 "golang.org/x/exp/maps"
64 "github.com/prometheus/client_golang/prometheus"
65 "github.com/prometheus/client_golang/prometheus/promauto"
67 "github.com/mjl-/bstore"
69 "github.com/mjl-/mox/config"
70 "github.com/mjl-/mox/message"
71 "github.com/mjl-/mox/metrics"
72 "github.com/mjl-/mox/mlog"
73 "github.com/mjl-/mox/mox-"
74 "github.com/mjl-/mox/moxio"
75 "github.com/mjl-/mox/moxvar"
76 "github.com/mjl-/mox/ratelimit"
77 "github.com/mjl-/mox/scram"
78 "github.com/mjl-/mox/store"
82 metricIMAPConnection = promauto.NewCounterVec(
83 prometheus.CounterOpts{
84 Name: "mox_imap_connection_total",
85 Help: "Incoming IMAP connections.",
88 "service", // imap, imaps
91 metricIMAPCommands = promauto.NewHistogramVec(
92 prometheus.HistogramOpts{
93 Name: "mox_imap_command_duration_seconds",
94 Help: "IMAP command duration and result codes in seconds.",
95 Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20},
99 "result", // ok, panic, ioerror, badsyntax, servererror, usererror, error
104var limiterConnectionrate, limiterConnections *ratelimit.Limiter
107 // Also called by tests, so they don't trigger the rate limiter.
113 limiterConnectionrate = &ratelimit.Limiter{
114 WindowLimits: []ratelimit.WindowLimit{
117 Limits: [...]int64{300, 900, 2700},
121 limiterConnections = &ratelimit.Limiter{
122 WindowLimits: []ratelimit.WindowLimit{
124 Window: time.Duration(math.MaxInt64), // All of time.
125 Limits: [...]int64{30, 90, 270},
131// Delay after bad/suspicious behaviour. Tests set these to zero.
132var badClientDelay = time.Second // Before reads and after 1-byte writes for probably spammers.
133var authFailDelay = time.Second // After authentication failure.
135// Capabilities (extensions) the server supports. Connections will add a few more, e.g. STARTTLS, LOGINDISABLED, AUTH=PLAIN.
161// We always announce support for SCRAM PLUS-variants, also on connections without
162// TLS. The client should not be selecting PLUS variants on non-TLS connections,
163// instead opting to do the bare SCRAM variant without indicating the server claims
164// to support the PLUS variant (skipping the server downgrade detection check).
165const serverCapabilities = "IMAP4rev2 IMAP4rev1 ENABLE LITERAL+ IDLE SASL-IR BINARY UNSELECT UIDPLUS ESEARCH SEARCHRES MOVE UTF8=ACCEPT LIST-EXTENDED SPECIAL-USE LIST-STATUS AUTH=SCRAM-SHA-256-PLUS AUTH=SCRAM-SHA-256 AUTH=SCRAM-SHA-1-PLUS AUTH=SCRAM-SHA-1 AUTH=CRAM-MD5 ID APPENDLIMIT=9223372036854775807 CONDSTORE QRESYNC STATUS=SIZE QUOTA QUOTA=RES-STORAGE"
171 tls bool // Whether TLS has been initialized.
172 br *bufio.Reader // From remote, with TLS unwrapped in case of TLS.
173 line chan lineErr // If set, instead of reading from br, a line is read from this channel. For reading a line in IDLE while also waiting for mailbox/account updates.
174 lastLine string // For detecting if syntax error is fatal, i.e. if this ends with a literal. Without crlf.
175 bw *bufio.Writer // To remote, with TLS added in case of TLS.
176 tr *moxio.TraceReader // Kept to change trace level when reading/writing cmd/auth/data.
177 tw *moxio.TraceWriter
178 slow bool // If set, reads are done with a 1 second sleep, and writes are done 1 byte at a time, to keep spammers busy.
179 lastlog time.Time // For printing time since previous log line.
180 baseTLSConfig *tls.Config // Base TLS config to use for handshake.
182 noRequireSTARTTLS bool
183 cmd string // Currently executing, for deciding to applyChanges and logging.
184 cmdMetric string // Currently executing, for metrics.
186 ncmds int // Number of commands processed. Used to abort connection when first incoming command is unknown/invalid.
188 enabled map[capability]bool // All upper-case.
190 // Set by SEARCH with SAVE. Can be used by commands accepting a sequence-set with
191 // value "$". When used, UIDs must be verified to still exist, because they may
192 // have been expunged. Cleared by a SELECT or EXAMINE.
193 // Nil means no searchResult is present. An empty list is a valid searchResult,
194 // just not matching any messages.
196 searchResult []store.UID
198 // Only set when connection has been authenticated. These can be set even when
199 // c.state is stateNotAuthenticated, for TLS client certificate authentication. In
200 // that case, credentials aren't used until the authentication command with the
201 // SASL "EXTERNAL" mechanism.
202 authFailed int // Number of failed auth attempts. For slowing down remote with many failures.
203 noPreauth bool // If set, don't switch connection to "authenticated" after TLS handshake with client certificate authentication.
204 username string // Full username as used during login.
205 account *store.Account
206 comm *store.Comm // For sending/receiving changes on mailboxes in account, e.g. from messages incoming on smtp, or another imap client.
208 mailboxID int64 // Only for StateSelected.
209 readonly bool // If opened mailbox is readonly.
210 uids []store.UID // UIDs known in this session, sorted. todo future: store more space-efficiently, as ranges.
213// capability for use with ENABLED and CAPABILITY. We always keep this upper case,
214// e.g. IMAP4REV2. These values are treated case-insensitive, but it's easier for
215// comparison to just always have the same case.
216type capability string
219 capIMAP4rev2 capability = "IMAP4REV2"
220 capUTF8Accept capability = "UTF8=ACCEPT"
221 capCondstore capability = "CONDSTORE"
222 capQresync capability = "QRESYNC"
233 stateNotAuthenticated state = iota
238func stateCommands(cmds ...string) map[string]struct{} {
239 r := map[string]struct{}{}
240 for _, cmd := range cmds {
247 commandsStateAny = stateCommands("capability", "noop", "logout", "id")
248 commandsStateNotAuthenticated = stateCommands("starttls", "authenticate", "login")
249 commandsStateAuthenticated = stateCommands("enable", "select", "examine", "create", "delete", "rename", "subscribe", "unsubscribe", "list", "namespace", "status", "append", "idle", "lsub", "getquotaroot", "getquota")
250 commandsStateSelected = stateCommands("close", "unselect", "expunge", "search", "fetch", "store", "copy", "move", "uid expunge", "uid search", "uid fetch", "uid store", "uid copy", "uid move")
253var commands = map[string]func(c *conn, tag, cmd string, p *parser){
255 "capability": (*conn).cmdCapability,
256 "noop": (*conn).cmdNoop,
257 "logout": (*conn).cmdLogout,
261 "starttls": (*conn).cmdStarttls,
262 "authenticate": (*conn).cmdAuthenticate,
263 "login": (*conn).cmdLogin,
265 // Authenticated and selected.
266 "enable": (*conn).cmdEnable,
267 "select": (*conn).cmdSelect,
268 "examine": (*conn).cmdExamine,
269 "create": (*conn).cmdCreate,
270 "delete": (*conn).cmdDelete,
271 "rename": (*conn).cmdRename,
272 "subscribe": (*conn).cmdSubscribe,
273 "unsubscribe": (*conn).cmdUnsubscribe,
274 "list": (*conn).cmdList,
275 "lsub": (*conn).cmdLsub,
276 "namespace": (*conn).cmdNamespace,
277 "status": (*conn).cmdStatus,
278 "append": (*conn).cmdAppend,
279 "idle": (*conn).cmdIdle,
280 "getquotaroot": (*conn).cmdGetquotaroot,
281 "getquota": (*conn).cmdGetquota,
284 "check": (*conn).cmdCheck,
285 "close": (*conn).cmdClose,
286 "unselect": (*conn).cmdUnselect,
287 "expunge": (*conn).cmdExpunge,
288 "uid expunge": (*conn).cmdUIDExpunge,
289 "search": (*conn).cmdSearch,
290 "uid search": (*conn).cmdUIDSearch,
291 "fetch": (*conn).cmdFetch,
292 "uid fetch": (*conn).cmdUIDFetch,
293 "store": (*conn).cmdStore,
294 "uid store": (*conn).cmdUIDStore,
295 "copy": (*conn).cmdCopy,
296 "uid copy": (*conn).cmdUIDCopy,
297 "move": (*conn).cmdMove,
298 "uid move": (*conn).cmdUIDMove,
301var errIO = errors.New("io error") // For read/write errors and errors that should close the connection.
302var errProtocol = errors.New("protocol error") // For protocol errors for which a stack trace should be printed.
306// check err for sanity.
307// if not nil and checkSanity true (set during tests), then panic. if not nil during normal operation, just log.
308func (c *conn) xsanity(err error, format string, args ...any) {
313 panic(fmt.Errorf("%s: %s", fmt.Sprintf(format, args...), err))
315 c.log.Errorx(fmt.Sprintf(format, args...), err)
320// Listen initializes all imap listeners for the configuration, and stores them for Serve to start them.
322 names := maps.Keys(mox.Conf.Static.Listeners)
324 for _, name := range names {
325 listener := mox.Conf.Static.Listeners[name]
327 var tlsConfig *tls.Config
328 if listener.TLS != nil {
329 tlsConfig = listener.TLS.Config
332 if listener.IMAP.Enabled {
333 port := config.Port(listener.IMAP.Port, 143)
334 for _, ip := range listener.IPs {
335 listen1("imap", name, ip, port, tlsConfig, false, listener.IMAP.NoRequireSTARTTLS)
339 if listener.IMAPS.Enabled {
340 port := config.Port(listener.IMAPS.Port, 993)
341 for _, ip := range listener.IPs {
342 listen1("imaps", name, ip, port, tlsConfig, true, false)
350func listen1(protocol, listenerName, ip string, port int, tlsConfig *tls.Config, xtls, noRequireSTARTTLS bool) {
351 log := mlog.New("imapserver", nil)
352 addr := net.JoinHostPort(ip, fmt.Sprintf("%d", port))
353 if os.Getuid() == 0 {
354 log.Print("listening for imap",
355 slog.String("listener", listenerName),
356 slog.String("addr", addr),
357 slog.String("protocol", protocol))
359 network := mox.Network(ip)
360 ln, err := mox.Listen(network, addr)
362 log.Fatalx("imap: listen for imap", err, slog.String("protocol", protocol), slog.String("listener", listenerName))
365 // Each listener gets its own copy of the config, so session keys between different
366 // ports on same listener aren't shared. We rotate session keys explicitly in this
367 // base TLS config because each connection clones the TLS config before using. The
368 // base TLS config would never get automatically managed/rotated session keys.
369 if tlsConfig != nil {
370 tlsConfig = tlsConfig.Clone()
371 mox.StartTLSSessionTicketKeyRefresher(mox.Shutdown, log, tlsConfig)
376 conn, err := ln.Accept()
378 log.Infox("imap: accept", err, slog.String("protocol", protocol), slog.String("listener", listenerName))
382 metricIMAPConnection.WithLabelValues(protocol).Inc()
383 go serve(listenerName, mox.Cid(), tlsConfig, conn, xtls, noRequireSTARTTLS)
387 servers = append(servers, serve)
390// Serve starts serving on all listeners, launching a goroutine per listener.
392 for _, serve := range servers {
398// returns whether this connection accepts utf-8 in strings.
399func (c *conn) utf8strings() bool {
400 return c.enabled[capIMAP4rev2] || c.enabled[capUTF8Accept]
403func (c *conn) encodeMailbox(s string) string {
410func (c *conn) xdbwrite(fn func(tx *bstore.Tx)) {
411 err := c.account.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
415 xcheckf(err, "transaction")
418func (c *conn) xdbread(fn func(tx *bstore.Tx)) {
419 err := c.account.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
423 xcheckf(err, "transaction")
426// Closes the currently selected/active mailbox, setting state from selected to authenticated.
427// Does not remove messages marked for deletion.
428func (c *conn) unselect() {
429 if c.state == stateSelected {
430 c.state = stateAuthenticated
436func (c *conn) setSlow(on bool) {
438 c.log.Debug("connection changed to slow")
439 } else if !on && c.slow {
440 c.log.Debug("connection restored to regular pace")
445// Write makes a connection an io.Writer. It panics for i/o errors. These errors
446// are handled in the connection command loop.
447func (c *conn) Write(buf []byte) (int, error) {
455 err := c.conn.SetWriteDeadline(time.Now().Add(30 * time.Second))
456 c.log.Check(err, "setting write deadline")
458 nn, err := c.conn.Write(buf[:chunk])
460 panic(fmt.Errorf("write: %s (%w)", err, errIO))
464 if len(buf) > 0 && badClientDelay > 0 {
465 mox.Sleep(mox.Context, badClientDelay)
471func (c *conn) xtrace(level slog.Level) func() {
477 c.tr.SetTrace(mlog.LevelTrace)
478 c.tw.SetTrace(mlog.LevelTrace)
482// Cache of line buffers for reading commands.
484var bufpool = moxio.NewBufpool(8, 16*1024)
486// read line from connection, not going through line channel.
487func (c *conn) readline0() (string, error) {
488 if c.slow && badClientDelay > 0 {
489 mox.Sleep(mox.Context, badClientDelay)
492 d := 30 * time.Minute
493 if c.state == stateNotAuthenticated {
496 err := c.conn.SetReadDeadline(time.Now().Add(d))
497 c.log.Check(err, "setting read deadline")
499 line, err := bufpool.Readline(c.log, c.br)
500 if err != nil && errors.Is(err, moxio.ErrLineTooLong) {
501 return "", fmt.Errorf("%s (%w)", err, errProtocol)
502 } else if err != nil {
503 return "", fmt.Errorf("%s (%w)", err, errIO)
508func (c *conn) lineChan() chan lineErr {
510 c.line = make(chan lineErr, 1)
512 line, err := c.readline0()
513 c.line <- lineErr{line, err}
519// readline from either the c.line channel, or otherwise read from connection.
520func (c *conn) readline(readCmd bool) string {
526 line, err = le.line, le.err
528 line, err = c.readline0()
531 if readCmd && errors.Is(err, os.ErrDeadlineExceeded) {
532 err := c.conn.SetWriteDeadline(time.Now().Add(10 * time.Second))
533 c.log.Check(err, "setting write deadline")
534 c.writelinef("* BYE inactive")
536 if !errors.Is(err, errIO) && !errors.Is(err, errProtocol) {
537 err = fmt.Errorf("%s (%w)", err, errIO)
543 // We typically respond immediately (IDLE is an exception).
544 // The client may not be reading, or may have disappeared.
545 // Don't wait more than 5 minutes before closing down the connection.
546 // The write deadline is managed in IDLE as well.
547 // For unauthenticated connections, we require the client to read faster.
548 wd := 5 * time.Minute
549 if c.state == stateNotAuthenticated {
550 wd = 30 * time.Second
552 err = c.conn.SetWriteDeadline(time.Now().Add(wd))
553 c.log.Check(err, "setting write deadline")
558// write tagged command response, but first write pending changes.
559func (c *conn) writeresultf(format string, args ...any) {
560 c.bwriteresultf(format, args...)
564// write buffered tagged command response, but first write pending changes.
565func (c *conn) bwriteresultf(format string, args ...any) {
567 case "fetch", "store", "search":
571 c.applyChanges(c.comm.Get(), false)
574 c.bwritelinef(format, args...)
577func (c *conn) writelinef(format string, args ...any) {
578 c.bwritelinef(format, args...)
582// Buffer line for write.
583func (c *conn) bwritelinef(format string, args ...any) {
585 fmt.Fprintf(c.bw, format, args...)
588func (c *conn) xflush() {
590 xcheckf(err, "flush") // Should never happen, the Write caused by the Flush should panic on i/o error.
593func (c *conn) readCommand(tag *string) (cmd string, p *parser) {
594 line := c.readline(true)
595 p = newParser(line, c)
601 return cmd, newParser(p.remainder(), c)
604func (c *conn) xreadliteral(size int64, sync bool) string {
608 buf := make([]byte, size)
610 if err := c.conn.SetReadDeadline(time.Now().Add(30 * time.Second)); err != nil {
611 c.log.Errorx("setting read deadline", err)
614 _, err := io.ReadFull(c.br, buf)
616 // Cannot use xcheckf due to %w handling of errIO.
617 panic(fmt.Errorf("reading literal: %s (%w)", err, errIO))
623func (c *conn) xhighestModSeq(tx *bstore.Tx, mailboxID int64) store.ModSeq {
624 qms := bstore.QueryTx[store.Message](tx)
625 qms.FilterNonzero(store.Message{MailboxID: mailboxID})
626 qms.SortDesc("ModSeq")
629 if err == bstore.ErrAbsent {
630 return store.ModSeq(0)
632 xcheckf(err, "looking up highest modseq for mailbox")
636var cleanClose struct{} // Sentinel value for panic/recover indicating clean close of connection.
638func serve(listenerName string, cid int64, tlsConfig *tls.Config, nc net.Conn, xtls, noRequireSTARTTLS bool) {
640 if a, ok := nc.RemoteAddr().(*net.TCPAddr); ok {
643 // For net.Pipe, during tests.
644 remoteIP = net.ParseIP("127.0.0.10")
652 baseTLSConfig: tlsConfig,
654 noRequireSTARTTLS: noRequireSTARTTLS,
655 enabled: map[capability]bool{},
657 cmdStart: time.Now(),
659 var logmutex sync.Mutex
660 c.log = mlog.New("imapserver", nil).WithFunc(func() []slog.Attr {
662 defer logmutex.Unlock()
665 slog.Int64("cid", c.cid),
666 slog.Duration("delta", now.Sub(c.lastlog)),
669 if c.username != "" {
670 l = append(l, slog.String("username", c.username))
674 c.tr = moxio.NewTraceReader(c.log, "C: ", c.conn)
675 // todo: tracing should be done on whatever comes out of c.br. the remote connection write a command plus data, and bufio can read it in one read, causing a command parser that sets the tracing level to data to have no effect. we are now typically logging sent messages, when mail clients append to the Sent mailbox.
676 c.br = bufio.NewReader(c.tr)
677 c.tw = moxio.NewTraceWriter(c.log, "S: ", c)
678 c.bw = bufio.NewWriter(c.tw)
680 // Many IMAP connections use IDLE to wait for new incoming messages. We'll enable
681 // keepalive to get a higher chance of the connection staying alive, or otherwise
682 // detecting broken connections early.
683 if tcpconn, ok := c.conn.(*net.TCPConn); ok {
684 if err := tcpconn.SetKeepAlivePeriod(5 * time.Minute); err != nil {
685 c.log.Errorx("setting keepalive period", err)
686 } else if err := tcpconn.SetKeepAlive(true); err != nil {
687 c.log.Errorx("enabling keepalive", err)
691 c.log.Info("new connection",
692 slog.Any("remote", c.conn.RemoteAddr()),
693 slog.Any("local", c.conn.LocalAddr()),
694 slog.Bool("tls", xtls),
695 slog.String("listener", listenerName))
700 if c.account != nil {
702 err := c.account.Close()
703 c.xsanity(err, "close account")
709 if x == nil || x == cleanClose {
710 c.log.Info("connection closed")
711 } else if err, ok := x.(error); ok && isClosed(err) {
712 c.log.Infox("connection closed", err)
714 c.log.Error("unhandled panic", slog.Any("err", x))
716 metrics.PanicInc(metrics.Imapserver)
721 // Start TLS on connection. We perform the handshake explicitly, so we can set a
722 // timeout, do client certificate authentication, log TLS details afterwards.
723 c.xtlsHandshakeAndAuthenticate(c.conn)
727 case <-mox.Shutdown.Done():
729 c.writelinef("* BYE mox shutting down")
734 if !limiterConnectionrate.Add(c.remoteIP, time.Now(), 1) {
735 c.writelinef("* BYE connection rate from your ip or network too high, slow down please")
739 // If remote IP/network resulted in too many authentication failures, refuse to serve.
740 if !mox.LimiterFailedAuth.CanAdd(c.remoteIP, time.Now(), 1) {
741 metrics.AuthenticationRatelimitedInc("imap")
742 c.log.Debug("refusing connection due to many auth failures", slog.Any("remoteip", c.remoteIP))
743 c.writelinef("* BYE too many auth failures")
747 if !limiterConnections.Add(c.remoteIP, time.Now(), 1) {
748 c.log.Debug("refusing connection due to many open connections", slog.Any("remoteip", c.remoteIP))
749 c.writelinef("* BYE too many open connections from your ip or network")
752 defer limiterConnections.Add(c.remoteIP, time.Now(), -1)
754 // We register and unregister the original connection, in case it c.conn is
755 // replaced with a TLS connection later on.
756 mox.Connections.Register(nc, "imap", listenerName)
757 defer mox.Connections.Unregister(nc)
759 if c.account != nil && !c.noPreauth {
760 c.state = stateAuthenticated
761 c.writelinef("* PREAUTH [CAPABILITY %s] mox imap welcomes %s", c.capabilities(), c.username)
763 c.writelinef("* OK [CAPABILITY %s] mox imap", c.capabilities())
768 c.xflush() // For flushing errors, or possibly commands that did not flush explicitly.
772// isClosed returns whether i/o failed, typically because the connection is closed.
773// For connection errors, we often want to generate fewer logs.
774func isClosed(err error) bool {
775 return errors.Is(err, errIO) || errors.Is(err, errProtocol) || moxio.IsClosed(err)
778// makeTLSConfig makes a new tls config that is bound to the connection for
779// possible client certificate authentication.
780func (c *conn) makeTLSConfig() *tls.Config {
781 // We clone the config so we can set VerifyPeerCertificate below to a method bound
782 // to this connection. Earlier, we set session keys explicitly on the base TLS
783 // config, so they can be used for this connection too.
784 tlsConf := c.baseTLSConfig.Clone()
786 // Allow client certificate authentication, for use with the sasl "external"
787 // authentication mechanism.
788 tlsConf.ClientAuth = tls.RequestClientCert
790 // We verify the client certificate during the handshake. The TLS handshake is
791 // initiated explicitly for incoming connections and during starttls, so we can
792 // immediately extract the account name and address used for authentication.
793 tlsConf.VerifyPeerCertificate = c.tlsClientAuthVerifyPeerCert
798// tlsClientAuthVerifyPeerCert can be used as tls.Config.VerifyPeerCertificate, and
799// sets authentication-related fields on conn. This is not called on resumed TLS
801func (c *conn) tlsClientAuthVerifyPeerCert(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
802 if len(rawCerts) == 0 {
806 // If we had too many authentication failures from this IP, don't attempt
807 // authentication. If this is a new incoming connetion, it is closed after the TLS
809 if !mox.LimiterFailedAuth.CanAdd(c.remoteIP, time.Now(), 1) {
813 cert, err := x509.ParseCertificate(rawCerts[0])
815 c.log.Debugx("parsing tls client certificate", err)
818 if err := c.tlsClientAuthVerifyPeerCertParsed(cert); err != nil {
819 c.log.Debugx("verifying tls client certificate", err)
820 return fmt.Errorf("verifying client certificate: %w", err)
825// tlsClientAuthVerifyPeerCertParsed verifies a client certificate. Called both for
826// fresh and resumed TLS connections.
827func (c *conn) tlsClientAuthVerifyPeerCertParsed(cert *x509.Certificate) error {
828 if c.account != nil {
829 return fmt.Errorf("cannot authenticate with tls client certificate after previous authentication")
832 authResult := "error"
834 metrics.AuthenticationInc("imap", "tlsclientauth", authResult)
835 if authResult == "ok" {
836 mox.LimiterFailedAuth.Reset(c.remoteIP, time.Now())
838 mox.LimiterFailedAuth.Add(c.remoteIP, time.Now(), 1)
842 // For many failed auth attempts, slow down verification attempts.
843 if c.authFailed > 3 && authFailDelay > 0 {
844 mox.Sleep(mox.Context, time.Duration(c.authFailed-3)*authFailDelay)
846 c.authFailed++ // Compensated on success.
848 // On the 3rd failed authentication, start responding slowly. Successful auth will
849 // cause fast responses again.
850 if c.authFailed >= 3 {
855 shabuf := sha256.Sum256(cert.RawSubjectPublicKeyInfo)
856 fp := base64.RawURLEncoding.EncodeToString(shabuf[:])
857 pubKey, err := store.TLSPublicKeyGet(context.TODO(), fp)
859 if err == bstore.ErrAbsent {
860 authResult = "badcreds"
862 return fmt.Errorf("looking up tls public key with fingerprint %s: %v", fp, err)
865 // Verify account exists and still matches address.
866 acc, _, err := store.OpenEmail(c.log, pubKey.LoginAddress)
868 return fmt.Errorf("opening account for address %s for public key %s: %w", pubKey.LoginAddress, fp, err)
873 c.xsanity(err, "close account")
876 if acc.Name != pubKey.Account {
877 return fmt.Errorf("tls client public key %s is for account %s, but email address %s is for account %s", fp, pubKey.Account, pubKey.LoginAddress, acc.Name)
882 c.noPreauth = pubKey.NoIMAPPreauth
884 acc = nil // Prevent cleanup by defer.
885 c.username = pubKey.LoginAddress
886 c.comm = store.RegisterComm(c.account)
887 c.log.Debug("tls client authenticated with client certificate",
888 slog.String("fingerprint", fp),
889 slog.String("username", c.username),
890 slog.String("account", c.account.Name),
891 slog.Any("remote", c.remoteIP))
895// xtlsHandshakeAndAuthenticate performs the TLS handshake, and verifies a client
896// certificate if present.
897func (c *conn) xtlsHandshakeAndAuthenticate(conn net.Conn) {
898 tlsConn := tls.Server(conn, c.makeTLSConfig())
900 c.tr = moxio.NewTraceReader(c.log, "C: ", c.conn)
901 c.br = bufio.NewReader(c.tr)
903 cidctx := context.WithValue(mox.Context, mlog.CidKey, c.cid)
904 ctx, cancel := context.WithTimeout(cidctx, time.Minute)
906 c.log.Debug("starting tls server handshake")
907 if err := tlsConn.HandshakeContext(ctx); err != nil {
908 panic(fmt.Errorf("tls handshake: %s (%w)", err, errIO))
912 cs := tlsConn.ConnectionState()
913 if cs.DidResume && len(cs.PeerCertificates) > 0 {
914 // Verify client after session resumption.
915 err := c.tlsClientAuthVerifyPeerCertParsed(cs.PeerCertificates[0])
917 c.bwritelinef("* BYE [ALERT] Error verifying client certificate after TLS session resumption: %s", err)
918 panic(fmt.Errorf("tls verify client certificate after resumption: %s (%w)", err, errIO))
922 attrs := []slog.Attr{
923 slog.Any("version", tlsVersion(cs.Version)),
924 slog.String("ciphersuite", tls.CipherSuiteName(cs.CipherSuite)),
925 slog.String("sni", cs.ServerName),
926 slog.Bool("resumed", cs.DidResume),
927 slog.Int("clientcerts", len(cs.PeerCertificates)),
929 if c.account != nil {
930 attrs = append(attrs,
931 slog.String("account", c.account.Name),
932 slog.String("username", c.username),
935 c.log.Debug("tls handshake completed", attrs...)
938type tlsVersion uint16
940func (v tlsVersion) String() string {
941 return strings.ReplaceAll(strings.ToLower(tls.VersionName(uint16(v))), " ", "-")
944func (c *conn) command() {
945 var tag, cmd, cmdlow string
951 metricIMAPCommands.WithLabelValues(c.cmdMetric, result).Observe(float64(time.Since(c.cmdStart)) / float64(time.Second))
954 logFields := []slog.Attr{
955 slog.String("cmd", c.cmd),
956 slog.Duration("duration", time.Since(c.cmdStart)),
961 if x == nil || x == cleanClose {
962 c.log.Debug("imap command done", logFields...)
971 c.log.Error("imap command panic", append([]slog.Attr{slog.Any("panic", x)}, logFields...)...)
976 var sxerr syntaxError
980 c.log.Infox("imap command ioerror", err, logFields...)
982 if errors.Is(err, errProtocol) {
986 } else if errors.As(err, &sxerr) {
989 // Other side is likely speaking something else than IMAP, send error message and
990 // stop processing because there is a good chance whatever they sent has multiple
992 c.writelinef("* BYE please try again speaking imap")
995 c.log.Debugx("imap command syntax error", sxerr.err, logFields...)
996 c.log.Info("imap syntax error", slog.String("lastline", c.lastLine))
997 fatal := strings.HasSuffix(c.lastLine, "+}")
999 err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
1000 c.log.Check(err, "setting write deadline")
1002 if sxerr.line != "" {
1003 c.bwritelinef("%s", sxerr.line)
1006 if sxerr.code != "" {
1007 code = "[" + sxerr.code + "] "
1009 c.bwriteresultf("%s BAD %s%s unrecognized syntax/command: %v", tag, code, cmd, sxerr.errmsg)
1012 panic(fmt.Errorf("aborting connection after syntax error for command with non-sync literal: %w", errProtocol))
1014 } else if errors.As(err, &serr) {
1015 result = "servererror"
1016 c.log.Errorx("imap command server error", err, logFields...)
1018 c.bwriteresultf("%s NO %s %v", tag, cmd, err)
1019 } else if errors.As(err, &uerr) {
1020 result = "usererror"
1021 c.log.Debugx("imap command user error", err, logFields...)
1022 if uerr.code != "" {
1023 c.bwriteresultf("%s NO [%s] %s %v", tag, uerr.code, cmd, err)
1025 c.bwriteresultf("%s NO %s %v", tag, cmd, err)
1028 // Other type of panic, we pass it on, aborting the connection.
1030 c.log.Errorx("imap command panic", err, logFields...)
1036 cmd, p = c.readCommand(&tag)
1037 cmdlow = strings.ToLower(cmd)
1039 c.cmdStart = time.Now()
1040 c.cmdMetric = "(unrecognized)"
1043 case <-mox.Shutdown.Done():
1045 c.writelinef("* BYE shutting down")
1050 fn := commands[cmdlow]
1052 xsyntaxErrorf("unknown command %q", cmd)
1057 // Check if command is allowed in this state.
1058 if _, ok1 := commandsStateAny[cmdlow]; ok1 {
1059 } else if _, ok2 := commandsStateNotAuthenticated[cmdlow]; ok2 && c.state == stateNotAuthenticated {
1060 } else if _, ok3 := commandsStateAuthenticated[cmdlow]; ok3 && c.state == stateAuthenticated || c.state == stateSelected {
1061 } else if _, ok4 := commandsStateSelected[cmdlow]; ok4 && c.state == stateSelected {
1062 } else if ok1 || ok2 || ok3 || ok4 {
1063 xuserErrorf("not allowed in this connection state")
1065 xserverErrorf("unrecognized command")
1071func (c *conn) broadcast(changes []store.Change) {
1072 if len(changes) == 0 {
1075 c.log.Debug("broadcast changes", slog.Any("changes", changes))
1076 c.comm.Broadcast(changes)
1079// matchStringer matches a string against reference + mailbox patterns.
1080type matchStringer interface {
1081 MatchString(s string) bool
1084type noMatch struct{}
1086// MatchString for noMatch always returns false.
1087func (noMatch) MatchString(s string) bool {
1091// xmailboxPatternMatcher returns a matcher for mailbox names given the reference and patterns.
1092// Patterns can include "%" and "*", matching any character excluding and including a slash respectively.
1093func xmailboxPatternMatcher(ref string, patterns []string) matchStringer {
1094 if strings.HasPrefix(ref, "/") {
1099 for _, pat := range patterns {
1100 if strings.HasPrefix(pat, "/") {
1106 s = path.Join(ref, pat)
1109 // Fix casing for all Inbox paths.
1110 first := strings.SplitN(s, "/", 2)[0]
1111 if strings.EqualFold(first, "Inbox") {
1112 s = "Inbox" + s[len("Inbox"):]
1117 for _, c := range s {
1120 } else if c == '*' {
1123 rs += regexp.QuoteMeta(string(c))
1126 subs = append(subs, rs)
1132 rs := "^(" + strings.Join(subs, "|") + ")$"
1133 re, err := regexp.Compile(rs)
1134 xcheckf(err, "compiling regexp for mailbox patterns")
1138func (c *conn) sequence(uid store.UID) msgseq {
1139 return uidSearch(c.uids, uid)
1142func uidSearch(uids []store.UID, uid store.UID) msgseq {
1149 return msgseq(i + 1)
1159func (c *conn) xsequence(uid store.UID) msgseq {
1160 seq := c.sequence(uid)
1162 xserverErrorf("unknown uid %d (%w)", uid, errProtocol)
1167func (c *conn) sequenceRemove(seq msgseq, uid store.UID) {
1169 if c.uids[i] != uid {
1170 xserverErrorf("got uid %d at msgseq %d, expected uid %d", uid, seq, c.uids[i])
1172 copy(c.uids[i:], c.uids[i+1:])
1173 c.uids = c.uids[:len(c.uids)-1]
1179// add uid to the session. care must be taken that pending changes are fetched
1180// while holding the account wlock, and applied before adding this uid, because
1181// those pending changes may contain another new uid that has to be added first.
1182func (c *conn) uidAppend(uid store.UID) {
1183 if uidSearch(c.uids, uid) > 0 {
1184 xserverErrorf("uid already present (%w)", errProtocol)
1186 if len(c.uids) > 0 && uid < c.uids[len(c.uids)-1] {
1187 xserverErrorf("new uid %d is smaller than last uid %d (%w)", uid, c.uids[len(c.uids)-1], errProtocol)
1189 c.uids = append(c.uids, uid)
1195// sanity check that uids are in ascending order.
1196func checkUIDs(uids []store.UID) {
1197 for i, uid := range uids {
1198 if uid == 0 || i > 0 && uid <= uids[i-1] {
1199 xserverErrorf("bad uids %v", uids)
1204func (c *conn) xnumSetUIDs(isUID bool, nums numSet) []store.UID {
1205 _, uids := c.xnumSetConditionUIDs(false, true, isUID, nums)
1209func (c *conn) xnumSetCondition(isUID bool, nums numSet) []any {
1210 uidargs, _ := c.xnumSetConditionUIDs(true, false, isUID, nums)
1214func (c *conn) xnumSetConditionUIDs(forDB, returnUIDs bool, isUID bool, nums numSet) ([]any, []store.UID) {
1215 if nums.searchResult {
1216 // Update previously stored UIDs. Some may have been deleted.
1217 // Once deleted a UID will never come back, so we'll just remove those uids.
1219 for _, uid := range c.searchResult {
1220 if uidSearch(c.uids, uid) > 0 {
1221 c.searchResult[o] = uid
1225 c.searchResult = c.searchResult[:o]
1226 uidargs := make([]any, len(c.searchResult))
1227 for i, uid := range c.searchResult {
1230 return uidargs, c.searchResult
1234 var uids []store.UID
1236 add := func(uid store.UID) {
1238 uidargs = append(uidargs, uid)
1241 uids = append(uids, uid)
1246 // Sequence numbers that don't exist, or * on an empty mailbox, should result in a BAD response.
../rfc/9051:7018
1247 for _, r := range nums.ranges {
1250 if len(c.uids) == 0 {
1251 xsyntaxErrorf("invalid seqset * on empty mailbox")
1253 ia = len(c.uids) - 1
1255 ia = int(r.first.number - 1)
1256 if ia >= len(c.uids) {
1257 xsyntaxErrorf("msgseq %d not in mailbox", r.first.number)
1266 if len(c.uids) == 0 {
1267 xsyntaxErrorf("invalid seqset * on empty mailbox")
1269 ib = len(c.uids) - 1
1271 ib = int(r.last.number - 1)
1272 if ib >= len(c.uids) {
1273 xsyntaxErrorf("msgseq %d not in mailbox", r.last.number)
1279 for _, uid := range c.uids[ia : ib+1] {
1283 return uidargs, uids
1286 // UIDs that do not exist can be ignored.
1287 if len(c.uids) == 0 {
1291 for _, r := range nums.ranges {
1297 uida := store.UID(r.first.number)
1299 uida = c.uids[len(c.uids)-1]
1302 uidb := store.UID(last.number)
1304 uidb = c.uids[len(c.uids)-1]
1308 uida, uidb = uidb, uida
1311 // Binary search for uida.
1316 if uida < c.uids[m] {
1318 } else if uida > c.uids[m] {
1325 for _, uid := range c.uids[s:] {
1326 if uid >= uida && uid <= uidb {
1328 } else if uid > uidb {
1334 return uidargs, uids
1337func (c *conn) ok(tag, cmd string) {
1338 c.bwriteresultf("%s OK %s done", tag, cmd)
1342// xcheckmailboxname checks if name is valid, returning an INBOX-normalized name.
1343// I.e. it changes various casings of INBOX and INBOX/* to Inbox and Inbox/*.
1344// Name is invalid if it contains leading/trailing/double slashes, or when it isn't
1345// unicode-normalized, or when empty or has special characters.
1346func xcheckmailboxname(name string, allowInbox bool) string {
1347 name, isinbox, err := store.CheckMailboxName(name, allowInbox)
1349 xuserErrorf("special mailboxname Inbox not allowed")
1350 } else if err != nil {
1351 xusercodeErrorf("CANNOT", "%s", err)
1356// Lookup mailbox by name.
1357// If the mailbox does not exist, panic is called with a user error.
1358// Must be called with account rlock held.
1359func (c *conn) xmailbox(tx *bstore.Tx, name string, missingErrCode string) store.Mailbox {
1360 mb, err := c.account.MailboxFind(tx, name)
1361 xcheckf(err, "finding mailbox")
1363 // missingErrCode can be empty, or e.g. TRYCREATE or ALREADYEXISTS.
1364 xusercodeErrorf(missingErrCode, "%w", store.ErrUnknownMailbox)
1369// Lookup mailbox by ID.
1370// If the mailbox does not exist, panic is called with a user error.
1371// Must be called with account rlock held.
1372func (c *conn) xmailboxID(tx *bstore.Tx, id int64) store.Mailbox {
1373 mb := store.Mailbox{ID: id}
1375 if err == bstore.ErrAbsent {
1376 xuserErrorf("%w", store.ErrUnknownMailbox)
1381// Apply changes to our session state.
1382// If initial is false, updates like EXISTS and EXPUNGE are written to the client.
1383// If initial is true, we only apply the changes.
1384// Should not be called while holding locks, as changes are written to client connections, which can block.
1385// Does not flush output.
1386func (c *conn) applyChanges(changes []store.Change, initial bool) {
1387 if len(changes) == 0 {
1391 err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Minute))
1392 c.log.Check(err, "setting write deadline")
1394 c.log.Debug("applying changes", slog.Any("changes", changes))
1396 // Only keep changes for the selected mailbox, and changes that are always relevant.
1397 var n []store.Change
1398 for _, change := range changes {
1400 switch ch := change.(type) {
1401 case store.ChangeAddUID:
1403 case store.ChangeRemoveUIDs:
1405 case store.ChangeFlags:
1407 case store.ChangeRemoveMailbox, store.ChangeAddMailbox, store.ChangeRenameMailbox, store.ChangeAddSubscription:
1408 n = append(n, change)
1410 case store.ChangeMailboxCounts, store.ChangeMailboxSpecialUse, store.ChangeMailboxKeywords, store.ChangeThread:
1412 panic(fmt.Errorf("missing case for %#v", change))
1414 if c.state == stateSelected && mbID == c.mailboxID {
1415 n = append(n, change)
1420 qresync := c.enabled[capQresync]
1421 condstore := c.enabled[capCondstore]
1424 for i < len(changes) {
1425 // First process all new uids. So we only send a single EXISTS.
1426 var adds []store.ChangeAddUID
1427 for ; i < len(changes); i++ {
1428 ch, ok := changes[i].(store.ChangeAddUID)
1432 seq := c.sequence(ch.UID)
1433 if seq > 0 && initial {
1437 adds = append(adds, ch)
1443 // Write the exists, and the UID and flags as well. Hopefully the client waits for
1444 // long enough after the EXISTS to see these messages, and doesn't request them
1445 // again with a FETCH.
1446 c.bwritelinef("* %d EXISTS", len(c.uids))
1447 for _, add := range adds {
1448 seq := c.xsequence(add.UID)
1449 var modseqStr string
1451 modseqStr = fmt.Sprintf(" MODSEQ (%d)", add.ModSeq.Client())
1453 c.bwritelinef("* %d FETCH (UID %d FLAGS %s%s)", seq, add.UID, flaglist(add.Flags, add.Keywords).pack(c), modseqStr)
1458 change := changes[i]
1461 switch ch := change.(type) {
1462 case store.ChangeRemoveUIDs:
1463 var vanishedUIDs numSet
1464 for _, uid := range ch.UIDs {
1467 seq = c.sequence(uid)
1472 seq = c.xsequence(uid)
1474 c.sequenceRemove(seq, uid)
1477 vanishedUIDs.append(uint32(uid))
1479 c.bwritelinef("* %d EXPUNGE", seq)
1485 for _, s := range vanishedUIDs.Strings(4*1024 - 32) {
1486 c.bwritelinef("* VANISHED %s", s)
1489 case store.ChangeFlags:
1490 // The uid can be unknown if we just expunged it while another session marked it as deleted just before.
1491 seq := c.sequence(ch.UID)
1496 var modseqStr string
1498 modseqStr = fmt.Sprintf(" MODSEQ (%d)", ch.ModSeq.Client())
1500 c.bwritelinef("* %d FETCH (UID %d FLAGS %s%s)", seq, ch.UID, flaglist(ch.Flags, ch.Keywords).pack(c), modseqStr)
1502 case store.ChangeRemoveMailbox:
1503 // Only announce \NonExistent to modern clients, otherwise they may ignore the
1504 // unrecognized \NonExistent and interpret this as a newly created mailbox, while
1505 // the goal was to remove it...
1506 if c.enabled[capIMAP4rev2] {
1507 c.bwritelinef(`* LIST (\NonExistent) "/" %s`, astring(c.encodeMailbox(ch.Name)).pack(c))
1509 case store.ChangeAddMailbox:
1510 c.bwritelinef(`* LIST (%s) "/" %s`, strings.Join(ch.Flags, " "), astring(c.encodeMailbox(ch.Mailbox.Name)).pack(c))
1511 case store.ChangeRenameMailbox:
1514 if c.enabled[capIMAP4rev2] {
1515 oldname = fmt.Sprintf(` ("OLDNAME" (%s))`, string0(c.encodeMailbox(ch.OldName)).pack(c))
1517 c.bwritelinef(`* LIST (%s) "/" %s%s`, strings.Join(ch.Flags, " "), astring(c.encodeMailbox(ch.NewName)).pack(c), oldname)
1518 case store.ChangeAddSubscription:
1519 c.bwritelinef(`* LIST (%s) "/" %s`, strings.Join(append([]string{`\Subscribed`}, ch.Flags...), " "), astring(c.encodeMailbox(ch.Name)).pack(c))
1521 panic(fmt.Sprintf("internal error, missing case for %#v", change))
1526// Capability returns the capabilities this server implements and currently has
1527// available given the connection state.
1530func (c *conn) cmdCapability(tag, cmd string, p *parser) {
1536 caps := c.capabilities()
1539 c.bwritelinef("* CAPABILITY %s", caps)
1543// capabilities returns non-empty string with available capabilities based on connection state.
1544// For use in cmdCapability and untagged OK responses on connection start, login and authenticate.
1545func (c *conn) capabilities() string {
1546 caps := serverCapabilities
1548 // We only allow starting without TLS when explicitly configured, in violation of RFC.
1549 if !c.tls && c.baseTLSConfig != nil {
1552 if c.tls || c.noRequireSTARTTLS {
1553 caps += " AUTH=PLAIN"
1555 caps += " LOGINDISABLED"
1557 if c.tls && len(c.conn.(*tls.Conn).ConnectionState().PeerCertificates) > 0 {
1558 caps += " AUTH=EXTERNAL"
1563// No op, but useful for retrieving pending changes as untagged responses, e.g. of
1567func (c *conn) cmdNoop(tag, cmd string, p *parser) {
1575// Logout, after which server closes the connection.
1578func (c *conn) cmdLogout(tag, cmd string, p *parser) {
1585 c.state = stateNotAuthenticated
1587 c.bwritelinef("* BYE thanks")
1592// Clients can use ID to tell the server which software they are using. Servers can
1593// respond with their version. For statistics/logging/debugging purposes.
1596func (c *conn) cmdID(tag, cmd string, p *parser) {
1601 var params map[string]string
1603 params = map[string]string{}
1605 if len(params) > 0 {
1611 if _, ok := params[k]; ok {
1612 xsyntaxErrorf("duplicate key %q", k)
1621 // We just log the client id.
1622 c.log.Info("client id", slog.Any("params", params))
1626 c.bwritelinef(`* ID ("name" "mox" "version" %s)`, string0(moxvar.Version).pack(c))
1630// STARTTLS enables TLS on the connection, after a plain text start.
1631// Only allowed if TLS isn't already enabled, either through connecting to a
1632// TLS-enabled TCP port, or a previous STARTTLS command.
1633// After STARTTLS, plain text authentication typically becomes available.
1635// Status: Not authenticated.
1636func (c *conn) cmdStarttls(tag, cmd string, p *parser) {
1645 if c.baseTLSConfig == nil {
1646 xsyntaxErrorf("starttls not announced")
1650 if n := c.br.Buffered(); n > 0 {
1651 buf := make([]byte, n)
1652 _, err := io.ReadFull(c.br, buf)
1653 xcheckf(err, "reading buffered data for tls handshake")
1654 conn = &prefixConn{buf, conn}
1656 // We add the cid to facilitate debugging in case of TLS connection failure.
1657 c.ok(tag, cmd+" ("+mox.ReceivedID(c.cid)+")")
1659 c.xtlsHandshakeAndAuthenticate(conn)
1662 // We are not sending unsolicited CAPABILITIES for newly available authentication
1663 // mechanisms, clients can't depend on us sending it and should ask it themselves.
1667// Authenticate using SASL. Supports multiple back and forths between client and
1668// server to finish authentication, unlike LOGIN which is just a single
1669// username/password.
1671// We may already have ambient TLS credentials that have not been activated.
1673// Status: Not authenticated.
1674func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
1678 // For many failed auth attempts, slow down verification attempts.
1679 if c.authFailed > 3 && authFailDelay > 0 {
1680 mox.Sleep(mox.Context, time.Duration(c.authFailed-3)*authFailDelay)
1683 // If authentication fails due to missing derived secrets, we don't hold it against
1684 // the connection. There is no way to indicate server support for an authentication
1685 // mechanism, but that a mechanism won't work for an account.
1686 var missingDerivedSecrets bool
1688 c.authFailed++ // Compensated on success.
1690 if missingDerivedSecrets {
1693 // On the 3rd failed authentication, start responding slowly. Successful auth will
1694 // cause fast responses again.
1695 if c.authFailed >= 3 {
1700 var authVariant string // Only known strings, used in metrics.
1701 authResult := "error"
1703 metrics.AuthenticationInc("imap", authVariant, authResult)
1704 if authResult == "ok" {
1705 mox.LimiterFailedAuth.Reset(c.remoteIP, time.Now())
1706 } else if !missingDerivedSecrets {
1707 mox.LimiterFailedAuth.Add(c.remoteIP, time.Now(), 1)
1713 authType := p.xatom()
1715 xreadInitial := func() []byte {
1719 line = c.readline(false)
1723 line = p.remainder()
1726 line = "" // Base64 decode will result in empty buffer.
1731 authResult = "aborted"
1732 xsyntaxErrorf("authenticate aborted by client")
1734 buf, err := base64.StdEncoding.DecodeString(line)
1736 xsyntaxErrorf("parsing base64: %v", err)
1741 xreadContinuation := func() []byte {
1742 line := c.readline(false)
1744 authResult = "aborted"
1745 xsyntaxErrorf("authenticate aborted by client")
1747 buf, err := base64.StdEncoding.DecodeString(line)
1749 xsyntaxErrorf("parsing base64: %v", err)
1754 // The various authentication mechanisms set account and username. We may already
1755 // have an account and username from TLS client authentication. Afterwards, we
1756 // check that the account is the same.
1757 var account *store.Account
1761 err := account.Close()
1762 c.xsanity(err, "close account")
1766 switch strings.ToUpper(authType) {
1768 authVariant = "plain"
1770 if !c.noRequireSTARTTLS && !c.tls {
1772 xusercodeErrorf("PRIVACYREQUIRED", "tls required for login")
1775 // Plain text passwords, mark as traceauth.
1776 defer c.xtrace(mlog.LevelTraceauth)()
1777 buf := xreadInitial()
1778 c.xtrace(mlog.LevelTrace) // Restore.
1779 plain := bytes.Split(buf, []byte{0})
1780 if len(plain) != 3 {
1781 xsyntaxErrorf("bad plain auth data, expected 3 nul-separated tokens, got %d tokens", len(plain))
1783 authz := string(plain[0])
1784 username = string(plain[1])
1785 password := string(plain[2])
1787 if authz != "" && authz != username {
1788 xusercodeErrorf("AUTHORIZATIONFAILED", "cannot assume role")
1792 account, err = store.OpenEmailAuth(c.log, username, password)
1794 if errors.Is(err, store.ErrUnknownCredentials) {
1795 authResult = "badcreds"
1796 c.log.Info("authentication failed", slog.String("username", username))
1797 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
1799 xusercodeErrorf("", "error")
1803 authVariant = strings.ToLower(authType)
1809 chal := fmt.Sprintf("<%d.%d@%s>", uint64(mox.CryptoRandInt()), time.Now().UnixNano(), mox.Conf.Static.HostnameDomain.ASCII)
1810 c.writelinef("+ %s", base64.StdEncoding.EncodeToString([]byte(chal)))
1812 resp := xreadContinuation()
1813 t := strings.Split(string(resp), " ")
1814 if len(t) != 2 || len(t[1]) != 2*md5.Size {
1815 xsyntaxErrorf("malformed cram-md5 response")
1818 c.log.Debug("cram-md5 auth", slog.String("address", username))
1820 account, _, err = store.OpenEmail(c.log, username)
1822 if errors.Is(err, store.ErrUnknownCredentials) {
1823 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
1824 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
1826 xserverErrorf("looking up address: %v", err)
1828 var ipadhash, opadhash hash.Hash
1829 account.WithRLock(func() {
1830 err := account.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
1831 password, err := bstore.QueryTx[store.Password](tx).Get()
1832 if err == bstore.ErrAbsent {
1833 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
1834 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
1840 ipadhash = password.CRAMMD5.Ipad
1841 opadhash = password.CRAMMD5.Opad
1844 xcheckf(err, "tx read")
1846 if ipadhash == nil || opadhash == nil {
1847 c.log.Info("cram-md5 auth attempt without derived secrets set, save password again to store secrets", slog.String("username", username))
1848 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
1849 missingDerivedSecrets = true
1850 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
1854 ipadhash.Write([]byte(chal))
1855 opadhash.Write(ipadhash.Sum(nil))
1856 digest := fmt.Sprintf("%x", opadhash.Sum(nil))
1858 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
1859 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
1862 case "SCRAM-SHA-256-PLUS", "SCRAM-SHA-256", "SCRAM-SHA-1-PLUS", "SCRAM-SHA-1":
1863 // todo: improve handling of errors during scram. e.g. invalid parameters. should we abort the imap command, or continue until the end and respond with a scram-level error?
1864 // todo: use single implementation between ../imapserver/server.go and ../smtpserver/server.go
1866 // No plaintext credentials, we can log these normally.
1868 authVariant = strings.ToLower(authType)
1869 var h func() hash.Hash
1870 switch authVariant {
1871 case "scram-sha-1", "scram-sha-1-plus":
1873 case "scram-sha-256", "scram-sha-256-plus":
1876 xserverErrorf("missing case for scram variant")
1879 var cs *tls.ConnectionState
1880 requireChannelBinding := strings.HasSuffix(authVariant, "-plus")
1881 if requireChannelBinding && !c.tls {
1882 xuserErrorf("cannot use plus variant with tls channel binding without tls")
1885 xcs := c.conn.(*tls.Conn).ConnectionState()
1888 c0 := xreadInitial()
1889 ss, err := scram.NewServer(h, c0, cs, requireChannelBinding)
1891 c.log.Infox("scram protocol error", err, slog.Any("remote", c.remoteIP))
1892 xuserErrorf("scram protocol error: %s", err)
1894 username = ss.Authentication
1895 c.log.Debug("scram auth", slog.String("authentication", username))
1896 account, _, err = store.OpenEmail(c.log, username)
1898 // todo: we could continue scram with a generated salt, deterministically generated
1899 // from the username. that way we don't have to store anything but attackers cannot
1900 // learn if an account exists. same for absent scram saltedpassword below.
1901 xuserErrorf("scram not possible")
1903 if ss.Authorization != "" && ss.Authorization != username {
1904 xuserErrorf("authentication with authorization for different user not supported")
1906 var xscram store.SCRAM
1907 account.WithRLock(func() {
1908 err := account.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
1909 password, err := bstore.QueryTx[store.Password](tx).Get()
1910 if err == bstore.ErrAbsent {
1911 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
1912 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
1914 xcheckf(err, "fetching credentials")
1915 switch authVariant {
1916 case "scram-sha-1", "scram-sha-1-plus":
1917 xscram = password.SCRAMSHA1
1918 case "scram-sha-256", "scram-sha-256-plus":
1919 xscram = password.SCRAMSHA256
1921 xserverErrorf("missing case for scram credentials")
1923 if len(xscram.Salt) == 0 || xscram.Iterations == 0 || len(xscram.SaltedPassword) == 0 {
1924 missingDerivedSecrets = true
1925 c.log.Info("scram auth attempt without derived secrets set, save password again to store secrets", slog.String("username", username))
1926 xuserErrorf("scram not possible")
1930 xcheckf(err, "read tx")
1932 s1, err := ss.ServerFirst(xscram.Iterations, xscram.Salt)
1933 xcheckf(err, "scram first server step")
1934 c.writelinef("+ %s", base64.StdEncoding.EncodeToString([]byte(s1)))
1935 c2 := xreadContinuation()
1936 s3, err := ss.Finish(c2, xscram.SaltedPassword)
1938 c.writelinef("+ %s", base64.StdEncoding.EncodeToString([]byte(s3)))
1941 c.readline(false) // Should be "*" for cancellation.
1942 if errors.Is(err, scram.ErrInvalidProof) {
1943 authResult = "badcreds"
1944 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
1945 xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
1946 } else if errors.Is(err, scram.ErrChannelBindingsDontMatch) {
1947 authResult = "badchanbind"
1948 c.log.Warn("bad channel binding during authentication, potential mitm", slog.String("username", username), slog.Any("remote", c.remoteIP))
1949 xusercodeErrorf("AUTHENTICATIONFAILED", "channel bindings do not match, potential mitm")
1950 } else if errors.Is(err, scram.ErrInvalidEncoding) {
1951 c.log.Infox("bad scram protocol message", err, slog.String("username", username), slog.Any("remote", c.remoteIP))
1952 xuserErrorf("bad scram protocol message: %s", err)
1954 xuserErrorf("server final: %w", err)
1958 // The message should be empty. todo: should we require it is empty?
1962 authVariant = strings.ToLower(authType)
1965 buf := xreadInitial()
1966 username = string(buf)
1969 xusercodeErrorf("AUTHENTICATIONFAILED", "tls required for tls client certificate authentication")
1971 if c.account == nil {
1972 xusercodeErrorf("AUTHENTICATIONFAILED", "missing client certificate, required for tls client certificate authentication")
1976 username = c.username
1979 account, _, err = store.OpenEmail(c.log, username)
1980 xcheckf(err, "looking up username from tls client authentication")
1983 xuserErrorf("method not supported")
1986 // We may already have TLS credentials. They won't have been enabled, or we could
1987 // get here due to the state machine that doesn't allow authentication while being
1988 // authenticated. But allow another SASL authentication, but it has to be for the
1989 // same account. It can be for a different username (email address) of the account.
1990 if c.account != nil {
1991 if account != c.account {
1992 c.log.Debug("sasl authentication for different account than tls client authentication, aborting connection",
1993 slog.String("saslmechanism", authVariant),
1994 slog.String("saslaccount", account.Name),
1995 slog.String("tlsaccount", c.account.Name),
1996 slog.String("saslusername", username),
1997 slog.String("tlsusername", c.username),
1999 xusercodeErrorf("AUTHENTICATIONFAILED", "authentication failed, tls client certificate public key belongs to another account")
2000 } else if username != c.username {
2001 c.log.Debug("sasl authentication for different username than tls client certificate authentication, switching to sasl username",
2002 slog.String("saslmechanism", authVariant),
2003 slog.String("saslusername", username),
2004 slog.String("tlsusername", c.username),
2005 slog.String("account", c.account.Name),
2010 account = nil // Prevent cleanup.
2012 c.username = username
2014 c.comm = store.RegisterComm(c.account)
2020 c.state = stateAuthenticated
2021 c.writeresultf("%s OK [CAPABILITY %s] authenticate done", tag, c.capabilities())
2024// Login logs in with username and password.
2026// Status: Not authenticated.
2027func (c *conn) cmdLogin(tag, cmd string, p *parser) {
2030 authResult := "error"
2032 metrics.AuthenticationInc("imap", "login", authResult)
2033 if authResult == "ok" {
2034 mox.LimiterFailedAuth.Reset(c.remoteIP, time.Now())
2036 mox.LimiterFailedAuth.Add(c.remoteIP, time.Now(), 1)
2040 // todo: get this line logged with traceauth. the plaintext password is included on the command line, which we've already read (before dispatching to this function).
2044 username := p.xastring()
2046 password := p.xastring()
2049 if !c.noRequireSTARTTLS && !c.tls {
2051 xusercodeErrorf("PRIVACYREQUIRED", "tls required for login")
2054 // For many failed auth attempts, slow down verification attempts.
2055 if c.authFailed > 3 && authFailDelay > 0 {
2056 mox.Sleep(mox.Context, time.Duration(c.authFailed-3)*authFailDelay)
2058 c.authFailed++ // Compensated on success.
2060 // On the 3rd failed authentication, start responding slowly. Successful auth will
2061 // cause fast responses again.
2062 if c.authFailed >= 3 {
2067 account, err := store.OpenEmailAuth(c.log, username, password)
2069 authResult = "badcreds"
2071 if errors.Is(err, store.ErrUnknownCredentials) {
2072 code = "AUTHENTICATIONFAILED"
2073 c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
2075 xusercodeErrorf(code, "login failed")
2079 err := account.Close()
2080 c.log.Check(err, "close account")
2084 // We may already have TLS credentials. They won't have been enabled, or we could
2085 // get here due to the state machine that doesn't allow authentication while being
2086 // authenticated. But allow another SASL authentication, but it has to be for the
2087 // same account. It can be for a different username (email address) of the account.
2088 if c.account != nil {
2089 if account != c.account {
2090 c.log.Debug("sasl authentication for different account than tls client authentication, aborting connection",
2091 slog.String("saslmechanism", "login"),
2092 slog.String("saslaccount", account.Name),
2093 slog.String("tlsaccount", c.account.Name),
2094 slog.String("saslusername", username),
2095 slog.String("tlsusername", c.username),
2097 xusercodeErrorf("AUTHENTICATIONFAILED", "authentication failed, tls client certificate public key belongs to another account")
2098 } else if username != c.username {
2099 c.log.Debug("sasl authentication for different username than tls client certificate authentication, switching to sasl username",
2100 slog.String("saslmechanism", "login"),
2101 slog.String("saslusername", username),
2102 slog.String("tlsusername", c.username),
2103 slog.String("account", c.account.Name),
2108 account = nil // Prevent cleanup.
2110 c.username = username
2112 c.comm = store.RegisterComm(c.account)
2116 c.state = stateAuthenticated
2118 c.writeresultf("%s OK [CAPABILITY %s] login done", tag, c.capabilities())
2121// Enable explicitly opts in to an extension. A server can typically send new kinds
2122// of responses to a client. Most extensions do not require an ENABLE because a
2123// client implicitly opts in to new response syntax by making a requests that uses
2124// new optional extension request syntax.
2126// State: Authenticated and selected.
2127func (c *conn) cmdEnable(tag, cmd string, p *parser) {
2133 caps := []string{p.xatom()}
2136 caps = append(caps, p.xatom())
2139 // Clients should only send capabilities that need enabling.
2140 // We should only echo that we recognize as needing enabling.
2143 for _, s := range caps {
2144 cap := capability(strings.ToUpper(s))
2149 c.enabled[cap] = true
2152 c.enabled[cap] = true
2158 if qresync && !c.enabled[capCondstore] {
2159 c.xensureCondstore(nil)
2160 enabled += " CONDSTORE"
2164 c.bwritelinef("* ENABLED%s", enabled)
2169// If a mailbox is selected, an untagged OK with HIGHESTMODSEQ is written to the
2170// client. If tx is non-nil, it is used to read the HIGHESTMODSEQ from the
2171// database. Otherwise a new read-only transaction is created.
2172func (c *conn) xensureCondstore(tx *bstore.Tx) {
2173 if !c.enabled[capCondstore] {
2174 c.enabled[capCondstore] = true
2175 // todo spec: can we send an untagged enabled response?
2177 if c.mailboxID <= 0 {
2180 var modseq store.ModSeq
2182 modseq = c.xhighestModSeq(tx, c.mailboxID)
2184 c.xdbread(func(tx *bstore.Tx) {
2185 modseq = c.xhighestModSeq(tx, c.mailboxID)
2188 c.bwritelinef("* OK [HIGHESTMODSEQ %d] after condstore-enabling command", modseq.Client())
2192// State: Authenticated and selected.
2193func (c *conn) cmdSelect(tag, cmd string, p *parser) {
2194 c.cmdSelectExamine(true, tag, cmd, p)
2197// State: Authenticated and selected.
2198func (c *conn) cmdExamine(tag, cmd string, p *parser) {
2199 c.cmdSelectExamine(false, tag, cmd, p)
2202// Select and examine are almost the same commands. Select just opens a mailbox for
2203// read/write and examine opens a mailbox readonly.
2205// State: Authenticated and selected.
2206func (c *conn) cmdSelectExamine(isselect bool, tag, cmd string, p *parser) {
2214 name := p.xmailbox()
2216 var qruidvalidity uint32
2217 var qrmodseq int64 // QRESYNC required parameters.
2218 var qrknownUIDs, qrknownSeqSet, qrknownUIDSet *numSet // QRESYNC optional parameters.
2220 seen := map[string]bool{}
2222 for len(seen) == 0 || !p.take(")") {
2223 w := p.xtakelist("CONDSTORE", "QRESYNC")
2225 xsyntaxErrorf("duplicate select parameter %s", w)
2235 // Note: unlike with CONDSTORE, there are no QRESYNC-related commands/parameters
2236 // that enable capabilities.
2237 if !c.enabled[capQresync] {
2239 xsyntaxErrorf("QRESYNC must first be enabled")
2245 qrmodseq = p.xnznumber64()
2247 seqMatchData := p.take("(")
2251 seqMatchData = p.take(" (")
2254 ss0 := p.xnumSet0(false, false)
2255 qrknownSeqSet = &ss0
2257 ss1 := p.xnumSet0(false, false)
2258 qrknownUIDSet = &ss1
2264 panic("missing case for select param " + w)
2270 // Deselect before attempting the new select. This means we will deselect when an
2271 // error occurs during select.
2273 if c.state == stateSelected {
2275 c.bwritelinef("* OK [CLOSED] x")
2279 name = xcheckmailboxname(name, true)
2281 var highestModSeq store.ModSeq
2282 var highDeletedModSeq store.ModSeq
2283 var firstUnseen msgseq = 0
2284 var mb store.Mailbox
2285 c.account.WithRLock(func() {
2286 c.xdbread(func(tx *bstore.Tx) {
2287 mb = c.xmailbox(tx, name, "")
2289 q := bstore.QueryTx[store.Message](tx)
2290 q.FilterNonzero(store.Message{MailboxID: mb.ID})
2291 q.FilterEqual("Expunged", false)
2293 c.uids = []store.UID{}
2295 err := q.ForEach(func(m store.Message) error {
2296 c.uids = append(c.uids, m.UID)
2297 if firstUnseen == 0 && !m.Seen {
2306 xcheckf(err, "fetching uids")
2308 // Condstore extension, find the highest modseq.
2309 if c.enabled[capCondstore] {
2310 highestModSeq = c.xhighestModSeq(tx, mb.ID)
2312 // For QRESYNC, we need to know the highest modset of deleted expunged records to
2313 // maintain synchronization.
2314 if c.enabled[capQresync] {
2315 highDeletedModSeq, err = c.account.HighestDeletedModSeq(tx)
2316 xcheckf(err, "getting highest deleted modseq")
2320 c.applyChanges(c.comm.Get(), true)
2323 if len(mb.Keywords) > 0 {
2324 flags = " " + strings.Join(mb.Keywords, " ")
2326 c.bwritelinef(`* FLAGS (\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent%s)`, flags)
2327 c.bwritelinef(`* OK [PERMANENTFLAGS (\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent \*)] x`)
2328 if !c.enabled[capIMAP4rev2] {
2329 c.bwritelinef(`* 0 RECENT`)
2331 c.bwritelinef(`* %d EXISTS`, len(c.uids))
2332 if !c.enabled[capIMAP4rev2] && firstUnseen > 0 {
2334 c.bwritelinef(`* OK [UNSEEN %d] x`, firstUnseen)
2336 c.bwritelinef(`* OK [UIDVALIDITY %d] x`, mb.UIDValidity)
2337 c.bwritelinef(`* OK [UIDNEXT %d] x`, mb.UIDNext)
2338 c.bwritelinef(`* LIST () "/" %s`, astring(c.encodeMailbox(mb.Name)).pack(c))
2339 if c.enabled[capCondstore] {
2342 c.bwritelinef(`* OK [HIGHESTMODSEQ %d] x`, highestModSeq.Client())
2346 if qruidvalidity == mb.UIDValidity {
2347 // We send the vanished UIDs at the end, so we can easily combine the modseq
2348 // changes and vanished UIDs that result from that, with the vanished UIDs from the
2349 // case where we don't store enough history.
2350 vanishedUIDs := map[store.UID]struct{}{}
2352 var preVanished store.UID
2353 var oldClientUID store.UID
2354 // If samples of known msgseq and uid pairs are given (they must be in order), we
2355 // use them to determine the earliest UID for which we send VANISHED responses.
2357 if qrknownSeqSet != nil {
2358 if !qrknownSeqSet.isBasicIncreasing() {
2359 xuserErrorf("QRESYNC known message sequence set must be numeric and strictly increasing")
2361 if !qrknownUIDSet.isBasicIncreasing() {
2362 xuserErrorf("QRESYNC known uid set must be numeric and strictly increasing")
2364 seqiter := qrknownSeqSet.newIter()
2365 uiditer := qrknownUIDSet.newIter()
2367 msgseq, ok0 := seqiter.Next()
2368 uid, ok1 := uiditer.Next()
2371 } else if !ok0 || !ok1 {
2372 xsyntaxErrorf("invalid combination of known sequence set and uid set, must be of equal length")
2374 i := int(msgseq - 1)
2375 if i < 0 || i >= len(c.uids) || c.uids[i] != store.UID(uid) {
2376 if uidSearch(c.uids, store.UID(uid)) <= 0 {
2377 // We will check this old client UID for consistency below.
2378 oldClientUID = store.UID(uid)
2382 preVanished = store.UID(uid + 1)
2386 // We gather vanished UIDs and report them at the end. This seems OK because we
2387 // already sent HIGHESTMODSEQ, and a client should know not to commit that value
2388 // until after it has seen the tagged OK of this command. The RFC has a remark
2389 // about ordering of some untagged responses, it's not immediately clear what it
2390 // means, but given the examples appears to allude to servers that decide to not
2391 // send expunge/vanished before the tagged OK.
2394 // We are reading without account lock. Similar to when we process FETCH/SEARCH
2395 // requests. We don't have to reverify existence of the mailbox, so we don't
2396 // rlock, even briefly.
2397 c.xdbread(func(tx *bstore.Tx) {
2398 if oldClientUID > 0 {
2399 // The client sent a UID that is now removed. This is typically fine. But we check
2400 // that it is consistent with the modseq the client sent. If the UID already didn't
2401 // exist at that modseq, the client may be missing some information.
2402 q := bstore.QueryTx[store.Message](tx)
2403 q.FilterNonzero(store.Message{MailboxID: mb.ID, UID: oldClientUID})
2406 // If client claims to be up to date up to and including qrmodseq, and the message
2407 // was deleted at or before that time, we send changes from just before that
2408 // modseq, and we send vanished for all UIDs.
2409 if m.Expunged && qrmodseq >= m.ModSeq.Client() {
2410 qrmodseq = m.ModSeq.Client() - 1
2413 c.bwritelinef("* OK [ALERT] Synchronization inconsistency in client detected. Client tried to sync with a UID that was removed at or after the MODSEQ it sent in the request. Sending all historic message removals for selected mailbox. Full synchronization recommended.")
2415 } else if err != bstore.ErrAbsent {
2416 xcheckf(err, "checking old client uid")
2420 q := bstore.QueryTx[store.Message](tx)
2421 q.FilterNonzero(store.Message{MailboxID: mb.ID})
2422 // Note: we don't filter by Expunged.
2423 q.FilterGreater("ModSeq", store.ModSeqFromClient(qrmodseq))
2424 q.FilterLessEqual("ModSeq", highestModSeq)
2426 err := q.ForEach(func(m store.Message) error {
2427 if m.Expunged && m.UID < preVanished {
2431 if qrknownUIDs != nil && !qrknownUIDs.contains(uint32(m.UID)) {
2435 vanishedUIDs[m.UID] = struct{}{}
2438 msgseq := c.sequence(m.UID)
2440 c.bwritelinef("* %d FETCH (UID %d FLAGS %s MODSEQ (%d))", msgseq, m.UID, flaglist(m.Flags, m.Keywords).pack(c), m.ModSeq.Client())
2444 xcheckf(err, "listing changed messages")
2447 // Add UIDs from client's known UID set to vanished list if we don't have enough history.
2448 if qrmodseq < highDeletedModSeq.Client() {
2449 // If no known uid set was in the request, we substitute 1:max or the empty set.
2451 if qrknownUIDs == nil {
2452 if len(c.uids) > 0 {
2453 qrknownUIDs = &numSet{ranges: []numRange{{first: setNumber{number: 1}, last: &setNumber{number: uint32(c.uids[len(c.uids)-1])}}}}
2455 qrknownUIDs = &numSet{}
2459 iter := qrknownUIDs.newIter()
2461 v, ok := iter.Next()
2465 if c.sequence(store.UID(v)) <= 0 {
2466 vanishedUIDs[store.UID(v)] = struct{}{}
2471 // Now that we have all vanished UIDs, send them over compactly.
2472 if len(vanishedUIDs) > 0 {
2473 l := maps.Keys(vanishedUIDs)
2474 sort.Slice(l, func(i, j int) bool {
2478 for _, s := range compactUIDSet(l).Strings(4*1024 - 32) {
2479 c.bwritelinef("* VANISHED (EARLIER) %s", s)
2485 c.bwriteresultf("%s OK [READ-WRITE] x", tag)
2488 c.bwriteresultf("%s OK [READ-ONLY] x", tag)
2492 c.state = stateSelected
2493 c.searchResult = nil
2497// Create makes a new mailbox, and its parents too if absent.
2499// State: Authenticated and selected.
2500func (c *conn) cmdCreate(tag, cmd string, p *parser) {
2506 name := p.xmailbox()
2512 name = xcheckmailboxname(name, false)
2514 var changes []store.Change
2515 var created []string // Created mailbox names.
2517 c.account.WithWLock(func() {
2518 c.xdbwrite(func(tx *bstore.Tx) {
2521 changes, created, exists, err = c.account.MailboxCreate(tx, name)
2524 xuserErrorf("mailbox already exists")
2526 xcheckf(err, "creating mailbox")
2529 c.broadcast(changes)
2532 for _, n := range created {
2535 if c.enabled[capIMAP4rev2] && n == name && name != origName && !(name == "Inbox" || strings.HasPrefix(name, "Inbox/")) {
2536 oldname = fmt.Sprintf(` ("OLDNAME" (%s))`, string0(c.encodeMailbox(origName)).pack(c))
2538 c.bwritelinef(`* LIST (\Subscribed) "/" %s%s`, astring(c.encodeMailbox(n)).pack(c), oldname)
2543// Delete removes a mailbox and all its messages.
2544// Inbox cannot be removed.
2546// State: Authenticated and selected.
2547func (c *conn) cmdDelete(tag, cmd string, p *parser) {
2553 name := p.xmailbox()
2556 name = xcheckmailboxname(name, false)
2558 // Messages to remove after having broadcasted the removal of messages.
2559 var removeMessageIDs []int64
2561 c.account.WithWLock(func() {
2562 var mb store.Mailbox
2563 var changes []store.Change
2565 c.xdbwrite(func(tx *bstore.Tx) {
2566 mb = c.xmailbox(tx, name, "NONEXISTENT")
2568 var hasChildren bool
2570 changes, removeMessageIDs, hasChildren, err = c.account.MailboxDelete(context.TODO(), c.log, tx, mb)
2572 xusercodeErrorf("HASCHILDREN", "mailbox has a child, only leaf mailboxes can be deleted")
2574 xcheckf(err, "deleting mailbox")
2577 c.broadcast(changes)
2580 for _, mID := range removeMessageIDs {
2581 p := c.account.MessagePath(mID)
2583 c.log.Check(err, "removing message file for mailbox delete", slog.String("path", p))
2589// Rename changes the name of a mailbox.
2590// Renaming INBOX is special, it moves the inbox messages to a new mailbox, leaving inbox empty.
2591// Renaming a mailbox with submailboxes also renames all submailboxes.
2592// Subscriptions stay with the old name, though newly created missing parent
2593// mailboxes for the destination name are automatically subscribed.
2595// State: Authenticated and selected.
2596func (c *conn) cmdRename(tag, cmd string, p *parser) {
2607 src = xcheckmailboxname(src, true)
2608 dst = xcheckmailboxname(dst, false)
2610 c.account.WithWLock(func() {
2611 var changes []store.Change
2613 c.xdbwrite(func(tx *bstore.Tx) {
2614 srcMB := c.xmailbox(tx, src, "NONEXISTENT")
2616 // Inbox is very special. Unlike other mailboxes, its children are not moved. And
2617 // unlike a regular move, its messages are moved to a newly created mailbox. We do
2618 // indeed create a new destination mailbox and actually move the messages.
2621 exists, err := c.account.MailboxExists(tx, dst)
2622 xcheckf(err, "checking if destination mailbox exists")
2624 xusercodeErrorf("ALREADYEXISTS", "destination mailbox %q already exists", dst)
2627 xuserErrorf("cannot move inbox to itself")
2630 uidval, err := c.account.NextUIDValidity(tx)
2631 xcheckf(err, "next uid validity")
2633 dstMB := store.Mailbox{
2635 UIDValidity: uidval,
2637 Keywords: srcMB.Keywords,
2640 err = tx.Insert(&dstMB)
2641 xcheckf(err, "create new destination mailbox")
2643 modseq, err := c.account.NextModSeq(tx)
2644 xcheckf(err, "assigning next modseq")
2646 changes = make([]store.Change, 2) // Placeholders filled in below.
2648 // Move existing messages, with their ID's and on-disk files intact, to the new
2649 // mailbox. We keep the expunged messages, the destination mailbox doesn't care
2651 var oldUIDs []store.UID
2652 q := bstore.QueryTx[store.Message](tx)
2653 q.FilterNonzero(store.Message{MailboxID: srcMB.ID})
2654 q.FilterEqual("Expunged", false)
2656 err = q.ForEach(func(m store.Message) error {
2661 oldUIDs = append(oldUIDs, om.UID)
2663 mc := m.MailboxCounts()
2667 m.MailboxID = dstMB.ID
2668 m.UID = dstMB.UIDNext
2670 m.CreateSeq = modseq
2672 if err := tx.Update(&m); err != nil {
2673 return fmt.Errorf("updating message to move to new mailbox: %w", err)
2676 changes = append(changes, m.ChangeAddUID())
2678 if err := tx.Insert(&om); err != nil {
2679 return fmt.Errorf("adding empty expunge message record to inbox: %w", err)
2683 xcheckf(err, "moving messages from inbox to destination mailbox")
2685 err = tx.Update(&dstMB)
2686 xcheckf(err, "updating uidnext and counts in destination mailbox")
2688 err = tx.Update(&srcMB)
2689 xcheckf(err, "updating counts for inbox")
2691 var dstFlags []string
2692 if tx.Get(&store.Subscription{Name: dstMB.Name}) == nil {
2693 dstFlags = []string{`\Subscribed`}
2695 changes[0] = store.ChangeRemoveUIDs{MailboxID: srcMB.ID, UIDs: oldUIDs, ModSeq: modseq}
2696 changes[1] = store.ChangeAddMailbox{Mailbox: dstMB, Flags: dstFlags}
2697 // changes[2:...] are ChangeAddUIDs
2698 changes = append(changes, srcMB.ChangeCounts(), dstMB.ChangeCounts())
2702 var notExists, alreadyExists bool
2704 changes, _, notExists, alreadyExists, err = c.account.MailboxRename(tx, srcMB, dst)
2707 xusercodeErrorf("NONEXISTENT", "%s", err)
2708 } else if alreadyExists {
2709 xusercodeErrorf("ALREADYEXISTS", "%s", err)
2711 xcheckf(err, "renaming mailbox")
2713 c.broadcast(changes)
2719// Subscribe marks a mailbox path as subscribed. The mailbox does not have to
2720// exist. Subscribed may mean an email client will show the mailbox in its UI
2721// and/or periodically fetch new messages for the mailbox.
2723// State: Authenticated and selected.
2724func (c *conn) cmdSubscribe(tag, cmd string, p *parser) {
2730 name := p.xmailbox()
2733 name = xcheckmailboxname(name, true)
2735 c.account.WithWLock(func() {
2736 var changes []store.Change
2738 c.xdbwrite(func(tx *bstore.Tx) {
2740 changes, err = c.account.SubscriptionEnsure(tx, name)
2741 xcheckf(err, "ensuring subscription")
2744 c.broadcast(changes)
2750// Unsubscribe marks a mailbox as not subscribed. The mailbox doesn't have to exist.
2752// State: Authenticated and selected.
2753func (c *conn) cmdUnsubscribe(tag, cmd string, p *parser) {
2759 name := p.xmailbox()
2762 name = xcheckmailboxname(name, true)
2764 c.account.WithWLock(func() {
2765 c.xdbwrite(func(tx *bstore.Tx) {
2767 err := tx.Delete(&store.Subscription{Name: name})
2768 if err == bstore.ErrAbsent {
2769 exists, err := c.account.MailboxExists(tx, name)
2770 xcheckf(err, "checking if mailbox exists")
2772 xuserErrorf("mailbox does not exist")
2776 xcheckf(err, "removing subscription")
2779 // todo: can we send untagged message about a mailbox no longer being subscribed?
2785// LSUB command for listing subscribed mailboxes.
2786// Removed in IMAP4rev2, only in IMAP4rev1.
2788// State: Authenticated and selected.
2789func (c *conn) cmdLsub(tag, cmd string, p *parser) {
2797 pattern := p.xlistMailbox()
2800 re := xmailboxPatternMatcher(ref, []string{pattern})
2803 c.xdbread(func(tx *bstore.Tx) {
2804 q := bstore.QueryTx[store.Subscription](tx)
2806 subscriptions, err := q.List()
2807 xcheckf(err, "querying subscriptions")
2809 have := map[string]bool{}
2810 subscribedKids := map[string]bool{}
2811 ispercent := strings.HasSuffix(pattern, "%")
2812 for _, sub := range subscriptions {
2815 for p := path.Dir(name); p != "."; p = path.Dir(p) {
2816 subscribedKids[p] = true
2819 if !re.MatchString(name) {
2823 line := fmt.Sprintf(`* LSUB () "/" %s`, astring(c.encodeMailbox(name)).pack(c))
2824 lines = append(lines, line)
2832 qmb := bstore.QueryTx[store.Mailbox](tx)
2834 err = qmb.ForEach(func(mb store.Mailbox) error {
2835 if have[mb.Name] || !subscribedKids[mb.Name] || !re.MatchString(mb.Name) {
2838 line := fmt.Sprintf(`* LSUB (\NoSelect) "/" %s`, astring(c.encodeMailbox(mb.Name)).pack(c))
2839 lines = append(lines, line)
2842 xcheckf(err, "querying mailboxes")
2846 for _, line := range lines {
2847 c.bwritelinef("%s", line)
2852// The namespace command returns the mailbox path separator. We only implement
2853// the personal mailbox hierarchy, no shared/other.
2855// In IMAP4rev2, it was an extension before.
2857// State: Authenticated and selected.
2858func (c *conn) cmdNamespace(tag, cmd string, p *parser) {
2865 c.bwritelinef(`* NAMESPACE (("" "/")) NIL NIL`)
2869// The status command returns information about a mailbox, such as the number of
2870// messages, "uid validity", etc. Nowadays, the extended LIST command can return
2871// the same information about many mailboxes for one command.
2873// State: Authenticated and selected.
2874func (c *conn) cmdStatus(tag, cmd string, p *parser) {
2880 name := p.xmailbox()
2883 attrs := []string{p.xstatusAtt()}
2886 attrs = append(attrs, p.xstatusAtt())
2890 name = xcheckmailboxname(name, true)
2892 var mb store.Mailbox
2894 var responseLine string
2895 c.account.WithRLock(func() {
2896 c.xdbread(func(tx *bstore.Tx) {
2897 mb = c.xmailbox(tx, name, "")
2898 responseLine = c.xstatusLine(tx, mb, attrs)
2902 c.bwritelinef("%s", responseLine)
2907func (c *conn) xstatusLine(tx *bstore.Tx, mb store.Mailbox, attrs []string) string {
2908 status := []string{}
2909 for _, a := range attrs {
2910 A := strings.ToUpper(a)
2913 status = append(status, A, fmt.Sprintf("%d", mb.Total+mb.Deleted))
2915 status = append(status, A, fmt.Sprintf("%d", mb.UIDNext))
2917 status = append(status, A, fmt.Sprintf("%d", mb.UIDValidity))
2919 status = append(status, A, fmt.Sprintf("%d", mb.Unseen))
2921 status = append(status, A, fmt.Sprintf("%d", mb.Deleted))
2923 status = append(status, A, fmt.Sprintf("%d", mb.Size))
2925 status = append(status, A, "0")
2928 status = append(status, A, "NIL")
2929 case "HIGHESTMODSEQ":
2931 status = append(status, A, fmt.Sprintf("%d", c.xhighestModSeq(tx, mb.ID).Client()))
2932 case "DELETED-STORAGE":
2934 // How much storage space could be reclaimed by expunging messages with the
2935 // \Deleted flag. We could keep track of this number and return it efficiently.
2936 // Calculating it each time can be slow, and we don't know if clients request it.
2937 // Clients are not likely to set the deleted flag without immediately expunging
2938 // nowadays. Let's wait for something to need it to go through the trouble, and
2939 // always return 0 for now.
2940 status = append(status, A, "0")
2942 xsyntaxErrorf("unknown attribute %q", a)
2945 return fmt.Sprintf("* STATUS %s (%s)", astring(c.encodeMailbox(mb.Name)).pack(c), strings.Join(status, " "))
2948func flaglist(fl store.Flags, keywords []string) listspace {
2950 flag := func(v bool, s string) {
2952 l = append(l, bare(s))
2955 flag(fl.Seen, `\Seen`)
2956 flag(fl.Answered, `\Answered`)
2957 flag(fl.Flagged, `\Flagged`)
2958 flag(fl.Deleted, `\Deleted`)
2959 flag(fl.Draft, `\Draft`)
2960 flag(fl.Forwarded, `$Forwarded`)
2961 flag(fl.Junk, `$Junk`)
2962 flag(fl.Notjunk, `$NotJunk`)
2963 flag(fl.Phishing, `$Phishing`)
2964 flag(fl.MDNSent, `$MDNSent`)
2965 for _, k := range keywords {
2966 l = append(l, bare(k))
2971// Append adds a message to a mailbox.
2973// State: Authenticated and selected.
2974func (c *conn) cmdAppend(tag, cmd string, p *parser) {
2980 name := p.xmailbox()
2982 var storeFlags store.Flags
2983 var keywords []string
2984 if p.hasPrefix("(") {
2985 // Error must be a syntax error, to properly abort the connection due to literal.
2987 storeFlags, keywords, err = store.ParseFlagsKeywords(p.xflagList())
2989 xsyntaxErrorf("parsing flags: %v", err)
2994 if p.hasPrefix(`"`) {
3000 // todo: only with utf8 should we we accept message headers with utf-8. we currently always accept them.
3001 // todo: this is only relevant if we also support the CATENATE extension?
3003 utf8 := p.take("UTF8 (")
3004 size, sync := p.xliteralSize(utf8, false)
3006 name = xcheckmailboxname(name, true)
3007 c.xdbread(func(tx *bstore.Tx) {
3008 c.xmailbox(tx, name, "TRYCREATE")
3014 // Read the message into a temporary file.
3015 msgFile, err := store.CreateMessageTemp(c.log, "imap-append")
3016 xcheckf(err, "creating temp file for message")
3019 err := msgFile.Close()
3020 c.xsanity(err, "closing APPEND temporary file")
3022 c.xsanity(err, "removing APPEND temporary file")
3024 defer c.xtrace(mlog.LevelTracedata)()
3025 mw := message.NewWriter(msgFile)
3026 msize, err := io.Copy(mw, io.LimitReader(c.br, size))
3027 c.xtrace(mlog.LevelTrace) // Restore.
3029 // Cannot use xcheckf due to %w handling of errIO.
3030 panic(fmt.Errorf("reading literal message: %s (%w)", err, errIO))
3033 xserverErrorf("read %d bytes for message, expected %d (%w)", msize, size, errIO)
3037 line := c.readline(false)
3038 np := newParser(line, c)
3042 line := c.readline(false)
3043 np := newParser(line, c)
3048 name = xcheckmailboxname(name, true)
3051 var mb store.Mailbox
3053 var pendingChanges []store.Change
3055 c.account.WithWLock(func() {
3056 var changes []store.Change
3057 c.xdbwrite(func(tx *bstore.Tx) {
3058 mb = c.xmailbox(tx, name, "TRYCREATE")
3060 // Ensure keywords are stored in mailbox.
3061 var mbKwChanged bool
3062 mb.Keywords, mbKwChanged = store.MergeKeywords(mb.Keywords, keywords)
3064 changes = append(changes, mb.ChangeKeywords())
3069 MailboxOrigID: mb.ID,
3076 ok, maxSize, err := c.account.CanAddMessageSize(tx, m.Size)
3077 xcheckf(err, "checking quota")
3080 xusercodeErrorf("OVERQUOTA", "account over maximum total message size %d", maxSize)
3083 mb.Add(m.MailboxCounts())
3085 // Update mailbox before delivering, which updates uidnext which we mustn't overwrite.
3086 err = tx.Update(&mb)
3087 xcheckf(err, "updating mailbox counts")
3089 err = c.account.DeliverMessage(c.log, tx, &m, msgFile, true, false, false, true)
3090 xcheckf(err, "delivering message")
3093 // Fetch pending changes, possibly with new UIDs, so we can apply them before adding our own new UID.
3095 pendingChanges = c.comm.Get()
3098 // Broadcast the change to other connections.
3099 changes = append(changes, m.ChangeAddUID(), mb.ChangeCounts())
3100 c.broadcast(changes)
3103 if c.mailboxID == mb.ID {
3104 c.applyChanges(pendingChanges, false)
3106 // todo spec: with condstore/qresync, is there a mechanism to the client know the modseq for the appended uid? in theory an untagged fetch with the modseq after the OK APPENDUID could make sense, but this probably isn't allowed.
3107 c.bwritelinef("* %d EXISTS", len(c.uids))
3110 c.writeresultf("%s OK [APPENDUID %d %d] appended", tag, mb.UIDValidity, m.UID)
3113// Idle makes a client wait until the server sends untagged updates, e.g. about
3114// message delivery or mailbox create/rename/delete/subscription, etc. It allows a
3115// client to get updates in real-time, not needing the use for NOOP.
3117// State: Authenticated and selected.
3118func (c *conn) cmdIdle(tag, cmd string, p *parser) {
3125 c.writelinef("+ waiting")
3131 case le := <-c.lineChan():
3133 xcheckf(le.err, "get line")
3136 case <-c.comm.Pending:
3137 c.applyChanges(c.comm.Get(), false)
3139 case <-mox.Shutdown.Done():
3141 c.writelinef("* BYE shutting down")
3146 // Reset the write deadline. In case of little activity, with a command timeout of
3147 // 30 minutes, we have likely passed it.
3148 err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Minute))
3149 c.log.Check(err, "setting write deadline")
3151 if strings.ToUpper(line) != "DONE" {
3152 // We just close the connection because our protocols are out of sync.
3153 panic(fmt.Errorf("%w: in IDLE, expected DONE", errIO))
3159// Return the quota root for a mailbox name and any current quota's.
3161// State: Authenticated and selected.
3162func (c *conn) cmdGetquotaroot(tag, cmd string, p *parser) {
3167 name := p.xmailbox()
3170 // This mailbox does not have to exist. Caller just wants to know which limits
3171 // would apply. We only have one limit, so we don't use the name otherwise.
3173 name = xcheckmailboxname(name, true)
3175 // Get current usage for account.
3176 var quota, size int64 // Account only has a quota if > 0.
3177 c.account.WithRLock(func() {
3178 quota = c.account.QuotaMessageSize()
3180 c.xdbread(func(tx *bstore.Tx) {
3181 du := store.DiskUsage{ID: 1}
3183 xcheckf(err, "gather used quota")
3184 size = du.MessageSize
3189 // We only have one per account quota, we name it "" like the examples in the RFC.
3191 c.bwritelinef(`* QUOTAROOT %s ""`, astring(name).pack(c))
3193 // We only write the quota response if there is a limit. The syntax doesn't allow
3194 // an empty list, so we cannot send the current disk usage if there is no limit.
3197 c.bwritelinef(`* QUOTA "" (STORAGE %d %d)`, (size+1024-1)/1024, (quota+1024-1)/1024)
3202// Return the quota for a quota root.
3204// State: Authenticated and selected.
3205func (c *conn) cmdGetquota(tag, cmd string, p *parser) {
3210 root := p.xastring()
3213 // We only have a per-account root called "".
3215 xuserErrorf("unknown quota root")
3218 var quota, size int64
3219 c.account.WithRLock(func() {
3220 quota = c.account.QuotaMessageSize()
3222 c.xdbread(func(tx *bstore.Tx) {
3223 du := store.DiskUsage{ID: 1}
3225 xcheckf(err, "gather used quota")
3226 size = du.MessageSize
3231 // We only write the quota response if there is a limit. The syntax doesn't allow
3232 // an empty list, so we cannot send the current disk usage if there is no limit.
3235 c.bwritelinef(`* QUOTA "" (STORAGE %d %d)`, (size+1024-1)/1024, (quota+1024-1)/1024)
3240// Check is an old deprecated command that is supposed to execute some mailbox consistency checks.
3243func (c *conn) cmdCheck(tag, cmd string, p *parser) {
3249 c.account.WithRLock(func() {
3250 c.xdbread(func(tx *bstore.Tx) {
3251 c.xmailboxID(tx, c.mailboxID) // Validate.
3258// Close undoes select/examine, closing the currently opened mailbox and deleting
3259// messages that were marked for deletion with the \Deleted flag.
3262func (c *conn) cmdClose(tag, cmd string, p *parser) {
3274 remove, _ := c.xexpunge(nil, true)
3277 for _, m := range remove {
3278 p := c.account.MessagePath(m.ID)
3280 c.xsanity(err, "removing message file for expunge for close")
3288// expunge messages marked for deletion in currently selected/active mailbox.
3289// if uidSet is not nil, only messages matching the set are deleted.
3291// messages that have been marked expunged from the database are returned, but the
3292// corresponding files still have to be removed.
3294// the highest modseq in the mailbox is returned, typically associated with the
3295// removal of the messages, but if no messages were expunged the current latest max
3296// modseq for the mailbox is returned.
3297func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) (remove []store.Message, highestModSeq store.ModSeq) {
3298 var modseq store.ModSeq
3300 c.account.WithWLock(func() {
3301 var mb store.Mailbox
3303 c.xdbwrite(func(tx *bstore.Tx) {
3304 mb = store.Mailbox{ID: c.mailboxID}
3306 if err == bstore.ErrAbsent {
3307 if missingMailboxOK {
3310 xuserErrorf("%w", store.ErrUnknownMailbox)
3313 qm := bstore.QueryTx[store.Message](tx)
3314 qm.FilterNonzero(store.Message{MailboxID: c.mailboxID})
3315 qm.FilterEqual("Deleted", true)
3316 qm.FilterEqual("Expunged", false)
3317 qm.FilterFn(func(m store.Message) bool {
3318 // Only remove if this session knows about the message and if present in optional uidSet.
3319 return uidSearch(c.uids, m.UID) > 0 && (uidSet == nil || uidSet.containsUID(m.UID, c.uids, c.searchResult))
3322 remove, err = qm.List()
3323 xcheckf(err, "listing messages to delete")
3325 if len(remove) == 0 {
3326 highestModSeq = c.xhighestModSeq(tx, c.mailboxID)
3330 // Assign new modseq.
3331 modseq, err = c.account.NextModSeq(tx)
3332 xcheckf(err, "assigning next modseq")
3333 highestModSeq = modseq
3335 removeIDs := make([]int64, len(remove))
3336 anyIDs := make([]any, len(remove))
3338 for i, m := range remove {
3341 mb.Sub(m.MailboxCounts())
3343 // Update "remove", because RetrainMessage below will save the message.
3344 remove[i].Expunged = true
3345 remove[i].ModSeq = modseq
3347 qmr := bstore.QueryTx[store.Recipient](tx)
3348 qmr.FilterEqual("MessageID", anyIDs...)
3349 _, err = qmr.Delete()
3350 xcheckf(err, "removing message recipients")
3352 qm = bstore.QueryTx[store.Message](tx)
3353 qm.FilterIDs(removeIDs)
3354 n, err := qm.UpdateNonzero(store.Message{Expunged: true, ModSeq: modseq})
3355 if err == nil && n != len(removeIDs) {
3356 err = fmt.Errorf("only %d messages set to expunged, expected %d", n, len(removeIDs))
3358 xcheckf(err, "marking messages marked for deleted as expunged")
3360 err = tx.Update(&mb)
3361 xcheckf(err, "updating mailbox counts")
3363 err = c.account.AddMessageSize(c.log, tx, -totalSize)
3364 xcheckf(err, "updating disk usage")
3366 // Mark expunged messages as not needing training, then retrain them, so if they
3367 // were trained, they get untrained.
3368 for i := range remove {
3369 remove[i].Junk = false
3370 remove[i].Notjunk = false
3372 err = c.account.RetrainMessages(context.TODO(), c.log, tx, remove, true)
3373 xcheckf(err, "untraining expunged messages")
3376 // Broadcast changes to other connections. We may not have actually removed any
3377 // messages, so take care not to send an empty update.
3378 if len(remove) > 0 {
3379 ouids := make([]store.UID, len(remove))
3380 for i, m := range remove {
3383 changes := []store.Change{
3384 store.ChangeRemoveUIDs{MailboxID: c.mailboxID, UIDs: ouids, ModSeq: modseq},
3387 c.broadcast(changes)
3390 return remove, highestModSeq
3393// Unselect is similar to close in that it closes the currently active mailbox, but
3394// it does not remove messages marked for deletion.
3397func (c *conn) cmdUnselect(tag, cmd string, p *parser) {
3407// Expunge deletes messages marked with \Deleted in the currently selected mailbox.
3408// Clients are wiser to use UID EXPUNGE because it allows a UID sequence set to
3409// explicitly opt in to removing specific messages.
3412func (c *conn) cmdExpunge(tag, cmd string, p *parser) {
3419 xuserErrorf("mailbox open in read-only mode")
3422 c.cmdxExpunge(tag, cmd, nil)
3425// UID expunge deletes messages marked with \Deleted in the currently selected
3426// mailbox if they match a UID sequence set.
3429func (c *conn) cmdUIDExpunge(tag, cmd string, p *parser) {
3434 uidSet := p.xnumSet()
3438 xuserErrorf("mailbox open in read-only mode")
3441 c.cmdxExpunge(tag, cmd, &uidSet)
3444// Permanently delete messages for the currently selected/active mailbox. If uidset
3445// is not nil, only those UIDs are removed.
3447func (c *conn) cmdxExpunge(tag, cmd string, uidSet *numSet) {
3450 remove, highestModSeq := c.xexpunge(uidSet, false)
3453 for _, m := range remove {
3454 p := c.account.MessagePath(m.ID)
3456 c.xsanity(err, "removing message file for expunge")
3461 var vanishedUIDs numSet
3462 qresync := c.enabled[capQresync]
3463 for _, m := range remove {
3464 seq := c.xsequence(m.UID)
3465 c.sequenceRemove(seq, m.UID)
3467 vanishedUIDs.append(uint32(m.UID))
3469 c.bwritelinef("* %d EXPUNGE", seq)
3472 if !vanishedUIDs.empty() {
3474 for _, s := range vanishedUIDs.Strings(4*1024 - 32) {
3475 c.bwritelinef("* VANISHED %s", s)
3479 if c.enabled[capCondstore] {
3480 c.writeresultf("%s OK [HIGHESTMODSEQ %d] expunged", tag, highestModSeq.Client())
3487func (c *conn) cmdSearch(tag, cmd string, p *parser) {
3488 c.cmdxSearch(false, tag, cmd, p)
3492func (c *conn) cmdUIDSearch(tag, cmd string, p *parser) {
3493 c.cmdxSearch(true, tag, cmd, p)
3497func (c *conn) cmdFetch(tag, cmd string, p *parser) {
3498 c.cmdxFetch(false, tag, cmd, p)
3502func (c *conn) cmdUIDFetch(tag, cmd string, p *parser) {
3503 c.cmdxFetch(true, tag, cmd, p)
3507func (c *conn) cmdStore(tag, cmd string, p *parser) {
3508 c.cmdxStore(false, tag, cmd, p)
3512func (c *conn) cmdUIDStore(tag, cmd string, p *parser) {
3513 c.cmdxStore(true, tag, cmd, p)
3517func (c *conn) cmdCopy(tag, cmd string, p *parser) {
3518 c.cmdxCopy(false, tag, cmd, p)
3522func (c *conn) cmdUIDCopy(tag, cmd string, p *parser) {
3523 c.cmdxCopy(true, tag, cmd, p)
3527func (c *conn) cmdMove(tag, cmd string, p *parser) {
3528 c.cmdxMove(false, tag, cmd, p)
3532func (c *conn) cmdUIDMove(tag, cmd string, p *parser) {
3533 c.cmdxMove(true, tag, cmd, p)
3536func (c *conn) gatherCopyMoveUIDs(isUID bool, nums numSet) ([]store.UID, []any) {
3537 // Gather uids, then sort so we can return a consistently simple and hard to
3538 // misinterpret COPYUID/MOVEUID response. It seems safer to have UIDs in ascending
3539 // order, because requested uid set of 12:10 is equal to 10:12, so if we would just
3540 // echo whatever the client sends us without reordering, the client can reorder our
3541 // response and interpret it differently than we intended.
3543 uids := c.xnumSetUIDs(isUID, nums)
3544 sort.Slice(uids, func(i, j int) bool {
3545 return uids[i] < uids[j]
3547 uidargs := make([]any, len(uids))
3548 for i, uid := range uids {
3551 return uids, uidargs
3554// Copy copies messages from the currently selected/active mailbox to another named
3558func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) {
3565 name := p.xmailbox()
3568 name = xcheckmailboxname(name, true)
3570 uids, uidargs := c.gatherCopyMoveUIDs(isUID, nums)
3572 // Files that were created during the copy. Remove them if the operation fails.
3573 var createdIDs []int64
3579 for _, id := range createdIDs {
3580 p := c.account.MessagePath(id)
3582 c.xsanity(err, "cleaning up created file")
3587 var mbDst store.Mailbox
3588 var origUIDs, newUIDs []store.UID
3589 var flags []store.Flags
3590 var keywords [][]string
3591 var modseq store.ModSeq // For messages in new mailbox, assigned when first message is copied.
3593 c.account.WithWLock(func() {
3594 var mbKwChanged bool
3596 c.xdbwrite(func(tx *bstore.Tx) {
3597 mbSrc := c.xmailboxID(tx, c.mailboxID) // Validate.
3598 mbDst = c.xmailbox(tx, name, "TRYCREATE")
3599 if mbDst.ID == mbSrc.ID {
3600 xuserErrorf("cannot copy to currently selected mailbox")
3603 if len(uidargs) == 0 {
3604 xuserErrorf("no matching messages to copy")
3608 modseq, err = c.account.NextModSeq(tx)
3609 xcheckf(err, "assigning next modseq")
3611 // Reserve the uids in the destination mailbox.
3612 uidFirst := mbDst.UIDNext
3613 mbDst.UIDNext += store.UID(len(uidargs))
3615 // Fetch messages from database.
3616 q := bstore.QueryTx[store.Message](tx)
3617 q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
3618 q.FilterEqual("UID", uidargs...)
3619 q.FilterEqual("Expunged", false)
3620 xmsgs, err := q.List()
3621 xcheckf(err, "fetching messages")
3623 if len(xmsgs) != len(uidargs) {
3624 xserverErrorf("uid and message mismatch")
3627 // See if quota allows copy.
3629 for _, m := range xmsgs {
3632 if ok, maxSize, err := c.account.CanAddMessageSize(tx, totalSize); err != nil {
3633 xcheckf(err, "checking quota")
3636 xusercodeErrorf("OVERQUOTA", "account over maximum total message size %d", maxSize)
3638 err = c.account.AddMessageSize(c.log, tx, totalSize)
3639 xcheckf(err, "updating disk usage")
3641 msgs := map[store.UID]store.Message{}
3642 for _, m := range xmsgs {
3645 nmsgs := make([]store.Message, len(xmsgs))
3647 conf, _ := c.account.Conf()
3649 mbKeywords := map[string]struct{}{}
3651 // Insert new messages into database.
3652 var origMsgIDs, newMsgIDs []int64
3653 for i, uid := range uids {
3656 xuserErrorf("messages changed, could not fetch requested uid")
3659 origMsgIDs = append(origMsgIDs, origID)
3661 m.UID = uidFirst + store.UID(i)
3662 m.CreateSeq = modseq
3664 m.MailboxID = mbDst.ID
3665 if m.IsReject && m.MailboxDestinedID != 0 {
3666 // Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message
3667 // is used for reputation calculation during future deliveries.
3668 m.MailboxOrigID = m.MailboxDestinedID
3672 m.JunkFlagsForMailbox(mbDst, conf)
3673 err := tx.Insert(&m)
3674 xcheckf(err, "inserting message")
3677 origUIDs = append(origUIDs, uid)
3678 newUIDs = append(newUIDs, m.UID)
3679 newMsgIDs = append(newMsgIDs, m.ID)
3680 flags = append(flags, m.Flags)
3681 keywords = append(keywords, m.Keywords)
3682 for _, kw := range m.Keywords {
3683 mbKeywords[kw] = struct{}{}
3686 qmr := bstore.QueryTx[store.Recipient](tx)
3687 qmr.FilterNonzero(store.Recipient{MessageID: origID})
3688 mrs, err := qmr.List()
3689 xcheckf(err, "listing message recipients")
3690 for _, mr := range mrs {
3693 err := tx.Insert(&mr)
3694 xcheckf(err, "inserting message recipient")
3697 mbDst.Add(m.MailboxCounts())
3700 mbDst.Keywords, mbKwChanged = store.MergeKeywords(mbDst.Keywords, maps.Keys(mbKeywords))
3702 err = tx.Update(&mbDst)
3703 xcheckf(err, "updating destination mailbox for uids, keywords and counts")
3705 // Copy message files to new message ID's.
3706 syncDirs := map[string]struct{}{}
3707 for i := range origMsgIDs {
3708 src := c.account.MessagePath(origMsgIDs[i])
3709 dst := c.account.MessagePath(newMsgIDs[i])
3710 dstdir := filepath.Dir(dst)
3711 if _, ok := syncDirs[dstdir]; !ok {
3712 os.MkdirAll(dstdir, 0770)
3713 syncDirs[dstdir] = struct{}{}
3715 err := moxio.LinkOrCopy(c.log, dst, src, nil, true)
3716 xcheckf(err, "link or copy file %q to %q", src, dst)
3717 createdIDs = append(createdIDs, newMsgIDs[i])
3720 for dir := range syncDirs {
3721 err := moxio.SyncDir(c.log, dir)
3722 xcheckf(err, "sync directory")
3725 err = c.account.RetrainMessages(context.TODO(), c.log, tx, nmsgs, false)
3726 xcheckf(err, "train copied messages")
3729 // Broadcast changes to other connections.
3730 if len(newUIDs) > 0 {
3731 changes := make([]store.Change, 0, len(newUIDs)+2)
3732 for i, uid := range newUIDs {
3733 changes = append(changes, store.ChangeAddUID{MailboxID: mbDst.ID, UID: uid, ModSeq: modseq, Flags: flags[i], Keywords: keywords[i]})
3735 changes = append(changes, mbDst.ChangeCounts())
3737 changes = append(changes, mbDst.ChangeKeywords())
3739 c.broadcast(changes)
3743 // All good, prevent defer above from cleaning up copied files.
3747 c.writeresultf("%s OK [COPYUID %d %s %s] copied", tag, mbDst.UIDValidity, compactUIDSet(origUIDs).String(), compactUIDSet(newUIDs).String())
3750// Move moves messages from the currently selected/active mailbox to a named mailbox.
3753func (c *conn) cmdxMove(isUID bool, tag, cmd string, p *parser) {
3760 name := p.xmailbox()
3763 name = xcheckmailboxname(name, true)
3766 xuserErrorf("mailbox open in read-only mode")
3769 uids, uidargs := c.gatherCopyMoveUIDs(isUID, nums)
3771 var mbSrc, mbDst store.Mailbox
3772 var changes []store.Change
3773 var newUIDs []store.UID
3774 var modseq store.ModSeq
3776 c.account.WithWLock(func() {
3777 c.xdbwrite(func(tx *bstore.Tx) {
3778 mbSrc = c.xmailboxID(tx, c.mailboxID) // Validate.
3779 mbDst = c.xmailbox(tx, name, "TRYCREATE")
3780 if mbDst.ID == c.mailboxID {
3781 xuserErrorf("cannot move to currently selected mailbox")
3784 if len(uidargs) == 0 {
3785 xuserErrorf("no matching messages to move")
3788 // Reserve the uids in the destination mailbox.
3789 uidFirst := mbDst.UIDNext
3791 mbDst.UIDNext += store.UID(len(uids))
3793 // Assign a new modseq, for the new records and for the expunged records.
3795 modseq, err = c.account.NextModSeq(tx)
3796 xcheckf(err, "assigning next modseq")
3798 // Update existing record with new UID and MailboxID in database for messages. We
3799 // add a new but expunged record again in the original/source mailbox, for qresync.
3800 // Keeping the original ID for the live message means we don't have to move the
3801 // on-disk message contents file.
3802 q := bstore.QueryTx[store.Message](tx)
3803 q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
3804 q.FilterEqual("UID", uidargs...)
3805 q.FilterEqual("Expunged", false)
3807 msgs, err := q.List()
3808 xcheckf(err, "listing messages to move")
3810 if len(msgs) != len(uidargs) {
3811 xserverErrorf("uid and message mismatch")
3814 keywords := map[string]struct{}{}
3816 conf, _ := c.account.Conf()
3817 for i := range msgs {
3819 if m.UID != uids[i] {
3820 xserverErrorf("internal error: got uid %d, expected %d, for index %d", m.UID, uids[i], i)
3823 mbSrc.Sub(m.MailboxCounts())
3825 // Copy of message record that we'll insert when UID is freed up.
3828 om.ID = 0 // Assign new ID.
3831 m.MailboxID = mbDst.ID
3832 if m.IsReject && m.MailboxDestinedID != 0 {
3833 // Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message
3834 // is used for reputation calculation during future deliveries.
3835 m.MailboxOrigID = m.MailboxDestinedID
3839 mbDst.Add(m.MailboxCounts())
3842 m.JunkFlagsForMailbox(mbDst, conf)
3845 xcheckf(err, "updating moved message in database")
3847 // Now that UID is unused, we can insert the old record again.
3848 err = tx.Insert(&om)
3849 xcheckf(err, "inserting record for expunge after moving message")
3851 for _, kw := range m.Keywords {
3852 keywords[kw] = struct{}{}
3856 // Ensure destination mailbox has keywords of the moved messages.
3857 var mbKwChanged bool
3858 mbDst.Keywords, mbKwChanged = store.MergeKeywords(mbDst.Keywords, maps.Keys(keywords))
3860 changes = append(changes, mbDst.ChangeKeywords())
3863 err = tx.Update(&mbSrc)
3864 xcheckf(err, "updating source mailbox counts")
3866 err = tx.Update(&mbDst)
3867 xcheckf(err, "updating destination mailbox for uids, keywords and counts")
3869 err = c.account.RetrainMessages(context.TODO(), c.log, tx, msgs, false)
3870 xcheckf(err, "retraining messages after move")
3872 // Prepare broadcast changes to other connections.
3873 changes = make([]store.Change, 0, 1+len(msgs)+2)
3874 changes = append(changes, store.ChangeRemoveUIDs{MailboxID: c.mailboxID, UIDs: uids, ModSeq: modseq})
3875 for _, m := range msgs {
3876 newUIDs = append(newUIDs, m.UID)
3877 changes = append(changes, m.ChangeAddUID())
3879 changes = append(changes, mbSrc.ChangeCounts(), mbDst.ChangeCounts())
3882 c.broadcast(changes)
3887 c.bwritelinef("* OK [COPYUID %d %s %s] moved", mbDst.UIDValidity, compactUIDSet(uids).String(), compactUIDSet(newUIDs).String())
3888 qresync := c.enabled[capQresync]
3889 var vanishedUIDs numSet
3890 for i := 0; i < len(uids); i++ {
3891 seq := c.xsequence(uids[i])
3892 c.sequenceRemove(seq, uids[i])
3894 vanishedUIDs.append(uint32(uids[i]))
3896 c.bwritelinef("* %d EXPUNGE", seq)
3899 if !vanishedUIDs.empty() {
3901 for _, s := range vanishedUIDs.Strings(4*1024 - 32) {
3902 c.bwritelinef("* VANISHED %s", s)
3906 if c.enabled[capQresync] {
3908 c.writeresultf("%s OK [HIGHESTMODSEQ %d] move", tag, modseq.Client())
3914// Store sets a full set of flags, or adds/removes specific flags.
3917func (c *conn) cmdxStore(isUID bool, tag, cmd string, p *parser) {
3924 var unchangedSince *int64
3927 p.xtake("UNCHANGEDSINCE")
3934 c.xensureCondstore(nil)
3936 var plus, minus bool
3939 } else if p.take("-") {
3943 silent := p.take(".SILENT")
3945 var flagstrs []string
3946 if p.hasPrefix("(") {
3947 flagstrs = p.xflagList()
3949 flagstrs = append(flagstrs, p.xflag())
3951 flagstrs = append(flagstrs, p.xflag())
3957 xuserErrorf("mailbox open in read-only mode")
3960 flags, keywords, err := store.ParseFlagsKeywords(flagstrs)
3962 xuserErrorf("parsing flags: %v", err)
3964 var mask store.Flags
3966 mask, flags = flags, store.FlagsAll
3968 mask, flags = flags, store.Flags{}
3970 mask = store.FlagsAll
3973 var mb, origmb store.Mailbox
3974 var updated []store.Message
3975 var changed []store.Message // ModSeq more recent than unchangedSince, will be in MODIFIED response code, and we will send untagged fetch responses so client is up to date.
3976 var modseq store.ModSeq // Assigned when needed.
3977 modified := map[int64]bool{}
3979 c.account.WithWLock(func() {
3980 var mbKwChanged bool
3981 var changes []store.Change
3983 c.xdbwrite(func(tx *bstore.Tx) {
3984 mb = c.xmailboxID(tx, c.mailboxID) // Validate.
3987 uidargs := c.xnumSetCondition(isUID, nums)
3989 if len(uidargs) == 0 {
3993 // Ensure keywords are in mailbox.
3995 mb.Keywords, mbKwChanged = store.MergeKeywords(mb.Keywords, keywords)
3997 err := tx.Update(&mb)
3998 xcheckf(err, "updating mailbox with keywords")
4002 q := bstore.QueryTx[store.Message](tx)
4003 q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
4004 q.FilterEqual("UID", uidargs...)
4005 q.FilterEqual("Expunged", false)
4006 err := q.ForEach(func(m store.Message) error {
4007 // Client may specify a message multiple times, but we only process it once.
../rfc/7162:823
4012 mc := m.MailboxCounts()
4014 origFlags := m.Flags
4015 m.Flags = m.Flags.Set(mask, flags)
4016 oldKeywords := append([]string{}, m.Keywords...)
4018 m.Keywords, _ = store.RemoveKeywords(m.Keywords, keywords)
4020 m.Keywords, _ = store.MergeKeywords(m.Keywords, keywords)
4022 m.Keywords = keywords
4025 keywordsChanged := func() bool {
4026 sort.Strings(oldKeywords)
4027 n := append([]string{}, m.Keywords...)
4029 return !slices.Equal(oldKeywords, n)
4032 // If the message has a more recent modseq than the check requires, we won't modify
4033 // it and report in the final command response.
4036 // unchangedSince 0 always fails the check, we don't turn it into 1 like with our
4037 // internal modseqs. RFC implies that is not required for non-system flags, but we
4039 if unchangedSince != nil && m.ModSeq.Client() > *unchangedSince {
4040 changed = append(changed, m)
4045 // It requires that we keep track of the flags we think the client knows (but only
4046 // on this connection). We don't track that. It also isn't clear why this is
4047 // allowed because it is skipping the condstore conditional check, and the new
4048 // combination of flags could be unintended.
4051 if origFlags == m.Flags && !keywordsChanged() {
4052 // Note: since we didn't update the modseq, we are not adding m.ID to "modified",
4053 // it would skip the modseq check above. We still add m to list of updated, so we
4054 // send an untagged fetch response. But we don't broadcast it.
4055 updated = append(updated, m)
4060 mb.Add(m.MailboxCounts())
4062 // Assign new modseq for first actual change.
4065 modseq, err = c.account.NextModSeq(tx)
4066 xcheckf(err, "next modseq")
4069 modified[m.ID] = true
4070 updated = append(updated, m)
4072 changes = append(changes, m.ChangeFlags(origFlags))
4074 return tx.Update(&m)
4076 xcheckf(err, "storing flags in messages")
4078 if mb.MailboxCounts != origmb.MailboxCounts {
4079 err := tx.Update(&mb)
4080 xcheckf(err, "updating mailbox counts")
4082 changes = append(changes, mb.ChangeCounts())
4085 changes = append(changes, mb.ChangeKeywords())
4088 err = c.account.RetrainMessages(context.TODO(), c.log, tx, updated, false)
4089 xcheckf(err, "training messages")
4092 c.broadcast(changes)
4095 // In the RFC, the section about STORE/UID STORE says we must return MODSEQ when
4096 // UNCHANGEDSINCE was specified. It does not specify it in case UNCHANGEDSINCE
4097 // isn't specified. For that case it does say MODSEQ is needed in unsolicited
4098 // untagged fetch responses. Implying that solicited untagged fetch responses
4099 // should not include MODSEQ (why else mention unsolicited explicitly?). But, in
4100 // the introduction to CONDSTORE it does explicitly specify MODSEQ should be
4101 // included in untagged fetch responses at all times with CONDSTORE-enabled
4102 // connections. It would have been better if the command behaviour was specified in
4103 // the command section, not the introduction to the extension.
4106 if !silent || c.enabled[capCondstore] {
4107 for _, m := range updated {
4110 flags = fmt.Sprintf(" FLAGS %s", flaglist(m.Flags, m.Keywords).pack(c))
4112 var modseqStr string
4113 if c.enabled[capCondstore] {
4114 modseqStr = fmt.Sprintf(" MODSEQ (%d)", m.ModSeq.Client())
4117 c.bwritelinef("* %d FETCH (UID %d%s%s)", c.xsequence(m.UID), m.UID, flags, modseqStr)
4121 // We don't explicitly send flags for failed updated with silent set. The regular
4122 // notification will get the flags to the client.
4125 if len(changed) == 0 {
4130 // Write unsolicited untagged fetch responses for messages that didn't pass the
4133 var mnums []store.UID
4134 for _, m := range changed {
4135 c.bwritelinef("* %d FETCH (UID %d FLAGS %s MODSEQ (%d))", c.xsequence(m.UID), m.UID, flaglist(m.Flags, m.Keywords).pack(c), m.ModSeq.Client())
4137 mnums = append(mnums, m.UID)
4139 mnums = append(mnums, store.UID(c.xsequence(m.UID)))
4143 sort.Slice(mnums, func(i, j int) bool {
4144 return mnums[i] < mnums[j]
4146 set := compactUIDSet(mnums)
4148 c.writeresultf("%s OK [MODIFIED %s] conditional store did not modify all", tag, set.String())