13 "github.com/mjl-/bstore"
15 "github.com/mjl-/mox/message"
16 "github.com/mjl-/mox/store"
19// If last search output was this long ago, we write an untagged inprogress
21var inProgressPeriod = time.Duration(10 * time.Second)
23// ESEARCH allows searching multiple mailboxes, referenced through mailbox filters
24// borrowed from the NOTIFY extension. Unlike the regular extended SEARCH/UID
25// SEARCH command that always returns an ESEARCH response, the ESEARCH command only
26// returns ESEARCH responses when there were matches in a mailbox.
29func (c *conn) cmdEsearch(tag, cmd string, p *parser) {
30 c.cmdxSearch(true, true, tag, cmd, p)
33// Search returns messages matching criteria specified in parameters.
35// State: Selected for SEARCH and UID SEARCH, Authenticated or selectd for ESEARCH.
36func (c *conn) cmdxSearch(isUID, isE bool, tag, cmd string, p *parser) {
41 // We will respond with ESEARCH instead of SEARCH if "RETURN" is present or for IMAP4rev2 or for isE (ESEARCH command).
42 var eargs map[string]bool // Options except SAVE. Nil means old-style SEARCH response.
43 var save bool // For SAVE option. Kept separately for easier handling of MIN/MAX later.
45 if c.enabled[capIMAP4rev2] || isE {
46 eargs = map[string]bool{}
49 // The ESEARCH command has various ways to specify which mailboxes are to be
50 // searched. We parse and gather the request first, and evaluate them to mailboxes
51 // after parsing, when we start and have a DB transaction.
52 var mailboxSpecs []mailboxSpecifier
55 if isE && p.take(" IN (") {
57 ms := p.xfilterMailbox(mbspecsEsearch)
58 mailboxSpecs = append(mailboxSpecs, ms)
65 // We are not parsing the scope-options since there aren't any defined yet.
../rfc/7377:469
68 if p.take(" RETURN (") {
69 eargs = map[string]bool{}
72 if len(eargs) > 0 || save {
75 if w, ok := p.takelist("MIN", "MAX", "ALL", "COUNT", "SAVE"); ok {
83 xsyntaxErrorf("ESEARCH result option %q not supported", w)
88 if eargs != nil && len(eargs) == 0 && !save {
92 // If UTF8=ACCEPT is enabled, we should not accept any charset. We are a bit more
94 if p.take(" CHARSET ") {
95 charset := strings.ToUpper(p.xastring())
96 if charset != "US-ASCII" && charset != "UTF-8" {
98 xusercodeErrorf("BADCHARSET", "only US-ASCII and UTF-8 supported")
103 searchKeys: []searchKey{*p.xsearchKey()},
107 sk.searchKeys = append(sk.searchKeys, *p.xsearchKey())
111 if c.uidonly && sk.hasSequenceNumbers() {
112 xsyntaxCodeErrorf("UIDREQUIRED", "cannot search message sequence numbers in search program with uidonly enabled")
115 // Even in case of error, we ensure search result is changed.
117 c.searchResult = []store.UID{}
120 // We gather word and not-word searches from the top-level, turn them
121 // into a WordSearch for a more efficient search.
122 // todo optimize: also gather them out of AND searches.
123 var textWords, textNotWords, bodyWords, bodyNotWords []string
125 for _, xsk := range sk.searchKeys {
128 bodyWords = append(bodyWords, xsk.astring)
131 textWords = append(textWords, xsk.astring)
134 switch xsk.searchKey.op {
136 bodyNotWords = append(bodyNotWords, xsk.searchKey.astring)
139 textNotWords = append(textNotWords, xsk.searchKey.astring)
143 sk.searchKeys[n] = xsk
146 // We may be left with an empty but non-nil sk.searchKeys, which is important for
148 sk.searchKeys = sk.searchKeys[:n]
149 var bodySearch, textSearch *store.WordSearch
150 if len(bodyWords) > 0 || len(bodyNotWords) > 0 {
151 ws := store.PrepareWordSearch(bodyWords, bodyNotWords)
154 if len(textWords) > 0 || len(textNotWords) > 0 {
155 ws := store.PrepareWordSearch(textWords, textNotWords)
159 // Note: we only hold the account rlock for verifying the mailbox at the start.
161 runlock := c.account.RUnlock
162 // Note: in a defer because we replace it below.
167 // If we only have a MIN and/or MAX, we can stop processing as soon as we
168 // have those matches.
177 // We'll have one Result per mailbox we are searching. For regular (UID) SEARCH
178 // commands, we'll have just one, for the selected mailbox.
180 Mailbox store.Mailbox
181 MaxModSeq store.ModSeq
186 // We periodically send an untagged OK with INPROGRESS code while searching, to let
187 // clients doing slow searches know we're still working.
188 inProgressLast := time.Now()
189 // Only respond with tag if it can't be confused as end of response code.
../rfc/9585:122
190 inProgressTag := "nil"
191 if !strings.Contains(tag, "]") {
192 inProgressTag = dquote(tag).pack(c)
195 c.xdbread(func(tx *bstore.Tx) {
196 // Gather mailboxes to operate on. Usually just the selected mailbox. But with the
197 // ESEARCH command, we may be searching multiple.
198 var mailboxes []store.Mailbox
199 if len(mailboxSpecs) > 0 {
201 m := map[int64]store.Mailbox{}
202 for _, ms := range mailboxSpecs {
206 if c.state != stateSelected {
207 xsyntaxErrorf("cannot use ESEARCH with selected when state is not selected")
210 mb := c.xmailboxID(tx, c.mailboxID) // Validate.
214 // Inbox and everything below. And we look at destinations and rulesets. We all
215 // mailboxes from the destinations, and all from the rulesets except when
216 // ListAllowDomain is non-empty.
218 q := bstore.QueryTx[store.Mailbox](tx)
219 q.FilterEqual("Expunged", false)
220 q.FilterGreaterEqual("Name", "Inbox")
222 for mb, err := range q.All() {
223 xcheckf(err, "list mailboxes")
224 if mb.Name != "Inbox" && !strings.HasPrefix(mb.Name, "Inbox/") {
230 conf, _ := c.account.Conf()
231 for _, dest := range conf.Destinations {
232 if dest.Mailbox != "" && dest.Mailbox != "Inbox" {
233 mb, err := c.account.MailboxFind(tx, dest.Mailbox)
234 xcheckf(err, "find mailbox from destination")
240 for _, rs := range dest.Rulesets {
241 if rs.ListAllowDomain != "" || rs.Mailbox == "" {
245 mb, err := c.account.MailboxFind(tx, rs.Mailbox)
246 xcheckf(err, "find mailbox from ruleset")
254 // All mailboxes in the personal namespace. Which is all mailboxes for us.
256 for mb, err := range bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).All() {
257 xcheckf(err, "list mailboxes")
261 case mbspecSubscribed:
262 // Mailboxes that are subscribed. Will typically be same as personal, since we
263 // subscribe to all mailboxes. But user can manage subscriptions differently.
265 for mb, err := range bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).All() {
266 xcheckf(err, "list mailboxes")
267 if err := tx.Get(&store.Subscription{Name: mb.Name}); err == nil {
269 } else if err != bstore.ErrAbsent {
270 xcheckf(err, "lookup subscription for mailbox")
274 case mbspecSubtree, mbspecSubtreeOne:
276 // SUBTREE is arbitrarily deep, SUBTREE-ONE is one level deeper than requested
279 // We don't have to worry about loops. Mailboxes are not in the file system.
282 for _, name := range ms.Mailboxes {
283 name = xcheckmailboxname(name, true)
285 one := ms.Kind == mbspecSubtreeOne
288 ntoken = len(strings.Split(name, "/")) + 1
291 q := bstore.QueryTx[store.Mailbox](tx)
292 q.FilterEqual("Expunged", false)
293 q.FilterGreaterEqual("Name", name)
295 for mb, err := range q.All() {
296 xcheckf(err, "list mailboxes")
297 if mb.Name != name && !strings.HasPrefix(mb.Name, name+"/") {
300 if !one || mb.Name == name || len(strings.Split(mb.Name, "/")) == ntoken {
306 case mbspecMailboxes:
308 for _, name := range ms.Mailboxes {
309 name = xcheckmailboxname(name, true)
311 // If a mailbox doesn't exist, we don't treat it as an error. Seems reasonable
312 // giving we are searching. Messages may not exist. And likewise for the mailbox.
313 // Just results in no hits.
314 mb, err := c.account.MailboxFind(tx, name)
315 xcheckf(err, "looking up mailbox")
322 panic("missing case")
325 mailboxes = slices.Collect(maps.Values(m))
326 slices.SortFunc(mailboxes, func(a, b store.Mailbox) int {
327 return cmp.Compare(a.Name, b.Name)
330 // If no source mailboxes were specified (no mailboxSpecs), the selected mailbox is
333 mb := c.xmailboxID(tx, c.mailboxID) // Validate.
334 mailboxes = []store.Mailbox{mb}
337 if save && !(len(mailboxes) == 1 && mailboxes[0].ID == c.mailboxID) {
339 xsyntaxErrorf("can only use SAVE on selected mailbox")
345 // Determine if search has a sequence set without search results. If so, we need
346 // sequence numbers for matching, and we must always go through the messages in
347 // forward order. No reverse search for MAX only.
348 needSeq := (len(mailboxes) > 1 || len(mailboxes) == 1 && mailboxes[0].ID != c.mailboxID) && sk.hasSequenceNumbers()
350 forward := eargs == nil || max1 == 0 || len(eargs) != 1 || needSeq
351 reverse := max1 == 1 && (len(eargs) == 1 || min1+max1 == len(eargs)) && !needSeq
353 // We set a worst-case "goal" of having gone through all messages in all mailboxes.
354 // Sometimes, we can be faster, when we only do a MIN and/or MAX query and we can
355 // stop early. We'll account for that as we go. For the selected mailbox, we'll
356 // only look at those the session has already seen.
359 for _, mb := range mailboxes {
360 if mb.ID == c.mailboxID && !c.uidonly {
363 total += uint32(mb.Total + mb.Deleted)
368 goal = fmt.Sprintf("%d", total)
372 for _, mb := range mailboxes {
373 var lastUID store.UID
375 result := Result{Mailbox: mb}
377 msgCount := uint32(mb.MailboxCounts.Total + mb.MailboxCounts.Deleted)
378 if mb.ID == c.mailboxID && !c.uidonly {
382 // Used for interpreting UID sets with a star, like "1:*" and "10:*". Only called
383 // for UIDs that are higher than the number, since "10:*" evaluates to "10:5" if 5
384 // is the highest UID, and UID 5-10 would all match.
385 var cachedHighestUID store.UID
386 xhighestUID := func() store.UID {
387 if cachedHighestUID > 0 {
388 return cachedHighestUID
391 q := bstore.QueryTx[store.Message](tx)
392 q.FilterNonzero(store.Message{MailboxID: mb.ID})
393 q.FilterEqual("Expunged", false)
394 if mb.ID == c.mailboxID {
395 q.FilterLess("UID", c.uidnext)
400 if err == bstore.ErrAbsent {
401 xuserErrorf("cannot use * on empty mailbox")
403 xcheckf(err, "get last uid")
404 cachedHighestUID = m.UID
405 return cachedHighestUID
408 progressOrig := progress
411 // We track this for non-selected mailboxes. searchMatch will look the message
412 // sequence number for this session up if we are searching the selected mailbox.
415 q := bstore.QueryTx[store.Message](tx)
416 q.FilterNonzero(store.Message{MailboxID: mb.ID})
417 q.FilterEqual("Expunged", false)
418 if mb.ID == c.mailboxID {
419 q.FilterLess("UID", c.uidnext)
422 for m, err := range q.All() {
423 xcheckf(err, "list messages in mailbox")
425 // We track this for the "reverse" case, we'll stop before seeing lastUID.
428 if time.Since(inProgressLast) > inProgressPeriod {
429 c.xwritelinef("* OK [INPROGRESS (%s %d %s)] still searching", inProgressTag, progress, goal)
430 inProgressLast = time.Now()
434 if c.searchMatch(tx, msgCount, seq, m, *sk, bodySearch, textSearch, xhighestUID) {
435 result.UIDs = append(result.UIDs, m.UID)
436 result.MaxModSeq = max(result.MaxModSeq, m.ModSeq)
437 if min1 == 1 && min1+max1 == len(eargs) {
441 // We only need a MIN and a MAX, but we also need sequence numbers so we are
442 // walking through and collecting all UIDs. Correct for that, keeping only the MIN
445 if len(result.UIDs) == 3 {
446 result.UIDs[1] = result.UIDs[2]
447 result.UIDs = result.UIDs[:2]
454 // And reverse search for MAX if we have only MAX or MAX combined with MIN, and
455 // don't need sequence numbers. We just need a single match, then we stop.
457 q := bstore.QueryTx[store.Message](tx)
458 q.FilterNonzero(store.Message{MailboxID: mb.ID})
459 q.FilterEqual("Expunged", false)
460 q.FilterGreater("UID", lastUID)
461 if mb.ID == c.mailboxID {
462 q.FilterLess("UID", c.uidnext)
465 for m, err := range q.All() {
466 xcheckf(err, "list messages in mailbox")
468 if time.Since(inProgressLast) > inProgressPeriod {
469 c.xwritelinef("* OK [INPROGRESS (%s %d %s)] still searching", inProgressTag, progress, goal)
470 inProgressLast = time.Now()
474 var seq msgseq // Filled in by searchMatch for messages in selected mailbox.
475 if c.searchMatch(tx, msgCount, seq, m, *sk, bodySearch, textSearch, xhighestUID) {
476 result.UIDs = append(result.UIDs, m.UID)
477 result.MaxModSeq = max(result.MaxModSeq, m.ModSeq)
483 // We could have finished searching the mailbox with fewer
484 mailboxProcessed := progress - progressOrig
485 mailboxTotal := uint32(mb.MailboxCounts.Total + mb.MailboxCounts.Deleted)
486 progress += max(0, mailboxTotal-mailboxProcessed)
488 results = append(results, result)
493 // We'll only have a result for the one selected mailbox.
497 if len(result.UIDs) == 0 {
498 c.xbwritelinef("* SEARCH")
501 // Old-style SEARCH response. We must spell out each number. So we may be splitting
503 for len(result.UIDs) > 0 {
504 n := min(100, len(result.UIDs))
506 for _, v := range result.UIDs[:n] {
508 v = store.UID(c.xsequence(v))
510 s += " " + fmt.Sprintf("%d", v)
513 // Since we don't have the max modseq for the possibly partial uid range we're
514 // writing here within hand reach, we conveniently interpret the ambiguous "for all
516 // write. And that clients only commit this value after they have seen the tagged
522 modseq = fmt.Sprintf(" (MODSEQ %d)", result.MaxModSeq.Client())
525 c.xbwritelinef("* SEARCH%s%s", s, modseq)
526 result.UIDs = result.UIDs[n:]
533 c.searchResult = results[0].UIDs
534 c.checkUIDs(c.searchResult, false)
539 for _, result := range results {
540 // For the ESEARCH command, we must not return a response if there were no matching
541 // messages. This is unlike the later IMAP4rev2, where an ESEARCH response must be
543 if isE && len(result.UIDs) == 0 {
547 // The tag was originally a string, became an astring in IMAP4rev2, better stick to
550 fmt.Fprintf(c.xbw, `* ESEARCH (TAG "%s" MAILBOX %s UIDVALIDITY %d)`, tag, result.Mailbox.Name, result.Mailbox.UIDValidity)
552 fmt.Fprintf(c.xbw, `* ESEARCH (TAG "%s")`, tag)
555 fmt.Fprintf(c.xbw, " UID")
558 // NOTE: we are potentially converting UIDs to msgseq, but keep the store.UID type
562 // If searchResult is hanging on to the slice, we need to work on a copy.
564 nums = slices.Clone(nums)
566 for i, uid := range nums {
567 nums[i] = store.UID(c.xsequence(uid))
572 if eargs["MIN"] && len(nums) > 0 {
573 fmt.Fprintf(c.xbw, " MIN %d", nums[0])
575 if eargs["MAX"] && len(result.UIDs) > 0 {
576 fmt.Fprintf(c.xbw, " MAX %d", nums[len(nums)-1])
579 fmt.Fprintf(c.xbw, " COUNT %d", len(nums))
581 if eargs["ALL"] && len(nums) > 0 {
582 fmt.Fprintf(c.xbw, " ALL %s", compactUIDSet(nums).String())
586 // Summary: send the highest modseq of the returned messages.
587 if sk.hasModseq() && len(nums) > 0 {
588 fmt.Fprintf(c.xbw, " MODSEQ %d", result.MaxModSeq.Client())
602 msgCount uint32 // Number of messages in mailbox (or session when selected).
603 seq msgseq // Can be 0, for other mailboxes than selected in case of MAX.
607 xhighestUID func() store.UID
610func (c *conn) searchMatch(tx *bstore.Tx, msgCount uint32, seq msgseq, m store.Message, sk searchKey, bodySearch, textSearch *store.WordSearch, xhighestUID func() store.UID) bool {
611 if m.MailboxID == c.mailboxID {
612 // If session doesn't know about the message yet, don't return it.
614 if m.UID >= c.uidnext {
618 // Set seq for use in evaluations.
619 seq = c.sequence(m.UID)
626 s := search{c: c, tx: tx, msgCount: msgCount, seq: seq, m: m, xhighestUID: xhighestUID}
630 c.xsanity(err, "closing messagereader")
634 return s.match(sk, bodySearch, textSearch)
637func (s *search) match(sk searchKey, bodySearch, textSearch *store.WordSearch) (match bool) {
639 if match && bodySearch != nil {
640 if !s.xensurePart() {
645 match, err = bodySearch.MatchPart(s.c.log, s.p, false)
646 xcheckf(err, "search words in bodies")
648 if match && textSearch != nil {
649 if !s.xensurePart() {
654 match, err = textSearch.MatchPart(s.c.log, s.p, true)
655 xcheckf(err, "search words in headers and bodies")
660// ensure message, reader and part are loaded. returns whether that was
662func (s *search) xensurePart() bool {
667 // Closed by searchMatch after all (recursive) search.match calls are finished.
668 s.mr = s.c.account.MessageReader(s.m)
670 if s.m.ParsedBuf == nil {
671 s.c.log.Error("missing parsed message")
674 p, err := s.m.LoadPart(s.mr)
675 xcheckf(err, "load parsed message")
680func (s *search) match0(sk searchKey) bool {
683 // Difference between sk.searchKeys nil and length 0 is important. Because we take
684 // out word/notword searches, the list may be empty but non-nil.
685 if sk.searchKeys != nil {
686 for _, ssk := range sk.searchKeys {
692 } else if sk.seqSet != nil {
693 if sk.seqSet.searchResult {
694 // Interpreting search results on a mailbox that isn't selected during multisearch
696 if s.m.MailboxID != c.mailboxID {
697 xuserErrorf("can only use search result with the selected mailbox")
699 return uidSearch(c.searchResult, s.m.UID) > 0
701 // For multisearch, we have arranged to have a seq for non-selected mailboxes too.
702 return sk.seqSet.containsSeqCount(s.seq, s.msgCount)
705 filterHeader := func(field, value string) bool {
706 lower := strings.ToLower(value)
707 h, err := s.p.Header()
709 c.log.Debugx("parsing message header", err, slog.Any("uid", s.m.UID), slog.Int64("msgid", s.m.ID))
712 for _, v := range h.Values(field) {
713 if strings.Contains(strings.ToLower(v), lower) {
720 // We handle ops by groups that need increasing details about the message.
726 // We do not implement the RECENT flag, so messages cannot be NEW.
729 // We treat all messages as non-recent, so this means all messages.
732 // We do not implement the RECENT flag. All messages are not recent.
735 return !s.match0(*sk.searchKey)
737 return s.match0(*sk.searchKey) || s.match0(*sk.searchKey2)
739 if sk.uidSet.searchResult && s.m.MailboxID != c.mailboxID {
740 // Interpreting search results on a mailbox that isn't selected during multisearch
742 xuserErrorf("cannot use search result from another mailbox")
744 return sk.uidSet.xcontainsKnownUID(s.m.UID, c.searchResult, s.xhighestUID)
748 if !s.xensurePart() {
752 // Parsed message, basic info.
761 kw := strings.ToLower(sk.atom)
774 return slices.Contains(s.m.Keywords, kw)
785 kw := strings.ToLower(sk.atom)
788 return !s.m.Forwarded
798 return !slices.Contains(s.m.Keywords, kw)
806 case "BEFORE", "ON", "SINCE":
807 skdt := sk.date.Format("2006-01-02")
808 rdt := s.m.Received.Format("2006-01-02")
817 panic("missing case")
819 return s.m.Size > sk.number
821 return s.m.Size < sk.number
824 return s.m.ModSeq.Client() >= *sk.clientModseq
825 case "SAVEDBEFORE", "SAVEDON", "SAVEDSINCE":
826 // If we don't have a savedate for this message (for messages received before we
827 // implemented this feature), we use the "internal date" (received timestamp) of
830 if s.m.SaveDate != nil {
834 skdt := sk.date.Format("2006-01-02")
835 rdt := rt.Format("2006-01-02")
844 panic("missing case")
845 case "SAVEDATESUPPORTED":
846 // We return whether we have a savedate for this message. We support it on all
847 // mailboxes, but we only have this metadata from the time we implemented this
849 return s.m.SaveDate != nil
852 seconds := int64(time.Since(s.m.Received) / time.Second)
853 return seconds >= sk.number
855 seconds := int64(time.Since(s.m.Received) / time.Second)
856 return seconds <= sk.number
860 c.log.Info("missing parsed message, not matching", slog.Any("uid", s.m.UID), slog.Int64("msgid", s.m.ID))
864 // Parsed message, more info.
867 return filterHeader("Bcc", sk.astring)
869 // We gathered word/notword searches from the top-level, but we can also get them
871 // todo optimize: handle deeper nested word/not-word searches more efficiently.
872 headerToo := sk.op == "TEXT"
873 match, err := store.PrepareWordSearch([]string{sk.astring}, nil).MatchPart(s.c.log, s.p, headerToo)
874 xcheckf(err, "word search")
877 return filterHeader("Cc", sk.astring)
879 return filterHeader("From", sk.astring)
881 return filterHeader("Subject", sk.astring)
883 return filterHeader("To", sk.astring)
886 lower := strings.ToLower(sk.astring)
887 h, err := s.p.Header()
889 c.log.Errorx("parsing header for search", err, slog.Any("uid", s.m.UID), slog.Int64("msgid", s.m.ID))
892 k := textproto.CanonicalMIMEHeaderKey(sk.headerField)
893 for _, v := range h.Values(k) {
894 if lower == "" || strings.Contains(strings.ToLower(v), lower) {
899 case "SENTBEFORE", "SENTON", "SENTSINCE":
900 if s.p.Envelope == nil || s.p.Envelope.Date.IsZero() {
903 dt := s.p.Envelope.Date.Format("2006-01-02")
904 skdt := sk.date.Format("2006-01-02")
913 panic("missing case")
915 panic(serverError{fmt.Errorf("missing case for search key op %q", sk.op)})