14 "github.com/mjl-/bstore"
16 "github.com/mjl-/mox/message"
17 "github.com/mjl-/mox/store"
20// If last search output was this long ago, we write an untagged inprogress
22var inProgressPeriod = time.Duration(10 * time.Second)
24// ESEARCH allows searching multiple mailboxes, referenced through mailbox filters
25// borrowed from the NOTIFY extension. Unlike the regular extended SEARCH/UID
26// SEARCH command that always returns an ESEARCH response, the ESEARCH command only
27// returns ESEARCH responses when there were matches in a mailbox.
30func (c *conn) cmdEsearch(tag, cmd string, p *parser) {
31 c.cmdxSearch(true, true, tag, cmd, p)
34// Search returns messages matching criteria specified in parameters.
36// State: Selected for SEARCH and UID SEARCH, Authenticated or selectd for ESEARCH.
37func (c *conn) cmdxSearch(isUID, isE bool, tag, cmd string, p *parser) {
42 // We will respond with ESEARCH instead of SEARCH if "RETURN" is present or for IMAP4rev2 or for isE (ESEARCH command).
43 var eargs map[string]bool // Options except SAVE. Nil means old-style SEARCH response.
44 var save bool // For SAVE option. Kept separately for easier handling of MIN/MAX later.
46 if c.enabled[capIMAP4rev2] || isE {
47 eargs = map[string]bool{}
50 // The ESEARCH command has various ways to specify which mailboxes are to be
51 // searched. We parse and gather the request first, and evaluate them to mailboxes
52 // after parsing, when we start and have a DB transaction.
53 var mailboxSpecs []mailboxSpecifier
56 if isE && p.take(" IN (") {
58 ms := p.xfilterMailbox(mbspecsEsearch)
59 mailboxSpecs = append(mailboxSpecs, ms)
66 // We are not parsing the scope-options since there aren't any defined yet.
../rfc/7377:469
69 if p.take(" RETURN (") {
70 eargs = map[string]bool{}
73 if len(eargs) > 0 || save {
76 if w, ok := p.takelist("MIN", "MAX", "ALL", "COUNT", "SAVE"); ok {
84 xsyntaxErrorf("ESEARCH result option %q not supported", w)
89 if eargs != nil && len(eargs) == 0 && !save {
93 // If UTF8=ACCEPT is enabled, we should not accept any charset. We are a bit more
95 if p.take(" CHARSET ") {
96 charset := strings.ToUpper(p.xastring())
97 if charset != "US-ASCII" && charset != "UTF-8" {
99 xusercodeErrorf("BADCHARSET", "only US-ASCII and UTF-8 supported")
104 searchKeys: []searchKey{*p.xsearchKey()},
108 sk.searchKeys = append(sk.searchKeys, *p.xsearchKey())
112 if c.uidonly && sk.hasSequenceNumbers() {
113 xsyntaxCodeErrorf("UIDREQUIRED", "cannot search message sequence numbers in search program with uidonly enabled")
116 // Even in case of error, we ensure search result is changed.
118 c.searchResult = []store.UID{}
121 // We gather word and not-word searches from the top-level, turn them
122 // into a WordSearch for a more efficient search.
123 // todo optimize: also gather them out of AND searches.
124 var textWords, textNotWords, bodyWords, bodyNotWords []string
126 for _, xsk := range sk.searchKeys {
129 bodyWords = append(bodyWords, xsk.astring)
132 textWords = append(textWords, xsk.astring)
135 switch xsk.searchKey.op {
137 bodyNotWords = append(bodyNotWords, xsk.searchKey.astring)
140 textNotWords = append(textNotWords, xsk.searchKey.astring)
144 sk.searchKeys[n] = xsk
147 // We may be left with an empty but non-nil sk.searchKeys, which is important for
149 sk.searchKeys = sk.searchKeys[:n]
150 var bodySearch, textSearch *store.WordSearch
151 if len(bodyWords) > 0 || len(bodyNotWords) > 0 {
152 ws := store.PrepareWordSearch(bodyWords, bodyNotWords)
155 if len(textWords) > 0 || len(textNotWords) > 0 {
156 ws := store.PrepareWordSearch(textWords, textNotWords)
160 // Note: we only hold the account rlock for verifying the mailbox at the start.
162 runlock := c.account.RUnlock
163 // Note: in a defer because we replace it below.
168 // If we only have a MIN and/or MAX, we can stop processing as soon as we
169 // have those matches.
178 // We'll have one Result per mailbox we are searching. For regular (UID) SEARCH
179 // commands, we'll have just one, for the selected mailbox.
181 Mailbox store.Mailbox
182 MaxModSeq store.ModSeq
187 // We periodically send an untagged OK with INPROGRESS code while searching, to let
188 // clients doing slow searches know we're still working.
189 inProgressLast := time.Now()
190 // Only respond with tag if it can't be confused as end of response code.
../rfc/9585:122
191 inProgressTag := "nil"
192 if !strings.Contains(tag, "]") {
193 inProgressTag = dquote(tag).pack(c)
196 c.xdbread(func(tx *bstore.Tx) {
197 // Gather mailboxes to operate on. Usually just the selected mailbox. But with the
198 // ESEARCH command, we may be searching multiple.
199 var mailboxes []store.Mailbox
200 if len(mailboxSpecs) > 0 {
202 m := map[int64]store.Mailbox{}
203 for _, ms := range mailboxSpecs {
207 if c.state != stateSelected {
208 xsyntaxErrorf("cannot use ESEARCH with selected when state is not selected")
211 mb := c.xmailboxID(tx, c.mailboxID) // Validate.
215 // Inbox and everything below. And we look at destinations and rulesets. We all
216 // mailboxes from the destinations, and all from the rulesets except when
217 // ListAllowDomain is non-empty.
219 q := bstore.QueryTx[store.Mailbox](tx)
220 q.FilterEqual("Expunged", false)
221 q.FilterGreaterEqual("Name", "Inbox")
223 for mb, err := range q.All() {
224 xcheckf(err, "list mailboxes")
225 if mb.Name != "Inbox" && !strings.HasPrefix(mb.Name, "Inbox/") {
231 conf, _ := c.account.Conf()
232 for _, dest := range conf.Destinations {
233 if dest.Mailbox != "" && dest.Mailbox != "Inbox" {
234 mb, err := c.account.MailboxFind(tx, dest.Mailbox)
235 xcheckf(err, "find mailbox from destination")
241 for _, rs := range dest.Rulesets {
242 if rs.ListAllowDomain != "" || rs.Mailbox == "" {
246 mb, err := c.account.MailboxFind(tx, rs.Mailbox)
247 xcheckf(err, "find mailbox from ruleset")
255 // All mailboxes in the personal namespace. Which is all mailboxes for us.
257 for mb, err := range bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).All() {
258 xcheckf(err, "list mailboxes")
262 case mbspecSubscribed:
263 // Mailboxes that are subscribed. Will typically be same as personal, since we
264 // subscribe to all mailboxes. But user can manage subscriptions differently.
266 for mb, err := range bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).All() {
267 xcheckf(err, "list mailboxes")
268 if err := tx.Get(&store.Subscription{Name: mb.Name}); err == nil {
270 } else if err != bstore.ErrAbsent {
271 xcheckf(err, "lookup subscription for mailbox")
275 case mbspecSubtree, mbspecSubtreeOne:
277 // SUBTREE is arbitrarily deep, SUBTREE-ONE is one level deeper than requested
280 // We don't have to worry about loops. Mailboxes are not in the file system.
283 for _, name := range ms.Mailboxes {
284 name = xcheckmailboxname(name, true)
286 one := ms.Kind == mbspecSubtreeOne
289 ntoken = len(strings.Split(name, "/")) + 1
292 q := bstore.QueryTx[store.Mailbox](tx)
293 q.FilterEqual("Expunged", false)
294 q.FilterGreaterEqual("Name", name)
296 for mb, err := range q.All() {
297 xcheckf(err, "list mailboxes")
298 if mb.Name != name && !strings.HasPrefix(mb.Name, name+"/") {
301 if !one || mb.Name == name || len(strings.Split(mb.Name, "/")) == ntoken {
307 case mbspecMailboxes:
309 for _, name := range ms.Mailboxes {
310 name = xcheckmailboxname(name, true)
312 // If a mailbox doesn't exist, we don't treat it as an error. Seems reasonable
313 // giving we are searching. Messages may not exist. And likewise for the mailbox.
314 // Just results in no hits.
315 mb, err := c.account.MailboxFind(tx, name)
316 xcheckf(err, "looking up mailbox")
323 panic("missing case")
326 mailboxes = slices.Collect(maps.Values(m))
327 slices.SortFunc(mailboxes, func(a, b store.Mailbox) int {
328 return cmp.Compare(a.Name, b.Name)
331 // If no source mailboxes were specified (no mailboxSpecs), the selected mailbox is
334 mb := c.xmailboxID(tx, c.mailboxID) // Validate.
335 mailboxes = []store.Mailbox{mb}
338 if save && !(len(mailboxes) == 1 && mailboxes[0].ID == c.mailboxID) {
340 xsyntaxErrorf("can only use SAVE on selected mailbox")
346 // Determine if search has a sequence set without search results. If so, we need
347 // sequence numbers for matching, and we must always go through the messages in
348 // forward order. No reverse search for MAX only.
349 needSeq := (len(mailboxes) > 1 || len(mailboxes) == 1 && mailboxes[0].ID != c.mailboxID) && sk.hasSequenceNumbers()
351 forward := eargs == nil || max1 == 0 || len(eargs) != 1 || needSeq
352 reverse := max1 == 1 && (len(eargs) == 1 || min1+max1 == len(eargs)) && !needSeq
354 // We set a worst-case "goal" of having gone through all messages in all mailboxes.
355 // Sometimes, we can be faster, when we only do a MIN and/or MAX query and we can
356 // stop early. We'll account for that as we go. For the selected mailbox, we'll
357 // only look at those the session has already seen.
360 for _, mb := range mailboxes {
361 if mb.ID == c.mailboxID && !c.uidonly {
364 total += uint32(mb.Total + mb.Deleted)
369 goal = fmt.Sprintf("%d", total)
373 for _, mb := range mailboxes {
374 var lastUID store.UID
376 result := Result{Mailbox: mb}
378 msgCount := uint32(mb.MailboxCounts.Total + mb.MailboxCounts.Deleted)
379 if mb.ID == c.mailboxID && !c.uidonly {
383 // Used for interpreting UID sets with a star, like "1:*" and "10:*". Only called
384 // for UIDs that are higher than the number, since "10:*" evaluates to "10:5" if 5
385 // is the highest UID, and UID 5-10 would all match.
386 var cachedHighestUID store.UID
387 xhighestUID := func() store.UID {
388 if cachedHighestUID > 0 {
389 return cachedHighestUID
392 q := bstore.QueryTx[store.Message](tx)
393 q.FilterNonzero(store.Message{MailboxID: mb.ID})
394 q.FilterEqual("Expunged", false)
395 if mb.ID == c.mailboxID {
396 q.FilterLess("UID", c.uidnext)
401 if err == bstore.ErrAbsent {
402 xuserErrorf("cannot use * on empty mailbox")
404 xcheckf(err, "get last uid")
405 cachedHighestUID = m.UID
406 return cachedHighestUID
409 progressOrig := progress
412 // We track this for non-selected mailboxes. searchMatch will look the message
413 // sequence number for this session up if we are searching the selected mailbox.
416 q := bstore.QueryTx[store.Message](tx)
417 q.FilterNonzero(store.Message{MailboxID: mb.ID})
418 q.FilterEqual("Expunged", false)
419 if mb.ID == c.mailboxID {
420 q.FilterLess("UID", c.uidnext)
423 for m, err := range q.All() {
424 xcheckf(err, "list messages in mailbox")
426 // We track this for the "reverse" case, we'll stop before seeing lastUID.
429 if time.Since(inProgressLast) > inProgressPeriod {
430 c.xwritelinef("* OK [INPROGRESS (%s %d %s)] still searching", inProgressTag, progress, goal)
431 inProgressLast = time.Now()
435 if c.searchMatch(tx, msgCount, seq, m, *sk, bodySearch, textSearch, xhighestUID) {
436 result.UIDs = append(result.UIDs, m.UID)
437 result.MaxModSeq = max(result.MaxModSeq, m.ModSeq)
438 if min1 == 1 && min1+max1 == len(eargs) {
442 // We only need a MIN and a MAX, but we also need sequence numbers so we are
443 // walking through and collecting all UIDs. Correct for that, keeping only the MIN
446 if len(result.UIDs) == 3 {
447 result.UIDs[1] = result.UIDs[2]
448 result.UIDs = result.UIDs[:2]
455 // And reverse search for MAX if we have only MAX or MAX combined with MIN, and
456 // don't need sequence numbers. We just need a single match, then we stop.
458 q := bstore.QueryTx[store.Message](tx)
459 q.FilterNonzero(store.Message{MailboxID: mb.ID})
460 q.FilterEqual("Expunged", false)
461 q.FilterGreater("UID", lastUID)
462 if mb.ID == c.mailboxID {
463 q.FilterLess("UID", c.uidnext)
466 for m, err := range q.All() {
467 xcheckf(err, "list messages in mailbox")
469 if time.Since(inProgressLast) > inProgressPeriod {
470 c.xwritelinef("* OK [INPROGRESS (%s %d %s)] still searching", inProgressTag, progress, goal)
471 inProgressLast = time.Now()
475 var seq msgseq // Filled in by searchMatch for messages in selected mailbox.
476 if c.searchMatch(tx, msgCount, seq, m, *sk, bodySearch, textSearch, xhighestUID) {
477 result.UIDs = append(result.UIDs, m.UID)
478 result.MaxModSeq = max(result.MaxModSeq, m.ModSeq)
484 // We could have finished searching the mailbox with fewer
485 mailboxProcessed := progress - progressOrig
486 mailboxTotal := uint32(mb.MailboxCounts.Total + mb.MailboxCounts.Deleted)
487 progress += max(0, mailboxTotal-mailboxProcessed)
489 results = append(results, result)
494 // We'll only have a result for the one selected mailbox.
497 // Old-style SEARCH response: a single untagged line listing all matching
498 // numbers. RFC 3501 specifies one response; clients like emersion/go-imap
499 // (used by aerc) only retain the last untagged SEARCH response, so any
500 // split truncates the visible result set. See issue #389.
502 var s strings.Builder
503 for _, v := range result.UIDs {
505 v = store.UID(c.xsequence(v))
508 s.WriteString(strconv.FormatUint(uint64(v), 10))
511 // MODSEQ is only attached when there were matches: RFC 7162 ties the
512 // returned modseq to "all messages being returned", so an empty result
516 if sk.hasModseq() && len(result.UIDs) > 0 {
517 modseq = fmt.Sprintf(" (MODSEQ %d)", result.MaxModSeq.Client())
519 c.xbwritelinef("* SEARCH%s%s", s.String(), modseq)
525 c.searchResult = results[0].UIDs
526 c.checkUIDs(c.searchResult, false)
531 for _, result := range results {
532 // For the ESEARCH command, we must not return a response if there were no matching
533 // messages. This is unlike the later IMAP4rev2, where an ESEARCH response must be
535 if isE && len(result.UIDs) == 0 {
539 // The tag was originally a string, became an astring in IMAP4rev2, better stick to
542 fmt.Fprintf(c.xbw, `* ESEARCH (TAG "%s" MAILBOX %s UIDVALIDITY %d)`, tag, result.Mailbox.Name, result.Mailbox.UIDValidity)
544 fmt.Fprintf(c.xbw, `* ESEARCH (TAG "%s")`, tag)
547 fmt.Fprintf(c.xbw, " UID")
550 // NOTE: we are potentially converting UIDs to msgseq, but keep the store.UID type
554 // If searchResult is hanging on to the slice, we need to work on a copy.
556 nums = slices.Clone(nums)
558 for i, uid := range nums {
559 nums[i] = store.UID(c.xsequence(uid))
564 if eargs["MIN"] && len(nums) > 0 {
565 fmt.Fprintf(c.xbw, " MIN %d", nums[0])
567 if eargs["MAX"] && len(result.UIDs) > 0 {
568 fmt.Fprintf(c.xbw, " MAX %d", nums[len(nums)-1])
571 fmt.Fprintf(c.xbw, " COUNT %d", len(nums))
573 if eargs["ALL"] && len(nums) > 0 {
574 fmt.Fprintf(c.xbw, " ALL %s", compactUIDSet(nums).String())
578 // Summary: send the highest modseq of the returned messages.
579 if sk.hasModseq() && len(nums) > 0 {
580 fmt.Fprintf(c.xbw, " MODSEQ %d", result.MaxModSeq.Client())
594 msgCount uint32 // Number of messages in mailbox (or session when selected).
595 seq msgseq // Can be 0, for other mailboxes than selected in case of MAX.
599 xhighestUID func() store.UID
602func (c *conn) searchMatch(tx *bstore.Tx, msgCount uint32, seq msgseq, m store.Message, sk searchKey, bodySearch, textSearch *store.WordSearch, xhighestUID func() store.UID) bool {
603 if m.MailboxID == c.mailboxID {
604 // If session doesn't know about the message yet, don't return it.
606 if m.UID >= c.uidnext {
610 // Set seq for use in evaluations.
611 seq = c.sequence(m.UID)
618 s := search{c: c, tx: tx, msgCount: msgCount, seq: seq, m: m, xhighestUID: xhighestUID}
622 c.xsanity(err, "closing messagereader")
626 return s.match(sk, bodySearch, textSearch)
629func (s *search) match(sk searchKey, bodySearch, textSearch *store.WordSearch) (match bool) {
631 if match && bodySearch != nil {
632 if !s.xensurePart() {
637 match, err = bodySearch.MatchPart(s.c.log, s.p, false)
638 xcheckf(err, "search words in bodies")
640 if match && textSearch != nil {
641 if !s.xensurePart() {
646 match, err = textSearch.MatchPart(s.c.log, s.p, true)
647 xcheckf(err, "search words in headers and bodies")
652// ensure message, reader and part are loaded. returns whether that was
654func (s *search) xensurePart() bool {
659 // Closed by searchMatch after all (recursive) search.match calls are finished.
660 s.mr = s.c.account.MessageReader(s.m)
662 if s.m.ParsedBuf == nil {
663 s.c.log.Error("missing parsed message")
666 p, err := s.m.LoadPart(s.mr)
667 xcheckf(err, "load parsed message")
672func (s *search) match0(sk searchKey) bool {
675 // Difference between sk.searchKeys nil and length 0 is important. Because we take
676 // out word/notword searches, the list may be empty but non-nil.
677 if sk.searchKeys != nil {
678 for _, ssk := range sk.searchKeys {
684 } else if sk.seqSet != nil {
685 if sk.seqSet.searchResult {
686 // Interpreting search results on a mailbox that isn't selected during multisearch
688 if s.m.MailboxID != c.mailboxID {
689 xuserErrorf("can only use search result with the selected mailbox")
691 return uidSearch(c.searchResult, s.m.UID) > 0
693 // For multisearch, we have arranged to have a seq for non-selected mailboxes too.
694 return sk.seqSet.containsSeqCount(s.seq, s.msgCount)
697 filterHeader := func(field, value string) bool {
698 lower := strings.ToLower(value)
699 h, err := s.p.Header()
701 c.log.Debugx("parsing message header", err, slog.Any("uid", s.m.UID), slog.Int64("msgid", s.m.ID))
704 for _, v := range h.Values(field) {
705 if strings.Contains(strings.ToLower(v), lower) {
712 // We handle ops by groups that need increasing details about the message.
718 // We do not implement the RECENT flag, so messages cannot be NEW.
721 // We treat all messages as non-recent, so this means all messages.
724 // We do not implement the RECENT flag. All messages are not recent.
727 return !s.match0(*sk.searchKey)
729 return s.match0(*sk.searchKey) || s.match0(*sk.searchKey2)
731 if sk.uidSet.searchResult && s.m.MailboxID != c.mailboxID {
732 // Interpreting search results on a mailbox that isn't selected during multisearch
734 xuserErrorf("cannot use search result from another mailbox")
736 return sk.uidSet.xcontainsKnownUID(s.m.UID, c.searchResult, s.xhighestUID)
740 if !s.xensurePart() {
744 // Parsed message, basic info.
753 kw := strings.ToLower(sk.atom)
766 return slices.Contains(s.m.Keywords, kw)
777 kw := strings.ToLower(sk.atom)
780 return !s.m.Forwarded
790 return !slices.Contains(s.m.Keywords, kw)
798 case "BEFORE", "ON", "SINCE":
799 skdt := sk.date.Format("2006-01-02")
800 rdt := s.m.Received.Format("2006-01-02")
809 panic("missing case")
811 return s.m.Size > sk.number
813 return s.m.Size < sk.number
816 return s.m.ModSeq.Client() >= *sk.clientModseq
817 case "SAVEDBEFORE", "SAVEDON", "SAVEDSINCE":
818 // If we don't have a savedate for this message (for messages received before we
819 // implemented this feature), we use the "internal date" (received timestamp) of
822 if s.m.SaveDate != nil {
826 skdt := sk.date.Format("2006-01-02")
827 rdt := rt.Format("2006-01-02")
836 panic("missing case")
837 case "SAVEDATESUPPORTED":
838 // We return whether we have a savedate for this message. We support it on all
839 // mailboxes, but we only have this metadata from the time we implemented this
841 return s.m.SaveDate != nil
844 seconds := int64(time.Since(s.m.Received) / time.Second)
845 return seconds >= sk.number
847 seconds := int64(time.Since(s.m.Received) / time.Second)
848 return seconds <= sk.number
852 c.log.Info("missing parsed message, not matching", slog.Any("uid", s.m.UID), slog.Int64("msgid", s.m.ID))
856 // Parsed message, more info.
859 return filterHeader("Bcc", sk.astring)
861 // We gathered word/notword searches from the top-level, but we can also get them
863 // todo optimize: handle deeper nested word/not-word searches more efficiently.
864 headerToo := sk.op == "TEXT"
865 match, err := store.PrepareWordSearch([]string{sk.astring}, nil).MatchPart(s.c.log, s.p, headerToo)
866 xcheckf(err, "word search")
869 return filterHeader("Cc", sk.astring)
871 return filterHeader("From", sk.astring)
873 return filterHeader("Subject", sk.astring)
875 return filterHeader("To", sk.astring)
878 lower := strings.ToLower(sk.astring)
879 h, err := s.p.Header()
881 c.log.Errorx("parsing header for search", err, slog.Any("uid", s.m.UID), slog.Int64("msgid", s.m.ID))
884 k := textproto.CanonicalMIMEHeaderKey(sk.headerField)
885 for _, v := range h.Values(k) {
886 if lower == "" || strings.Contains(strings.ToLower(v), lower) {
891 case "SENTBEFORE", "SENTON", "SENTSINCE":
892 if s.p.Envelope == nil || s.p.Envelope.Date.IsZero() {
895 dt := s.p.Envelope.Date.Format("2006-01-02")
896 skdt := sk.date.Format("2006-01-02")
905 panic("missing case")
907 panic(serverError{fmt.Errorf("missing case for search key op %q", sk.op)})