1package smtpserver
2
3import (
4 "context"
5 "fmt"
6 "log/slog"
7 "net"
8 "os"
9 "strings"
10 "time"
11
12 "github.com/mjl-/bstore"
13
14 "github.com/mjl-/mox/config"
15 "github.com/mjl-/mox/dkim"
16 "github.com/mjl-/mox/dmarc"
17 "github.com/mjl-/mox/dmarcrpt"
18 "github.com/mjl-/mox/dns"
19 "github.com/mjl-/mox/dnsbl"
20 "github.com/mjl-/mox/iprev"
21 "github.com/mjl-/mox/message"
22 "github.com/mjl-/mox/mlog"
23 "github.com/mjl-/mox/mox-"
24 "github.com/mjl-/mox/publicsuffix"
25 "github.com/mjl-/mox/smtp"
26 "github.com/mjl-/mox/store"
27 "github.com/mjl-/mox/subjectpass"
28 "github.com/mjl-/mox/tlsrpt"
29)
30
31type delivery struct {
32 tls bool
33 m *store.Message
34 dataFile *os.File
35 smtpRcptTo smtp.Path // As used in SMTP, possibly address of alias.
36 deliverTo smtp.Path // To deliver to, either smtpRcptTo or an alias member address.
37 destination config.Destination
38 canonicalAddress string
39 acc *store.Account
40 msgTo []message.Address
41 msgCc []message.Address
42 msgFrom smtp.Address
43 dnsBLs []dns.Domain
44 dmarcUse bool
45 dmarcResult dmarc.Result
46 dkimResults []dkim.Result
47 iprevStatus iprev.Status
48 smtputf8 bool
49}
50
51type analysis struct {
52 d delivery
53 accept bool
54 mailbox string
55 code int
56 secode string
57 userError bool
58 errmsg string
59 err error // For our own logging, not sent to remote.
60 dmarcReport *dmarcrpt.Feedback // Validated DMARC aggregate report, not yet stored.
61 tlsReport *tlsrpt.Report // Validated TLS report, not yet stored.
62 reason string // If non-empty, reason for this decision. Values from reputationMethod and reason* below.
63 reasonText []string // Additional details for reason, human-readable, added to X-Mox-Reason header.
64 dmarcOverrideReason string // If set, one of dmarcrpt.PolicyOverride
65 // Additional headers to add during delivery. Used for reasons a message to a
66 // dmarc/tls reporting address isn't processed.
67 headers string
68}
69
70const (
71 reasonListAllow = "list-allow"
72 reasonDMARCPolicy = "dmarc-policy"
73 reasonReputationError = "reputation-error"
74 reasonReporting = "reporting"
75 reasonSPFPolicy = "spf-policy"
76 reasonJunkClassifyError = "junk-classify-error"
77 reasonJunkFilterError = "junk-filter-error"
78 reasonGiveSubjectpass = "give-subjectpass"
79 reasonNoBadSignals = "no-bad-signals"
80 reasonJunkContent = "junk-content"
81 reasonJunkContentStrict = "junk-content-strict"
82 reasonDNSBlocklisted = "dns-blocklisted"
83 reasonSubjectpass = "subjectpass"
84 reasonSubjectpassError = "subjectpass-error"
85 reasonIPrev = "iprev" // No or mild junk reputation signals, and bad iprev.
86 reasonHighRate = "high-rate" // Too many messages, not added to rejects.
87 reasonMsgAuthRequired = "msg-auth-required"
88)
89
90func isListDomain(d delivery, ld dns.Domain) bool {
91 if d.m.MailFromValidated && ld.Name() == d.m.MailFromDomain {
92 return true
93 }
94 for _, r := range d.dkimResults {
95 if r.Status == dkim.StatusPass && r.Sig.Domain == ld {
96 return true
97 }
98 }
99 return false
100}
101
102func analyze(ctx context.Context, log mlog.Log, resolver dns.Resolver, d delivery) analysis {
103 var headers string
104
105 var reasonText []string
106 addReasonText := func(format string, args ...any) {
107 s := fmt.Sprintf(format, args...)
108 reasonText = append(reasonText, s)
109 }
110
111 // We don't want to let a single IP or network deliver too many messages to an
112 // account. They may fill up the mailbox, either with messages that have to be
113 // purged, or by filling the disk. We check both cases for IP's and networks.
114 var rateError bool // Whether returned error represents a rate error.
115 err := d.acc.DB.Read(ctx, func(tx *bstore.Tx) (retErr error) {
116 now := time.Now()
117 defer func() {
118 log.Debugx("checking message and size delivery rates", retErr, slog.Duration("duration", time.Since(now)))
119 }()
120
121 checkCount := func(msg store.Message, window time.Duration, limit int) {
122 if retErr != nil {
123 return
124 }
125 q := bstore.QueryTx[store.Message](tx)
126 q.FilterNonzero(msg)
127 q.FilterGreater("Received", now.Add(-window))
128 q.FilterEqual("Expunged", false)
129 n, err := q.Count()
130 if err != nil {
131 retErr = err
132 return
133 }
134 if n >= limit {
135 rateError = true
136 retErr = fmt.Errorf("more than %d messages in past %s from your ip/network", limit, window)
137 }
138 }
139
140 checkSize := func(msg store.Message, window time.Duration, limit int64) {
141 if retErr != nil {
142 return
143 }
144 q := bstore.QueryTx[store.Message](tx)
145 q.FilterNonzero(msg)
146 q.FilterGreater("Received", now.Add(-window))
147 q.FilterEqual("Expunged", false)
148 size := d.m.Size
149 err := q.ForEach(func(v store.Message) error {
150 size += v.Size
151 return nil
152 })
153 if err != nil {
154 retErr = err
155 return
156 }
157 if size > limit {
158 rateError = true
159 retErr = fmt.Errorf("more than %d bytes in past %s from your ip/network", limit, window)
160 }
161 }
162
163 // todo future: make these configurable
164 // todo: should we have a limit for forwarded messages? they are stored with empty RemoteIPMasked*
165
166 const day = 24 * time.Hour
167 checkCount(store.Message{RemoteIPMasked1: d.m.RemoteIPMasked1}, time.Minute, limitIPMasked1MessagesPerMinute)
168 checkCount(store.Message{RemoteIPMasked1: d.m.RemoteIPMasked1}, day, 20*500)
169 checkCount(store.Message{RemoteIPMasked2: d.m.RemoteIPMasked2}, time.Minute, 1500)
170 checkCount(store.Message{RemoteIPMasked2: d.m.RemoteIPMasked2}, day, 20*1500)
171 checkCount(store.Message{RemoteIPMasked3: d.m.RemoteIPMasked3}, time.Minute, 4500)
172 checkCount(store.Message{RemoteIPMasked3: d.m.RemoteIPMasked3}, day, 20*4500)
173
174 const MB = 1024 * 1024
175 checkSize(store.Message{RemoteIPMasked1: d.m.RemoteIPMasked1}, time.Minute, limitIPMasked1SizePerMinute)
176 checkSize(store.Message{RemoteIPMasked1: d.m.RemoteIPMasked1}, day, 3*1000*MB)
177 checkSize(store.Message{RemoteIPMasked2: d.m.RemoteIPMasked2}, time.Minute, 3000*MB)
178 checkSize(store.Message{RemoteIPMasked2: d.m.RemoteIPMasked2}, day, 3*3000*MB)
179 checkSize(store.Message{RemoteIPMasked3: d.m.RemoteIPMasked3}, time.Minute, 9000*MB)
180 checkSize(store.Message{RemoteIPMasked3: d.m.RemoteIPMasked3}, day, 3*9000*MB)
181
182 return retErr
183 })
184 if err != nil && !rateError {
185 log.Errorx("checking delivery rates", err)
186 metricDelivery.WithLabelValues("checkrates", "").Inc()
187 addReasonText("checking delivery rates: %v", err)
188 return analysis{d, false, "", smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing", err, nil, nil, reasonReputationError, reasonText, "", headers}
189 } else if err != nil {
190 log.Debugx("refusing due to high delivery rate", err)
191 metricDelivery.WithLabelValues("highrate", "").Inc()
192 addReasonText("high delivery rate")
193 return analysis{d, false, "", smtp.C452StorageFull, smtp.SeMailbox2Full2, true, err.Error(), err, nil, nil, reasonHighRate, reasonText, "", headers}
194 }
195
196 mailbox := d.destination.Mailbox
197 if mailbox == "" {
198 mailbox = "Inbox"
199 }
200
201 // If destination mailbox has a mailing list domain (for SPF/DKIM) configured,
202 // check it for a pass.
203 rs := store.MessageRuleset(log, d.destination, d.m, d.m.MsgPrefix, d.dataFile)
204 if rs != nil {
205 mailbox = rs.Mailbox
206 }
207 if rs != nil && !rs.ListAllowDNSDomain.IsZero() {
208 // todo: on temporary failures, reject temporarily?
209 if isListDomain(d, rs.ListAllowDNSDomain) {
210 addReasonText("validated message from a configured mailing list")
211 d.m.IsMailingList = true
212 return analysis{
213 d: d,
214 accept: true,
215 mailbox: mailbox,
216 reason: reasonListAllow,
217 reasonText: reasonText,
218 dmarcOverrideReason: string(dmarcrpt.PolicyOverrideMailingList),
219 headers: headers,
220 }
221 }
222 }
223
224 var dmarcOverrideReason string
225
226 // For forwarded messages, we have different junk analysis. We don't reject for
227 // failing DMARC, and we clear fields that could implicate the forwarding mail
228 // server during future classifications on incoming messages (the forwarding mail
229 // server isn't responsible for the message).
230 if rs != nil && rs.IsForward {
231 d.dmarcUse = false
232 d.m.IsForward = true
233 d.m.RemoteIPMasked1 = ""
234 d.m.RemoteIPMasked2 = ""
235 d.m.RemoteIPMasked3 = ""
236 d.m.OrigEHLODomain = d.m.EHLODomain
237 d.m.EHLODomain = ""
238 d.m.MailFromDomain = "" // Still available in MailFrom.
239 d.m.OrigDKIMDomains = d.m.DKIMDomains
240 dkimdoms := []string{}
241 for _, dom := range d.m.DKIMDomains {
242 if dom != rs.VerifiedDNSDomain.Name() {
243 dkimdoms = append(dkimdoms, dom)
244 }
245 }
246 d.m.DKIMDomains = dkimdoms
247 dmarcOverrideReason = string(dmarcrpt.PolicyOverrideForwarded)
248 log.Info("forwarded message, clearing identifying signals of forwarding mail server")
249 addReasonText("ruleset indicates forwarded message")
250 }
251
252 assignMailbox := func(tx *bstore.Tx) error {
253 // Set message MailboxID to which mail will be delivered. Reputation is
254 // per-mailbox. If referenced mailbox is not found (e.g. does not yet exist), we
255 // can still determine a reputation because we also base it on outgoing
256 // messages and those are account-global.
257 mb, err := d.acc.MailboxFind(tx, mailbox)
258 if err != nil {
259 return fmt.Errorf("finding destination mailbox: %w", err)
260 }
261 if mb != nil {
262 // We want to deliver to mb.ID, but this message may be rejected and sent to the
263 // Rejects mailbox instead, with MailboxID overwritten. Record the ID in
264 // MailboxDestinedID too. If the message is later moved out of the Rejects mailbox,
265 // we'll adjust the MailboxOrigID so it gets taken into account during reputation
266 // calculating in future deliveries. If we end up delivering to the intended
267 // mailbox (i.e. not rejecting), MailboxDestinedID is cleared during delivery so we
268 // don't store it unnecessarily.
269 d.m.MailboxID = mb.ID
270 d.m.MailboxDestinedID = mb.ID
271 } else {
272 log.Debug("mailbox not found in database", slog.String("mailbox", mailbox))
273 }
274 return nil
275 }
276
277 reject := func(code int, secode string, errmsg string, err error, reason string) analysis {
278 // We may have set MailboxDestinedID below already while we had a transaction. If
279 // not, do it now. This makes it possible to use the per-mailbox reputation when a
280 // user moves the message out of the Rejects mailbox to the intended mailbox
281 // (typically Inbox).
282 if d.m.MailboxDestinedID == 0 {
283 var mberr error
284 d.acc.WithRLock(func() {
285 mberr = d.acc.DB.Read(ctx, func(tx *bstore.Tx) error {
286 return assignMailbox(tx)
287 })
288 })
289 if mberr != nil {
290 addReasonText("error setting original destination mailbox for rejected message: %v", mberr)
291 return analysis{d, false, mailbox, smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing", err, nil, nil, reasonReputationError, reasonText, dmarcOverrideReason, headers}
292 }
293 d.m.MailboxID = 0 // We plan to reject, no need to set intended MailboxID.
294 }
295
296 accept := false
297 if rs != nil && rs.AcceptRejectsToMailbox != "" {
298 accept = true
299 mailbox = rs.AcceptRejectsToMailbox
300 d.m.IsReject = true
301 // Don't draw attention, but don't go so far as to mark as junk.
302 d.m.Seen = true
303 log.Info("accepting reject to configured mailbox due to ruleset")
304 addReasonText("accepting reject to mailbox due to ruleset")
305 }
306 return analysis{d, accept, mailbox, code, secode, err == nil, errmsg, err, nil, nil, reason, reasonText, dmarcOverrideReason, headers}
307 }
308
309 if d.dmarcUse && d.dmarcResult.Reject {
310 addReasonText("message does not pass domain dmarc policy which asks to reject")
311 return reject(smtp.C550MailboxUnavail, smtp.SePol7MultiAuthFails26, "rejecting per dmarc policy", nil, reasonDMARCPolicy)
312 } else if !d.dmarcUse {
313 addReasonText("not using any dmarc result")
314 } else {
315 addReasonText("dmarc ok")
316 }
317 // todo: should we also reject messages that have a dmarc pass but an spf record "v=spf1 -all"? suggested by m3aawg best practices.
318
319 // If destination is the DMARC reporting mailbox, do additional checks and keep
320 // track of the report. We'll check reputation, defaulting to accept.
321 var dmarcReport *dmarcrpt.Feedback
322 if d.destination.DMARCReports {
323 // Messages with DMARC aggregate reports must have a DMARC pass. ../rfc/7489:1866
324 if d.dmarcResult.Status != dmarc.StatusPass {
325 log.Info("received dmarc aggregate report without dmarc pass, not processing as dmarc report")
326 headers += "X-Mox-DMARCReport-Error: no DMARC pass\r\n"
327 } else if report, err := dmarcrpt.ParseMessageReport(log.Logger, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil {
328 log.Infox("parsing dmarc aggregate report", err)
329 headers += "X-Mox-DMARCReport-Error: could not parse report\r\n"
330 } else if d, err := dns.ParseDomain(report.PolicyPublished.Domain); err != nil {
331 log.Infox("parsing domain in dmarc aggregate report", err)
332 headers += "X-Mox-DMARCReport-Error: could not parse domain in published policy\r\n"
333 } else if _, ok := mox.Conf.Domain(d); !ok {
334 log.Info("dmarc aggregate report for domain not configured, ignoring", slog.Any("domain", d))
335 headers += "X-Mox-DMARCReport-Error: published policy domain unrecognized\r\n"
336 } else if report.ReportMetadata.DateRange.End > time.Now().Unix()+60 {
337 log.Info("dmarc aggregate report with end date in the future, ignoring", slog.Any("domain", d), slog.Time("end", time.Unix(report.ReportMetadata.DateRange.End, 0)))
338 headers += "X-Mox-DMARCReport-Error: report has end date in the future\r\n"
339 } else {
340 dmarcReport = report
341 }
342 }
343
344 // Similar to DMARC reporting, we check for the required DKIM. We'll check
345 // reputation, defaulting to accept.
346 var tlsReport *tlsrpt.Report
347 if d.destination.HostTLSReports || d.destination.DomainTLSReports {
348 matchesDomain := func(sigDomain dns.Domain) bool {
349 // RFC seems to require exact DKIM domain match with submitt and message From, we
350 // also allow msgFrom to be subdomain. ../rfc/8460:322
351 return sigDomain == d.msgFrom.Domain || strings.HasSuffix(d.msgFrom.Domain.ASCII, "."+sigDomain.ASCII) && publicsuffix.Lookup(ctx, log.Logger, d.msgFrom.Domain) == publicsuffix.Lookup(ctx, log.Logger, sigDomain)
352 }
353 // Valid DKIM signature for domain must be present. We take "valid" to assume
354 // "passing", not "syntactically valid". We also check for "tlsrpt" as service.
355 // This check is optional, but if anyone goes through the trouble to explicitly
356 // list allowed services, they would be surprised to see them ignored.
357 // ../rfc/8460:320
358 ok := false
359 for _, r := range d.dkimResults {
360 // The record should have an allowed service "tlsrpt". The RFC mentions it as if
361 // the service must be specified explicitly, but the default allowed services for a
362 // DKIM record are "*", which includes "tlsrpt". Unless a DKIM record explicitly
363 // specifies services (e.g. s=email), a record will work for TLS reports. The DKIM
364 // records seen used for TLS reporting in the wild don't explicitly set "s" for
365 // services.
366 // ../rfc/8460:326
367 if r.Status == dkim.StatusPass && matchesDomain(r.Sig.Domain) && r.Sig.Length < 0 && r.Record.ServiceAllowed("tlsrpt") {
368 ok = true
369 break
370 }
371 }
372
373 if !ok {
374 log.Info("received mail to tlsrpt without acceptable DKIM signature, not processing as tls report")
375 headers += "X-Mox-TLSReport-Error: no acceptable DKIM signature\r\n"
376 } else if reportJSON, err := tlsrpt.ParseMessage(log.Logger, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil {
377 log.Infox("parsing tls report", err)
378 headers += "X-Mox-TLSReport-Error: could not parse TLS report\r\n"
379 } else {
380 var known bool
381 for _, p := range reportJSON.Policies {
382 log.Info("tlsrpt policy domain", slog.String("domain", p.Policy.Domain))
383 if d, err := dns.ParseDomain(p.Policy.Domain); err != nil {
384 log.Infox("parsing domain in tls report", err)
385 } else if _, ok := mox.Conf.Domain(d); ok || d == mox.Conf.Static.HostnameDomain {
386 known = true
387 break
388 }
389 }
390 if !known {
391 log.Info("tls report without one of configured domains, ignoring")
392 headers += "X-Mox-TLSReport-Error: report for unknown domain\r\n"
393 } else {
394 report := reportJSON.Convert()
395 tlsReport = &report
396 }
397 }
398 }
399
400 // We may have to reject messages that don't pass a relaxed aligned SPF and/or DKIM
401 // check. Useful for services with autoresponders.
402 if d.destination.MessageAuthRequiredSMTPError != "" && !d.m.MsgFromValidated {
403 code := smtp.C550MailboxUnavail
404 msg := d.destination.MessageAuthRequiredSMTPError
405 if d.dmarcResult.Status == dmarc.StatusTemperror {
406 code = smtp.C451LocalErr
407 msg = "transient verification error: " + msg
408 }
409 addReasonText("message does not pass required aligned spf and/or dkim check required for destination")
410 return reject(code, smtp.SePol7MultiAuthFails26, msg, nil, reasonMsgAuthRequired)
411 }
412
413 // Determine if message is acceptable based on DMARC domain, DKIM identities, or
414 // host-based reputation.
415 var isjunk *bool
416 var conclusive bool
417 var method reputationMethod
418 var reason string
419 d.acc.WithRLock(func() {
420 err = d.acc.DB.Read(ctx, func(tx *bstore.Tx) error {
421 if err := assignMailbox(tx); err != nil {
422 return err
423 }
424
425 var text string
426 isjunk, conclusive, method, text, err = reputation(tx, log, d.m, d.smtputf8)
427 reason = string(method)
428 s := "address/dkim/spf/ip-based reputation ("
429 if isjunk != nil && *isjunk {
430 s += "junk, "
431 } else if isjunk != nil && !*isjunk {
432 s += "nonjunk, "
433 }
434 if conclusive {
435 s += "conclusive"
436 } else {
437 s += "inconclusive"
438 }
439 s += ", " + text + ")"
440 addReasonText("%s", s)
441 return err
442 })
443 })
444 if err != nil {
445 log.Infox("determining reputation", err, slog.Any("message", d.m))
446 addReasonText("determining reputation: %v", err)
447 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonReputationError)
448 }
449 log.Info("reputation analyzed",
450 slog.Bool("conclusive", conclusive),
451 slog.Any("isjunk", isjunk),
452 slog.String("method", string(method)))
453 if conclusive {
454 if !*isjunk {
455 return analysis{
456 d: d,
457 accept: true,
458 mailbox: mailbox,
459 dmarcReport: dmarcReport,
460 tlsReport: tlsReport,
461 reason: reason,
462 reasonText: reasonText,
463 dmarcOverrideReason: dmarcOverrideReason,
464 headers: headers,
465 }
466 }
467 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, string(method))
468 } else if dmarcReport != nil || tlsReport != nil {
469 log.Info("accepting message with dmarc aggregate report or tls report without reputation")
470 addReasonText("message inconclusive reputation but with dmarc or tls report")
471 return analysis{
472 d: d,
473 accept: true,
474 mailbox: mailbox,
475 dmarcReport: dmarcReport,
476 tlsReport: tlsReport,
477 reason: reasonReporting,
478 reasonText: reasonText,
479 dmarcOverrideReason: dmarcOverrideReason,
480 headers: headers,
481 }
482 }
483 // If there was no previous message from sender or its domain, and we have an SPF
484 // (soft)fail, reject the message.
485 switch method {
486 case methodDKIMSPF, methodIP1, methodIP2, methodIP3, methodNone:
487 switch d.m.MailFromValidation {
488 case store.ValidationFail, store.ValidationSoftfail:
489 addReasonText("no previous message from sender domain and spf result is (soft)fail")
490 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", nil, reasonSPFPolicy)
491 }
492 }
493
494 // Senders without reputation and without iprev pass, are likely spam.
495 var suspiciousIPrevFail bool
496 switch method {
497 case methodDKIMSPF, methodIP1, methodIP2, methodIP3, methodNone:
498 suspiciousIPrevFail = d.iprevStatus != iprev.StatusPass
499 }
500 if suspiciousIPrevFail {
501 addReasonText("suspicious iprev failure")
502 }
503
504 // With already a mild junk signal, an iprev fail on top is enough to reject.
505 if suspiciousIPrevFail && isjunk != nil && *isjunk {
506 addReasonText("message has a mild junk signal and mismatching reverse ip")
507 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", nil, reasonIPrev)
508 }
509
510 var subjectpassKey string
511 conf, _ := d.acc.Conf()
512 if conf.SubjectPass.Period > 0 {
513 subjectpassKey, err = d.acc.Subjectpass(d.canonicalAddress)
514 if err != nil {
515 log.Errorx("get key for verifying subject token", err)
516 addReasonText("subject pass error: %v", err)
517 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonSubjectpassError)
518 }
519 err = subjectpass.Verify(log.Logger, d.dataFile, []byte(subjectpassKey), conf.SubjectPass.Period)
520 pass := err == nil
521 log.Infox("pass by subject token", err, slog.Bool("pass", pass))
522 if pass {
523 addReasonText("message has valid subjectpass token in subject")
524 return analysis{
525 d: d,
526 accept: true,
527 mailbox: mailbox,
528 reason: reasonSubjectpass,
529 reasonText: reasonText,
530 dmarcOverrideReason: dmarcOverrideReason,
531 headers: headers,
532 }
533 }
534 }
535
536 reason = reasonNoBadSignals
537 accept := true
538 var junkSubjectpass bool
539 f, jf, err := d.acc.OpenJunkFilter(ctx, log)
540 if err == nil {
541 defer func() {
542 err := f.Close()
543 log.Check(err, "closing junkfilter")
544 }()
545 result, err := f.ClassifyMessageReader(ctx, store.FileMsgReader(d.m.MsgPrefix, d.dataFile), d.m.Size)
546 if err != nil {
547 log.Errorx("testing for spam", err)
548 addReasonText("classify message error: %v", err)
549 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonJunkClassifyError)
550 }
551 // todo: if isjunk is not nil (i.e. there was inconclusive reputation), use it in the probability calculation. give reputation a score of 0.25 or .75 perhaps?
552 // todo: if there aren't enough historic messages, we should just let messages in.
553 // todo: we could require nham and nspam to be above a certain number when there were plenty of words in the message, and in the database. can indicate a spammer is misspelling words. however, it can also mean a message in a different language/script...
554
555 // If we don't accept, we may still respond with a "subjectpass" hint below.
556 // We add some jitter to the threshold we use. So we don't act as too easy an
557 // oracle for words that are a strong indicator of haminess.
558 // todo: we should rate-limit uses of the junkfilter.
559 jitter := (jitterRand.Float64() - 0.5) / 10
560 threshold := jf.Threshold + jitter
561
562 rcptToMatch := func(l []message.Address) bool {
563 // todo: we use Go's net/mail to parse message header addresses. it does not allow empty quoted strings (contrary to spec), leaving To empty. so we don't verify To address for that unusual case for now. ../rfc/5322:961 ../rfc/5322:743
564 if d.smtpRcptTo.Localpart == "" {
565 return true
566 }
567 for _, a := range l {
568 dom, err := dns.ParseDomain(a.Host)
569 if err != nil {
570 continue
571 }
572 lp, err := smtp.ParseLocalpart(a.User)
573 if err == nil && dom == d.smtpRcptTo.IPDomain.Domain && lp == d.smtpRcptTo.Localpart {
574 return true
575 }
576 }
577 return false
578 }
579
580 // todo: some of these checks should also apply for reputation-based analysis with a weak signal, e.g. verified dkim/spf signal from new domain.
581 // With an iprev fail, non-TLS connection or our address not in To/Cc header, we set a higher bar for content.
582 reason = reasonJunkContent
583 var thresholdRemark string
584 if suspiciousIPrevFail && threshold > 0.25 {
585 threshold = 0.25
586 log.Info("setting junk threshold due to iprev fail", slog.Float64("threshold", threshold))
587 reason = reasonJunkContentStrict
588 thresholdRemark = " (stricter due to reverse ip mismatch)"
589 } else if !d.tls && threshold > 0.25 {
590 threshold = 0.25
591 log.Info("setting junk threshold due to plaintext smtp", slog.Float64("threshold", threshold))
592 reason = reasonJunkContentStrict
593 thresholdRemark = " (stricter due to missing tls)"
594 } else if (rs == nil || !rs.IsForward) && threshold > 0.25 && !rcptToMatch(d.msgTo) && !rcptToMatch(d.msgCc) {
595 // A common theme in junk messages is your recipient address not being in the To/Cc
596 // headers. We may be in Bcc, but that's unusual for first-time senders. Some
597 // providers (e.g. gmail) does not DKIM-sign Bcc headers, so junk messages can be
598 // sent with matching Bcc headers. We don't get here for known senders.
599 threshold = 0.25
600 log.Info("setting junk threshold due to smtp rcpt to and message to/cc address mismatch", slog.Float64("threshold", threshold))
601 reason = reasonJunkContentStrict
602 thresholdRemark = " (stricter due to recipient address not in to/cc header)"
603 }
604 accept = result.Probability <= threshold || (!result.Significant && !suspiciousIPrevFail)
605 junkSubjectpass = result.Probability < threshold-0.2
606 log.Info("content analyzed",
607 slog.Bool("accept", accept),
608 slog.Float64("contentprob", result.Probability),
609 slog.Bool("contentsignificant", result.Significant),
610 slog.Bool("subjectpass", junkSubjectpass))
611
612 s := "content: "
613 if accept {
614 s += "not junk"
615 } else {
616 s += "junk"
617 }
618 if !result.Significant {
619 s += " (not significant)"
620 }
621 s += fmt.Sprintf(", spamscore %.2f, threshold %.2f%s", result.Probability, threshold, thresholdRemark)
622 s += " (ham words: "
623 for i, w := range result.Hams {
624 if i > 0 {
625 s += ", "
626 }
627 word := w.Word
628 if !d.smtputf8 && !isASCII(word) {
629 word = "(non-ascii)"
630 }
631 s += fmt.Sprintf("%s %.3f", word, w.Score)
632 }
633 s += "), (spam words: "
634 for i, w := range result.Spams {
635 if i > 0 {
636 s += ", "
637 }
638 word := w.Word
639 if !d.smtputf8 && !isASCII(word) {
640 word = "(non-ascii)"
641 }
642 s += fmt.Sprintf("%s %.3f", word, w.Score)
643 }
644 s += ")"
645 addReasonText("%s", s)
646 } else if err != store.ErrNoJunkFilter {
647 log.Errorx("open junkfilter", err)
648 addReasonText("open junkfilter: %v", err)
649 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonJunkFilterError)
650 } else {
651 addReasonText("no junk filter configured")
652 }
653
654 // If content looks good, we'll still look at DNS block lists for a reason to
655 // reject. We normally won't get here if we've communicated with this sender
656 // before.
657 var dnsblocklisted bool
658 if accept {
659 blocked := func(zone dns.Domain) bool {
660 dnsblctx, dnsblcancel := context.WithTimeout(ctx, 30*time.Second)
661 defer dnsblcancel()
662 if !checkDNSBLHealth(dnsblctx, log, resolver, zone) {
663 log.Info("dnsbl not healthy, skipping", slog.Any("zone", zone))
664 return false
665 }
666
667 status, expl, err := dnsbl.Lookup(dnsblctx, log.Logger, resolver, zone, net.ParseIP(d.m.RemoteIP))
668 dnsblcancel()
669 if status == dnsbl.StatusFail {
670 log.Info("rejecting due to listing in dnsbl", slog.Any("zone", zone), slog.String("explanation", expl))
671 return true
672 } else if err != nil {
673 log.Infox("dnsbl lookup", err, slog.Any("zone", zone), slog.Any("status", status))
674 }
675 return false
676 }
677
678 // Note: We don't check in parallel, we are in no hurry to accept possible spam.
679 for _, zone := range d.dnsBLs {
680 if blocked(zone) {
681 accept = false
682 dnsblocklisted = true
683 reason = reasonDNSBlocklisted
684 addReasonText("dnsbl: ip %s listed in dnsbl %s", d.m.RemoteIP, zone.XName(d.smtputf8))
685 break
686 }
687 }
688 if !dnsblocklisted && len(d.dnsBLs) > 0 {
689 addReasonText("remote ip not blocklisted")
690 }
691 }
692
693 if accept {
694 addReasonText("no known reputation and no bad signals")
695 return analysis{
696 d: d,
697 accept: true,
698 mailbox: mailbox,
699 reason: reasonNoBadSignals,
700 reasonText: reasonText,
701 dmarcOverrideReason: dmarcOverrideReason,
702 headers: headers,
703 }
704 }
705
706 if subjectpassKey != "" && d.dmarcResult.Status == dmarc.StatusPass && method == methodNone && (dnsblocklisted || junkSubjectpass) {
707 log.Info("permanent reject with subjectpass hint of moderately spammy email without reputation")
708 pass := subjectpass.Generate(log.Logger, d.msgFrom, []byte(subjectpassKey), time.Now())
709 addReasonText("reject with request to try again with subjectpass token in subject")
710 return reject(smtp.C550MailboxUnavail, smtp.SePol7DeliveryUnauth1, subjectpass.Explanation+pass, nil, reasonGiveSubjectpass)
711 }
712
713 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", nil, reason)
714}
715
716func isASCII(s string) bool {
717 for _, b := range []byte(s) {
718 if b >= 0x80 {
719 return false
720 }
721 }
722 return true
723}
724