1package smtpserver

2

3import (

4 "context"

5 "fmt"

6 "log/slog"

7 "net"

8 "os"

9 "strings"

10 "time"

11

12 "github.com/mjl-/bstore"

13

14 "github.com/mjl-/mox/dkim"

15 "github.com/mjl-/mox/dmarc"

16 "github.com/mjl-/mox/dmarcrpt"

17 "github.com/mjl-/mox/dns"

18 "github.com/mjl-/mox/dnsbl"

19 "github.com/mjl-/mox/iprev"

20 "github.com/mjl-/mox/message"

21 "github.com/mjl-/mox/mlog"

22 "github.com/mjl-/mox/mox-"

23 "github.com/mjl-/mox/publicsuffix"

24 "github.com/mjl-/mox/smtp"

25 "github.com/mjl-/mox/store"

26 "github.com/mjl-/mox/subjectpass"

27 "github.com/mjl-/mox/tlsrpt"

28)

29

30type delivery struct {

31 tls bool

32 m *store.Message

33 dataFile *os.File

34 rcptAcc rcptAccount

35 acc *store.Account

36 msgTo []message.Address

37 msgCc []message.Address

38 msgFrom smtp.Address

39 dnsBLs []dns.Domain

40 dmarcUse bool

41 dmarcResult dmarc.Result

42 dkimResults []dkim.Result

43 iprevStatus iprev.Status

44}

45

46type analysis struct {

47 accept bool

48 mailbox string

49 code int

50 secode string

51 userError bool

52 errmsg string

53 err error // For our own logging, not sent to remote.

54 dmarcReport *dmarcrpt.Feedback // Validated DMARC aggregate report, not yet stored.

55 tlsReport *tlsrpt.Report // Validated TLS report, not yet stored.

56 reason string // If non-empty, reason for this decision. Can be one of reputationMethod and a few other tokens.

57 dmarcOverrideReason string // If set, one of dmarcrpt.PolicyOverride

58 // Additional headers to add during delivery. Used for reasons a message to a

59 // dmarc/tls reporting address isn't processed.

60 headers string

61}

62

63const (

64 reasonListAllow = "list-allow"

65 reasonDMARCPolicy = "dmarc-policy"

66 reasonReputationError = "reputation-error"

67 reasonReporting = "reporting"

68 reasonSPFPolicy = "spf-policy"

69 reasonJunkClassifyError = "junk-classify-error"

70 reasonJunkFilterError = "junk-filter-error"

71 reasonGiveSubjectpass = "give-subjectpass"

72 reasonNoBadSignals = "no-bad-signals"

73 reasonJunkContent = "junk-content"

74 reasonJunkContentStrict = "junk-content-strict"

75 reasonDNSBlocklisted = "dns-blocklisted"

76 reasonSubjectpass = "subjectpass"

77 reasonSubjectpassError = "subjectpass-error"

78 reasonIPrev = "iprev" // No or mild junk reputation signals, and bad iprev.

79)

80

81func isListDomain(d delivery, ld dns.Domain) bool {

82 if d.m.MailFromValidated && ld.Name() == d.m.MailFromDomain {

83 return true

84 }

85 for _, r := range d.dkimResults {

86 if r.Status == dkim.StatusPass && r.Sig.Domain == ld {

87 return true

88 }

89 }

90 return false

91}

92

93func analyze(ctx context.Context, log mlog.Log, resolver dns.Resolver, d delivery) analysis {

94 var headers string

95

96 mailbox := d.rcptAcc.destination.Mailbox

97 if mailbox == "" {

98 mailbox = "Inbox"

99 }

100

101 // If destination mailbox has a mailing list domain (for SPF/DKIM) configured,

102 // check it for a pass.

103 rs := store.MessageRuleset(log, d.rcptAcc.destination, d.m, d.m.MsgPrefix, d.dataFile)

104 if rs != nil {

105 mailbox = rs.Mailbox

106 }

107 if rs != nil && !rs.ListAllowDNSDomain.IsZero() {

108 // todo: on temporary failures, reject temporarily?

109 if isListDomain(d, rs.ListAllowDNSDomain) {

110 d.m.IsMailingList = true

111 return analysis{accept: true, mailbox: mailbox, reason: reasonListAllow, dmarcOverrideReason: string(dmarcrpt.PolicyOverrideMailingList), headers: headers}

112 }

113 }

114

115 var dmarcOverrideReason string

116

117 // For forwarded messages, we have different junk analysis. We don't reject for

118 // failing DMARC, and we clear fields that could implicate the forwarding mail

119 // server during future classifications on incoming messages (the forwarding mail

120 // server isn't responsible for the message).

121 if rs != nil && rs.IsForward {

122 d.dmarcUse = false

123 d.m.IsForward = true

124 d.m.RemoteIPMasked1 = ""

125 d.m.RemoteIPMasked2 = ""

126 d.m.RemoteIPMasked3 = ""

127 d.m.OrigEHLODomain = d.m.EHLODomain

128 d.m.EHLODomain = ""

129 d.m.MailFromDomain = "" // Still available in MailFrom.

130 d.m.OrigDKIMDomains = d.m.DKIMDomains

131 dkimdoms := []string{}

132 for _, dom := range d.m.DKIMDomains {

133 if dom != rs.VerifiedDNSDomain.Name() {

134 dkimdoms = append(dkimdoms, dom)

135 }

136 }

137 d.m.DKIMDomains = dkimdoms

138 dmarcOverrideReason = string(dmarcrpt.PolicyOverrideForwarded)

139 log.Info("forwarded message, clearing identifying signals of forwarding mail server")

140 }

141

142 assignMailbox := func(tx *bstore.Tx) error {

143 // Set message MailboxID to which mail will be delivered. Reputation is

144 // per-mailbox. If referenced mailbox is not found (e.g. does not yet exist), we

145 // can still determine a reputation because we also base it on outgoing

146 // messages and those are account-global.

147 mb, err := d.acc.MailboxFind(tx, mailbox)

148 if err != nil {

149 return fmt.Errorf("finding destination mailbox: %w", err)

150 }

151 if mb != nil {

152 // We want to deliver to mb.ID, but this message may be rejected and sent to the

153 // Rejects mailbox instead, with MailboxID overwritten. Record the ID in

154 // MailboxDestinedID too. If the message is later moved out of the Rejects mailbox,

155 // we'll adjust the MailboxOrigID so it gets taken into account during reputation

156 // calculating in future deliveries. If we end up delivering to the intended

157 // mailbox (i.e. not rejecting), MailboxDestinedID is cleared during delivery so we

158 // don't store it unnecessarily.

159 d.m.MailboxID = mb.ID

160 d.m.MailboxDestinedID = mb.ID

161 } else {

162 log.Debug("mailbox not found in database", slog.String("mailbox", mailbox))

163 }

164 return nil

165 }

166

167 reject := func(code int, secode string, errmsg string, err error, reason string) analysis {

168 // We may have set MailboxDestinedID below already while we had a transaction. If

169 // not, do it now. This makes it possible to use the per-mailbox reputation when a

170 // user moves the message out of the Rejects mailbox to the intended mailbox

171 // (typically Inbox).

172 if d.m.MailboxDestinedID == 0 {

173 var mberr error

174 d.acc.WithRLock(func() {

175 mberr = d.acc.DB.Read(ctx, func(tx *bstore.Tx) error {

176 return assignMailbox(tx)

177 })

178 })

179 if mberr != nil {

180 return analysis{false, mailbox, smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing", err, nil, nil, reasonReputationError, dmarcOverrideReason, headers}

181 }

182 d.m.MailboxID = 0 // We plan to reject, no need to set intended MailboxID.

183 }

184

185 accept := false

186 if rs != nil && rs.AcceptRejectsToMailbox != "" {

187 accept = true

188 mailbox = rs.AcceptRejectsToMailbox

189 d.m.IsReject = true

190 // Don't draw attention, but don't go so far as to mark as junk.

191 d.m.Seen = true

192 log.Info("accepting reject to configured mailbox due to ruleset")

193 }

194 return analysis{accept, mailbox, code, secode, err == nil, errmsg, err, nil, nil, reason, dmarcOverrideReason, headers}

195 }

196

197 if d.dmarcUse && d.dmarcResult.Reject {

198 return reject(smtp.C550MailboxUnavail, smtp.SePol7MultiAuthFails26, "rejecting per dmarc policy", nil, reasonDMARCPolicy)

199 }

200 // todo: should we also reject messages that have a dmarc pass but an spf record "v=spf1 -all"? suggested by m3aawg best practices.

201

202 // If destination is the DMARC reporting mailbox, do additional checks and keep

203 // track of the report. We'll check reputation, defaulting to accept.

204 var dmarcReport *dmarcrpt.Feedback

205 if d.rcptAcc.destination.DMARCReports {

206 // Messages with DMARC aggregate reports must have a DMARC pass. ../rfc/7489:1866

207 if d.dmarcResult.Status != dmarc.StatusPass {

208 log.Info("received dmarc aggregate report without dmarc pass, not processing as dmarc report")

209 headers += "X-Mox-DMARCReport-Error: no DMARC pass\r\n"

210 } else if report, err := dmarcrpt.ParseMessageReport(log.Logger, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil {

211 log.Infox("parsing dmarc aggregate report", err)

212 headers += "X-Mox-DMARCReport-Error: could not parse report\r\n"

213 } else if d, err := dns.ParseDomain(report.PolicyPublished.Domain); err != nil {

214 log.Infox("parsing domain in dmarc aggregate report", err)

215 headers += "X-Mox-DMARCReport-Error: could not parse domain in published policy\r\n"

216 } else if _, ok := mox.Conf.Domain(d); !ok {

217 log.Info("dmarc aggregate report for domain not configured, ignoring", slog.Any("domain", d))

218 headers += "X-Mox-DMARCReport-Error: published policy domain unrecognized\r\n"

219 } else if report.ReportMetadata.DateRange.End > time.Now().Unix()+60 {

220 log.Info("dmarc aggregate report with end date in the future, ignoring", slog.Any("domain", d), slog.Time("end", time.Unix(report.ReportMetadata.DateRange.End, 0)))

221 headers += "X-Mox-DMARCReport-Error: report has end date in the future\r\n"

222 } else {

223 dmarcReport = report

224 }

225 }

226

227 // Similar to DMARC reporting, we check for the required DKIM. We'll check

228 // reputation, defaulting to accept.

229 var tlsReport *tlsrpt.Report

230 if d.rcptAcc.destination.HostTLSReports || d.rcptAcc.destination.DomainTLSReports {

231 matchesDomain := func(sigDomain dns.Domain) bool {

232 // RFC seems to require exact DKIM domain match with submitt and message From, we

233 // also allow msgFrom to be subdomain. ../rfc/8460:322

234 return sigDomain == d.msgFrom.Domain || strings.HasSuffix(d.msgFrom.Domain.ASCII, "."+sigDomain.ASCII) && publicsuffix.Lookup(ctx, log.Logger, d.msgFrom.Domain) == publicsuffix.Lookup(ctx, log.Logger, sigDomain)

235 }

236 // Valid DKIM signature for domain must be present. We take "valid" to assume

237 // "passing", not "syntactically valid". We also check for "tlsrpt" as service.

238 // This check is optional, but if anyone goes through the trouble to explicitly

239 // list allowed services, they would be surprised to see them ignored.

240 // ../rfc/8460:320

241 ok := false

242 for _, r := range d.dkimResults {

243 // The record should have an allowed service "tlsrpt". The RFC mentions it as if

244 // the service must be specified explicitly, but the default allowed services for a

245 // DKIM record are "*", which includes "tlsrpt". Unless a DKIM record explicitly

246 // specifies services (e.g. s=email), a record will work for TLS reports. The DKIM

247 // records seen used for TLS reporting in the wild don't explicitly set "s" for

248 // services.

249 // ../rfc/8460:326

250 if r.Status == dkim.StatusPass && matchesDomain(r.Sig.Domain) && r.Sig.Length < 0 && r.Record.ServiceAllowed("tlsrpt") {

251 ok = true

252 break

253 }

254 }

255

256 if !ok {

257 log.Info("received mail to tlsrpt without acceptable DKIM signature, not processing as tls report")

258 headers += "X-Mox-TLSReport-Error: no acceptable DKIM signature\r\n"

259 } else if reportJSON, err := tlsrpt.ParseMessage(log.Logger, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil {

260 log.Infox("parsing tls report", err)

261 headers += "X-Mox-TLSReport-Error: could not parse TLS report\r\n"

262 } else {

263 var known bool

264 for _, p := range reportJSON.Policies {

265 log.Info("tlsrpt policy domain", slog.String("domain", p.Policy.Domain))

266 if d, err := dns.ParseDomain(p.Policy.Domain); err != nil {

267 log.Infox("parsing domain in tls report", err)

268 } else if _, ok := mox.Conf.Domain(d); ok || d == mox.Conf.Static.HostnameDomain {

269 known = true

270 break

271 }

272 }

273 if !known {

274 log.Info("tls report without one of configured domains, ignoring")

275 headers += "X-Mox-TLSReport-Error: report for unknown domain\r\n"

276 } else {

277 report := reportJSON.Convert()

278 tlsReport = &report

279 }

280 }

281 }

282

283 // Determine if message is acceptable based on DMARC domain, DKIM identities, or

284 // host-based reputation.

285 var isjunk *bool

286 var conclusive bool

287 var method reputationMethod

288 var reason string

289 var err error

290 d.acc.WithRLock(func() {

291 err = d.acc.DB.Read(ctx, func(tx *bstore.Tx) error {

292 if err := assignMailbox(tx); err != nil {

293 return err

294 }

295

296 isjunk, conclusive, method, err = reputation(tx, log, d.m)

297 reason = string(method)

298 return err

299 })

300 })

301 if err != nil {

302 log.Infox("determining reputation", err, slog.Any("message", d.m))

303 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonReputationError)

304 }

305 log.Info("reputation analyzed",

306 slog.Bool("conclusive", conclusive),

307 slog.Any("isjunk", isjunk),

308 slog.String("method", string(method)))

309 if conclusive {

310 if !*isjunk {

311 return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reason, dmarcOverrideReason: dmarcOverrideReason, headers: headers}

312 }

313 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, string(method))

314 } else if dmarcReport != nil || tlsReport != nil {

315 log.Info("accepting message with dmarc aggregate report or tls report without reputation")

316 return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reasonReporting, dmarcOverrideReason: dmarcOverrideReason, headers: headers}

317 }

318 // If there was no previous message from sender or its domain, and we have an SPF

319 // (soft)fail, reject the message.

320 switch method {

321 case methodDKIMSPF, methodIP1, methodIP2, methodIP3, methodNone:

322 switch d.m.MailFromValidation {

323 case store.ValidationFail, store.ValidationSoftfail:

324 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", nil, reasonSPFPolicy)

325 }

326 }

327

328 // Senders without reputation and without iprev pass, are likely spam.

329 var suspiciousIPrevFail bool

330 switch method {

331 case methodDKIMSPF, methodIP1, methodIP2, methodIP3, methodNone:

332 suspiciousIPrevFail = d.iprevStatus != iprev.StatusPass

333 }

334

335 // With already a mild junk signal, an iprev fail on top is enough to reject.

336 if suspiciousIPrevFail && isjunk != nil && *isjunk {

337 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", nil, reasonIPrev)

338 }

339

340 var subjectpassKey string

341 conf, _ := d.acc.Conf()

342 if conf.SubjectPass.Period > 0 {

343 subjectpassKey, err = d.acc.Subjectpass(d.rcptAcc.canonicalAddress)

344 if err != nil {

345 log.Errorx("get key for verifying subject token", err)

346 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonSubjectpassError)

347 }

348 err = subjectpass.Verify(log.Logger, d.dataFile, []byte(subjectpassKey), conf.SubjectPass.Period)

349 pass := err == nil

350 log.Infox("pass by subject token", err, slog.Bool("pass", pass))

351 if pass {

352 return analysis{accept: true, mailbox: mailbox, reason: reasonSubjectpass, dmarcOverrideReason: dmarcOverrideReason, headers: headers}

353 }

354 }

355

356 reason = reasonNoBadSignals

357 accept := true

358 var junkSubjectpass bool

359 f, jf, err := d.acc.OpenJunkFilter(ctx, log)

360 if err == nil {

361 defer func() {

362 err := f.Close()

363 log.Check(err, "closing junkfilter")

364 }()

365 contentProb, _, _, _, err := f.ClassifyMessageReader(ctx, store.FileMsgReader(d.m.MsgPrefix, d.dataFile), d.m.Size)

366 if err != nil {

367 log.Errorx("testing for spam", err)

368 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonJunkClassifyError)

369 }

370 // todo: if isjunk is not nil (i.e. there was inconclusive reputation), use it in the probability calculation. give reputation a score of 0.25 or .75 perhaps?

371 // todo: if there aren't enough historic messages, we should just let messages in.

372 // todo: we could require nham and nspam to be above a certain number when there were plenty of words in the message, and in the database. can indicate a spammer is misspelling words. however, it can also mean a message in a different language/script...

373

374 // If we don't accept, we may still respond with a "subjectpass" hint below.

375 // We add some jitter to the threshold we use. So we don't act as too easy an

376 // oracle for words that are a strong indicator of haminess.

377 // todo: we should rate-limit uses of the junkfilter.

378 jitter := (jitterRand.Float64() - 0.5) / 10

379 threshold := jf.Threshold + jitter

380

381 rcptToMatch := func(l []message.Address) bool {

382 // todo: we use Go's net/mail to parse message header addresses. it does not allow empty quoted strings (contrary to spec), leaving To empty. so we don't verify To address for that unusual case for now. ../rfc/5322:961 ../rfc/5322:743

383 if d.rcptAcc.rcptTo.Localpart == "" {

384 return true

385 }

386 for _, a := range l {

387 dom, err := dns.ParseDomain(a.Host)

388 if err != nil {

389 continue

390 }

391 lp, err := smtp.ParseLocalpart(a.User)

392 if err == nil && dom == d.rcptAcc.rcptTo.IPDomain.Domain && lp == d.rcptAcc.rcptTo.Localpart {

393 return true

394 }

395 }

396 return false

397 }

398

399 // todo: some of these checks should also apply for reputation-based analysis with a weak signal, e.g. verified dkim/spf signal from new domain.

400 // With an iprev fail, non-TLS connection or our address not in To/Cc header, we set a higher bar for content.

401 reason = reasonJunkContent

402 if suspiciousIPrevFail && threshold > 0.25 {

403 threshold = 0.25

404 log.Info("setting junk threshold due to iprev fail", slog.Float64("threshold", threshold))

405 reason = reasonJunkContentStrict

406 } else if !d.tls && threshold > 0.25 {

407 threshold = 0.25

408 log.Info("setting junk threshold due to plaintext smtp", slog.Float64("threshold", threshold))

409 reason = reasonJunkContentStrict

410 } else if (rs == nil || !rs.IsForward) && threshold > 0.25 && !rcptToMatch(d.msgTo) && !rcptToMatch(d.msgCc) {

411 // A common theme in junk messages is your recipient address not being in the To/Cc

412 // headers. We may be in Bcc, but that's unusual for first-time senders. Some

413 // providers (e.g. gmail) does not DKIM-sign Bcc headers, so junk messages can be

414 // sent with matching Bcc headers. We don't get here for known senders.

415 threshold = 0.25

416 log.Info("setting junk threshold due to smtp rcpt to and message to/cc address mismatch", slog.Float64("threshold", threshold))

417 reason = reasonJunkContentStrict

418 }

419 accept = contentProb <= threshold

420 junkSubjectpass = contentProb < threshold-0.2

421 log.Info("content analyzed",

422 slog.Bool("accept", accept),

423 slog.Float64("contentprob", contentProb),

424 slog.Bool("subjectpass", junkSubjectpass))

425 } else if err != store.ErrNoJunkFilter {

426 log.Errorx("open junkfilter", err)

427 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonJunkFilterError)

428 }

429

430 // If content looks good, we'll still look at DNS block lists for a reason to

431 // reject. We normally won't get here if we've communicated with this sender

432 // before.

433 var dnsblocklisted bool

434 if accept {

435 blocked := func(zone dns.Domain) bool {

436 dnsblctx, dnsblcancel := context.WithTimeout(ctx, 30*time.Second)

437 defer dnsblcancel()

438 if !checkDNSBLHealth(dnsblctx, log, resolver, zone) {

439 log.Info("dnsbl not healthy, skipping", slog.Any("zone", zone))

440 return false

441 }

442

443 status, expl, err := dnsbl.Lookup(dnsblctx, log.Logger, resolver, zone, net.ParseIP(d.m.RemoteIP))

444 dnsblcancel()

445 if status == dnsbl.StatusFail {

446 log.Info("rejecting due to listing in dnsbl", slog.Any("zone", zone), slog.String("explanation", expl))

447 return true

448 } else if err != nil {

449 log.Infox("dnsbl lookup", err, slog.Any("zone", zone), slog.Any("status", status))

450 }

451 return false

452 }

453

454 // Note: We don't check in parallel, we are in no hurry to accept possible spam.

455 for _, zone := range d.dnsBLs {

456 if blocked(zone) {

457 accept = false

458 dnsblocklisted = true

459 reason = reasonDNSBlocklisted

460 break

461 }

462 }

463 }

464

465 if accept {

466 return analysis{accept: true, mailbox: mailbox, reason: reasonNoBadSignals, dmarcOverrideReason: dmarcOverrideReason, headers: headers}

467 }

468

469 if subjectpassKey != "" && d.dmarcResult.Status == dmarc.StatusPass && method == methodNone && (dnsblocklisted || junkSubjectpass) {

470 log.Info("permanent reject with subjectpass hint of moderately spammy email without reputation")

471 pass := subjectpass.Generate(log.Logger, d.msgFrom, []byte(subjectpassKey), time.Now())

472 return reject(smtp.C550MailboxUnavail, smtp.SePol7DeliveryUnauth1, subjectpass.Explanation+pass, nil, reasonGiveSubjectpass)

473 }

474

475 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", nil, reason)

476}

477