1package smtpserver

2

3import (

4 "context"

5 "fmt"

6 "net"

7 "os"

8 "strings"

9 "time"

10

11 "golang.org/x/exp/slog"

12

13 "github.com/mjl-/bstore"

14

15 "github.com/mjl-/mox/dkim"

16 "github.com/mjl-/mox/dmarc"

17 "github.com/mjl-/mox/dmarcrpt"

18 "github.com/mjl-/mox/dns"

19 "github.com/mjl-/mox/dnsbl"

20 "github.com/mjl-/mox/iprev"

21 "github.com/mjl-/mox/message"

22 "github.com/mjl-/mox/mlog"

23 "github.com/mjl-/mox/mox-"

24 "github.com/mjl-/mox/publicsuffix"

25 "github.com/mjl-/mox/smtp"

26 "github.com/mjl-/mox/store"

27 "github.com/mjl-/mox/subjectpass"

28 "github.com/mjl-/mox/tlsrpt"

29)

30

31type delivery struct {

32 tls bool

33 m *store.Message

34 dataFile *os.File

35 rcptAcc rcptAccount

36 acc *store.Account

37 msgTo []message.Address

38 msgCc []message.Address

39 msgFrom smtp.Address

40 dnsBLs []dns.Domain

41 dmarcUse bool

42 dmarcResult dmarc.Result

43 dkimResults []dkim.Result

44 iprevStatus iprev.Status

45}

46

47type analysis struct {

48 accept bool

49 mailbox string

50 code int

51 secode string

52 userError bool

53 errmsg string

54 err error // For our own logging, not sent to remote.

55 dmarcReport *dmarcrpt.Feedback // Validated DMARC aggregate report, not yet stored.

56 tlsReport *tlsrpt.Report // Validated TLS report, not yet stored.

57 reason string // If non-empty, reason for this decision. Can be one of reputationMethod and a few other tokens.

58 dmarcOverrideReason string // If set, one of dmarcrpt.PolicyOverride

59 // Additional headers to add during delivery. Used for reasons a message to a

60 // dmarc/tls reporting address isn't processed.

61 headers string

62}

63

64const (

65 reasonListAllow = "list-allow"

66 reasonDMARCPolicy = "dmarc-policy"

67 reasonReputationError = "reputation-error"

68 reasonReporting = "reporting"

69 reasonSPFPolicy = "spf-policy"

70 reasonJunkClassifyError = "junk-classify-error"

71 reasonJunkFilterError = "junk-filter-error"

72 reasonGiveSubjectpass = "give-subjectpass"

73 reasonNoBadSignals = "no-bad-signals"

74 reasonJunkContent = "junk-content"

75 reasonJunkContentStrict = "junk-content-strict"

76 reasonDNSBlocklisted = "dns-blocklisted"

77 reasonSubjectpass = "subjectpass"

78 reasonSubjectpassError = "subjectpass-error"

79 reasonIPrev = "iprev" // No or mild junk reputation signals, and bad iprev.

80)

81

82func isListDomain(d delivery, ld dns.Domain) bool {

83 if d.m.MailFromValidated && ld.Name() == d.m.MailFromDomain {

84 return true

85 }

86 for _, r := range d.dkimResults {

87 if r.Status == dkim.StatusPass && r.Sig.Domain == ld {

88 return true

89 }

90 }

91 return false

92}

93

94func analyze(ctx context.Context, log mlog.Log, resolver dns.Resolver, d delivery) analysis {

95 var headers string

96

97 mailbox := d.rcptAcc.destination.Mailbox

98 if mailbox == "" {

99 mailbox = "Inbox"

100 }

101

102 // If destination mailbox has a mailing list domain (for SPF/DKIM) configured,

103 // check it for a pass.

104 rs := store.MessageRuleset(log, d.rcptAcc.destination, d.m, d.m.MsgPrefix, d.dataFile)

105 if rs != nil {

106 mailbox = rs.Mailbox

107 }

108 if rs != nil && !rs.ListAllowDNSDomain.IsZero() {

109 // todo: on temporary failures, reject temporarily?

110 if isListDomain(d, rs.ListAllowDNSDomain) {

111 d.m.IsMailingList = true

112 return analysis{accept: true, mailbox: mailbox, reason: reasonListAllow, dmarcOverrideReason: string(dmarcrpt.PolicyOverrideMailingList), headers: headers}

113 }

114 }

115

116 var dmarcOverrideReason string

117

118 // For forwarded messages, we have different junk analysis. We don't reject for

119 // failing DMARC, and we clear fields that could implicate the forwarding mail

120 // server during future classifications on incoming messages (the forwarding mail

121 // server isn't responsible for the message).

122 if rs != nil && rs.IsForward {

123 d.dmarcUse = false

124 d.m.IsForward = true

125 d.m.RemoteIPMasked1 = ""

126 d.m.RemoteIPMasked2 = ""

127 d.m.RemoteIPMasked3 = ""

128 d.m.OrigEHLODomain = d.m.EHLODomain

129 d.m.EHLODomain = ""

130 d.m.MailFromDomain = "" // Still available in MailFrom.

131 d.m.OrigDKIMDomains = d.m.DKIMDomains

132 dkimdoms := []string{}

133 for _, dom := range d.m.DKIMDomains {

134 if dom != rs.VerifiedDNSDomain.Name() {

135 dkimdoms = append(dkimdoms, dom)

136 }

137 }

138 d.m.DKIMDomains = dkimdoms

139 dmarcOverrideReason = string(dmarcrpt.PolicyOverrideForwarded)

140 log.Info("forwarded message, clearing identifying signals of forwarding mail server")

141 }

142

143 assignMailbox := func(tx *bstore.Tx) error {

144 // Set message MailboxID to which mail will be delivered. Reputation is

145 // per-mailbox. If referenced mailbox is not found (e.g. does not yet exist), we

146 // can still determine a reputation because we also base it on outgoing

147 // messages and those are account-global.

148 mb, err := d.acc.MailboxFind(tx, mailbox)

149 if err != nil {

150 return fmt.Errorf("finding destination mailbox: %w", err)

151 }

152 if mb != nil {

153 // We want to deliver to mb.ID, but this message may be rejected and sent to the

154 // Rejects mailbox instead, with MailboxID overwritten. Record the ID in

155 // MailboxDestinedID too. If the message is later moved out of the Rejects mailbox,

156 // we'll adjust the MailboxOrigID so it gets taken into account during reputation

157 // calculating in future deliveries. If we end up delivering to the intended

158 // mailbox (i.e. not rejecting), MailboxDestinedID is cleared during delivery so we

159 // don't store it unnecessarily.

160 d.m.MailboxID = mb.ID

161 d.m.MailboxDestinedID = mb.ID

162 } else {

163 log.Debug("mailbox not found in database", slog.String("mailbox", mailbox))

164 }

165 return nil

166 }

167

168 reject := func(code int, secode string, errmsg string, err error, reason string) analysis {

169 // We may have set MailboxDestinedID below already while we had a transaction. If

170 // not, do it now. This makes it possible to use the per-mailbox reputation when a

171 // user moves the message out of the Rejects mailbox to the intended mailbox

172 // (typically Inbox).

173 if d.m.MailboxDestinedID == 0 {

174 var mberr error

175 d.acc.WithRLock(func() {

176 mberr = d.acc.DB.Read(ctx, func(tx *bstore.Tx) error {

177 return assignMailbox(tx)

178 })

179 })

180 if mberr != nil {

181 return analysis{false, mailbox, smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing", err, nil, nil, reasonReputationError, dmarcOverrideReason, headers}

182 }

183 d.m.MailboxID = 0 // We plan to reject, no need to set intended MailboxID.

184 }

185

186 accept := false

187 if rs != nil && rs.AcceptRejectsToMailbox != "" {

188 accept = true

189 mailbox = rs.AcceptRejectsToMailbox

190 d.m.IsReject = true

191 // Don't draw attention, but don't go so far as to mark as junk.

192 d.m.Seen = true

193 log.Info("accepting reject to configured mailbox due to ruleset")

194 }

195 return analysis{accept, mailbox, code, secode, err == nil, errmsg, err, nil, nil, reason, dmarcOverrideReason, headers}

196 }

197

198 if d.dmarcUse && d.dmarcResult.Reject {

199 return reject(smtp.C550MailboxUnavail, smtp.SePol7MultiAuthFails26, "rejecting per dmarc policy", nil, reasonDMARCPolicy)

200 }

201 // todo: should we also reject messages that have a dmarc pass but an spf record "v=spf1 -all"? suggested by m3aawg best practices.

202

203 // If destination is the DMARC reporting mailbox, do additional checks and keep

204 // track of the report. We'll check reputation, defaulting to accept.

205 var dmarcReport *dmarcrpt.Feedback

206 if d.rcptAcc.destination.DMARCReports {

207 // Messages with DMARC aggregate reports must have a DMARC pass. ../rfc/7489:1866

208 if d.dmarcResult.Status != dmarc.StatusPass {

209 log.Info("received dmarc aggregate report without dmarc pass, not processing as dmarc report")

210 headers += "X-Mox-DMARCReport-Error: no DMARC pass\r\n"

211 } else if report, err := dmarcrpt.ParseMessageReport(log.Logger, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil {

212 log.Infox("parsing dmarc aggregate report", err)

213 headers += "X-Mox-DMARCReport-Error: could not parse report\r\n"

214 } else if d, err := dns.ParseDomain(report.PolicyPublished.Domain); err != nil {

215 log.Infox("parsing domain in dmarc aggregate report", err)

216 headers += "X-Mox-DMARCReport-Error: could not parse domain in published policy\r\n"

217 } else if _, ok := mox.Conf.Domain(d); !ok {

218 log.Info("dmarc aggregate report for domain not configured, ignoring", slog.Any("domain", d))

219 headers += "X-Mox-DMARCReport-Error: published policy domain unrecognized\r\n"

220 } else if report.ReportMetadata.DateRange.End > time.Now().Unix()+60 {

221 log.Info("dmarc aggregate report with end date in the future, ignoring", slog.Any("domain", d), slog.Time("end", time.Unix(report.ReportMetadata.DateRange.End, 0)))

222 headers += "X-Mox-DMARCReport-Error: report has end date in the future\r\n"

223 } else {

224 dmarcReport = report

225 }

226 }

227

228 // Similar to DMARC reporting, we check for the required DKIM. We'll check

229 // reputation, defaulting to accept.

230 var tlsReport *tlsrpt.Report

231 if d.rcptAcc.destination.HostTLSReports || d.rcptAcc.destination.DomainTLSReports {

232 matchesDomain := func(sigDomain dns.Domain) bool {

233 // RFC seems to require exact DKIM domain match with submitt and message From, we

234 // also allow msgFrom to be subdomain. ../rfc/8460:322

235 return sigDomain == d.msgFrom.Domain || strings.HasSuffix(d.msgFrom.Domain.ASCII, "."+sigDomain.ASCII) && publicsuffix.Lookup(ctx, log.Logger, d.msgFrom.Domain) == publicsuffix.Lookup(ctx, log.Logger, sigDomain)

236 }

237 // Valid DKIM signature for domain must be present. We take "valid" to assume

238 // "passing", not "syntactically valid". We also check for "tlsrpt" as service.

239 // This check is optional, but if anyone goes through the trouble to explicitly

240 // list allowed services, they would be surprised to see them ignored.

241 // ../rfc/8460:320

242 ok := false

243 for _, r := range d.dkimResults {

244 // The record should have an allowed service "tlsrpt". The RFC mentions it as if

245 // the service must be specified explicitly, but the default allowed services for a

246 // DKIM record are "*", which includes "tlsrpt". Unless a DKIM record explicitly

247 // specifies services (e.g. s=email), a record will work for TLS reports. The DKIM

248 // records seen used for TLS reporting in the wild don't explicitly set "s" for

249 // services.

250 // ../rfc/8460:326

251 if r.Status == dkim.StatusPass && matchesDomain(r.Sig.Domain) && r.Sig.Length < 0 && r.Record.ServiceAllowed("tlsrpt") {

252 ok = true

253 break

254 }

255 }

256

257 if !ok {

258 log.Info("received mail to tlsrpt without acceptable DKIM signature, not processing as tls report")

259 headers += "X-Mox-TLSReport-Error: no acceptable DKIM signature\r\n"

260 } else if reportJSON, err := tlsrpt.ParseMessage(log.Logger, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil {

261 log.Infox("parsing tls report", err)

262 headers += "X-Mox-TLSReport-Error: could not parse TLS report\r\n"

263 } else {

264 var known bool

265 for _, p := range reportJSON.Policies {

266 log.Info("tlsrpt policy domain", slog.String("domain", p.Policy.Domain))

267 if d, err := dns.ParseDomain(p.Policy.Domain); err != nil {

268 log.Infox("parsing domain in tls report", err)

269 } else if _, ok := mox.Conf.Domain(d); ok || d == mox.Conf.Static.HostnameDomain {

270 known = true

271 break

272 }

273 }

274 if !known {

275 log.Info("tls report without one of configured domains, ignoring")

276 headers += "X-Mox-TLSReport-Error: report for unknown domain\r\n"

277 } else {

278 report := reportJSON.Convert()

279 tlsReport = &report

280 }

281 }

282 }

283

284 // Determine if message is acceptable based on DMARC domain, DKIM identities, or

285 // host-based reputation.

286 var isjunk *bool

287 var conclusive bool

288 var method reputationMethod

289 var reason string

290 var err error

291 d.acc.WithRLock(func() {

292 err = d.acc.DB.Read(ctx, func(tx *bstore.Tx) error {

293 if err := assignMailbox(tx); err != nil {

294 return err

295 }

296

297 isjunk, conclusive, method, err = reputation(tx, log, d.m)

298 reason = string(method)

299 return err

300 })

301 })

302 if err != nil {

303 log.Infox("determining reputation", err, slog.Any("message", d.m))

304 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonReputationError)

305 }

306 log.Info("reputation analyzed",

307 slog.Bool("conclusive", conclusive),

308 slog.Any("isjunk", isjunk),

309 slog.String("method", string(method)))

310 if conclusive {

311 if !*isjunk {

312 return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reason, dmarcOverrideReason: dmarcOverrideReason, headers: headers}

313 }

314 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, string(method))

315 } else if dmarcReport != nil || tlsReport != nil {

316 log.Info("accepting message with dmarc aggregate report or tls report without reputation")

317 return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reasonReporting, dmarcOverrideReason: dmarcOverrideReason, headers: headers}

318 }

319 // If there was no previous message from sender or its domain, and we have an SPF

320 // (soft)fail, reject the message.

321 switch method {

322 case methodDKIMSPF, methodIP1, methodIP2, methodIP3, methodNone:

323 switch d.m.MailFromValidation {

324 case store.ValidationFail, store.ValidationSoftfail:

325 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", nil, reasonSPFPolicy)

326 }

327 }

328

329 // Senders without reputation and without iprev pass, are likely spam.

330 var suspiciousIPrevFail bool

331 switch method {

332 case methodDKIMSPF, methodIP1, methodIP2, methodIP3, methodNone:

333 suspiciousIPrevFail = d.iprevStatus != iprev.StatusPass

334 }

335

336 // With already a mild junk signal, an iprev fail on top is enough to reject.

337 if suspiciousIPrevFail && isjunk != nil && *isjunk {

338 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", nil, reasonIPrev)

339 }

340

341 var subjectpassKey string

342 conf, _ := d.acc.Conf()

343 if conf.SubjectPass.Period > 0 {

344 subjectpassKey, err = d.acc.Subjectpass(d.rcptAcc.canonicalAddress)

345 if err != nil {

346 log.Errorx("get key for verifying subject token", err)

347 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonSubjectpassError)

348 }

349 err = subjectpass.Verify(log.Logger, d.dataFile, []byte(subjectpassKey), conf.SubjectPass.Period)

350 pass := err == nil

351 log.Infox("pass by subject token", err, slog.Bool("pass", pass))

352 if pass {

353 return analysis{accept: true, mailbox: mailbox, reason: reasonSubjectpass, dmarcOverrideReason: dmarcOverrideReason, headers: headers}

354 }

355 }

356

357 reason = reasonNoBadSignals

358 accept := true

359 var junkSubjectpass bool

360 f, jf, err := d.acc.OpenJunkFilter(ctx, log)

361 if err == nil {

362 defer func() {

363 err := f.Close()

364 log.Check(err, "closing junkfilter")

365 }()

366 contentProb, _, _, _, err := f.ClassifyMessageReader(ctx, store.FileMsgReader(d.m.MsgPrefix, d.dataFile), d.m.Size)

367 if err != nil {

368 log.Errorx("testing for spam", err)

369 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonJunkClassifyError)

370 }

371 // todo: if isjunk is not nil (i.e. there was inconclusive reputation), use it in the probability calculation. give reputation a score of 0.25 or .75 perhaps?

372 // todo: if there aren't enough historic messages, we should just let messages in.

373 // todo: we could require nham and nspam to be above a certain number when there were plenty of words in the message, and in the database. can indicate a spammer is misspelling words. however, it can also mean a message in a different language/script...

374

375 // If we don't accept, we may still respond with a "subjectpass" hint below.

376 // We add some jitter to the threshold we use. So we don't act as too easy an

377 // oracle for words that are a strong indicator of haminess.

378 // todo: we should rate-limit uses of the junkfilter.

379 jitter := (jitterRand.Float64() - 0.5) / 10

380 threshold := jf.Threshold + jitter

381

382 rcptToMatch := func(l []message.Address) bool {

383 // todo: we use Go's net/mail to parse message header addresses. it does not allow empty quoted strings (contrary to spec), leaving To empty. so we don't verify To address for that unusual case for now. ../rfc/5322:961 ../rfc/5322:743

384 if d.rcptAcc.rcptTo.Localpart == "" {

385 return true

386 }

387 for _, a := range l {

388 dom, err := dns.ParseDomain(a.Host)

389 if err != nil {

390 continue

391 }

392 if dom == d.rcptAcc.rcptTo.IPDomain.Domain && smtp.Localpart(a.User) == d.rcptAcc.rcptTo.Localpart {

393 return true

394 }

395 }

396 return false

397 }

398

399 // todo: some of these checks should also apply for reputation-based analysis with a weak signal, e.g. verified dkim/spf signal from new domain.

400 // With an iprev fail, non-TLS connection or our address not in To/Cc header, we set a higher bar for content.

401 reason = reasonJunkContent

402 if suspiciousIPrevFail && threshold > 0.25 {

403 threshold = 0.25

404 log.Info("setting junk threshold due to iprev fail", slog.Float64("threshold", threshold))

405 reason = reasonJunkContentStrict

406 } else if !d.tls && threshold > 0.25 {

407 threshold = 0.25

408 log.Info("setting junk threshold due to plaintext smtp", slog.Float64("threshold", threshold))

409 reason = reasonJunkContentStrict

410 } else if (rs == nil || !rs.IsForward) && threshold > 0.25 && !rcptToMatch(d.msgTo) && !rcptToMatch(d.msgCc) {

411 // A common theme in junk messages is your recipient address not being in the To/Cc

412 // headers. We may be in Bcc, but that's unusual for first-time senders. Some

413 // providers (e.g. gmail) does not DKIM-sign Bcc headers, so junk messages can be

414 // sent with matching Bcc headers. We don't get here for known senders.

415 threshold = 0.25

416 log.Info("setting junk threshold due to smtp rcpt to and message to/cc address mismatch", slog.Float64("threshold", threshold))

417 reason = reasonJunkContentStrict

418 }

419 accept = contentProb <= threshold

420 junkSubjectpass = contentProb < threshold-0.2

421 log.Info("content analyzed",

422 slog.Bool("accept", accept),

423 slog.Float64("contentprob", contentProb),

424 slog.Bool("subjectpass", junkSubjectpass))

425 } else if err != store.ErrNoJunkFilter {

426 log.Errorx("open junkfilter", err)

427 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonJunkFilterError)

428 }

429

430 // If content looks good, we'll still look at DNS block lists for a reason to

431 // reject. We normally won't get here if we've communicated with this sender

432 // before.

433 var dnsblocklisted bool

434 if accept {

435 blocked := func(zone dns.Domain) bool {

436 dnsblctx, dnsblcancel := context.WithTimeout(ctx, 30*time.Second)

437 defer dnsblcancel()

438 if !checkDNSBLHealth(dnsblctx, log, resolver, zone) {

439 log.Info("dnsbl not healthy, skipping", slog.Any("zone", zone))

440 return false

441 }

442

443 status, expl, err := dnsbl.Lookup(dnsblctx, log.Logger, resolver, zone, net.ParseIP(d.m.RemoteIP))

444 dnsblcancel()

445 if status == dnsbl.StatusFail {

446 log.Info("rejecting due to listing in dnsbl", slog.Any("zone", zone), slog.String("explanation", expl))

447 return true

448 } else if err != nil {

449 log.Infox("dnsbl lookup", err, slog.Any("zone", zone), slog.Any("status", status))

450 }

451 return false

452 }

453

454 // Note: We don't check in parallel, we are in no hurry to accept possible spam.

455 for _, zone := range d.dnsBLs {

456 if blocked(zone) {

457 accept = false

458 dnsblocklisted = true

459 reason = reasonDNSBlocklisted

460 break

461 }

462 }

463 }

464

465 if accept {

466 return analysis{accept: true, mailbox: mailbox, reason: reasonNoBadSignals, dmarcOverrideReason: dmarcOverrideReason, headers: headers}

467 }

468

469 if subjectpassKey != "" && d.dmarcResult.Status == dmarc.StatusPass && method == methodNone && (dnsblocklisted || junkSubjectpass) {

470 log.Info("permanent reject with subjectpass hint of moderately spammy email without reputation")

471 pass := subjectpass.Generate(log.Logger, d.msgFrom, []byte(subjectpassKey), time.Now())

472 return reject(smtp.C550MailboxUnavail, smtp.SePol7DeliveryUnauth1, subjectpass.Explanation+pass, nil, reasonGiveSubjectpass)

473 }

474

475 return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", nil, reason)

476}

477