1// Package mtastsdb stores MTA-STS policies for later use.

2//

3// An MTA-STS policy can specify how long it may be cached. By storing a

4// policy, it does not have to be fetched again during email delivery, which

5// makes it harder for attackers to intervene.

6package mtastsdb

8import (

9 "context"

10 "crypto/tls"

11 "errors"

12 "fmt"

13 "log/slog"

14 "os"

15 "path/filepath"

16 "strings"

17 "time"

19 "github.com/prometheus/client_golang/prometheus"

20 "github.com/prometheus/client_golang/prometheus/promauto"

22 "github.com/mjl-/bstore"

24 "github.com/mjl-/mox/dns"

25 "github.com/mjl-/mox/mlog"

26 "github.com/mjl-/mox/mox-"

27 "github.com/mjl-/mox/moxvar"

28 "github.com/mjl-/mox/mtasts"

29 "github.com/mjl-/mox/tlsrpt"

30)

32var (

33 metricGet = promauto.NewCounterVec(

34 prometheus.CounterOpts{

35 Name: "mox_mtastsdb_get_total",

36 Help: "Number of Get by result.",

37 },

38 []string{"result"},

39 )

40)

42var timeNow = time.Now // Tests override this.

44// PolicyRecord is a cached policy or absence of a policy.

45type PolicyRecord struct {

46 Domain string // Domain name, with unicode characters.

47 Inserted time.Time `bstore:"default now"`

48 ValidEnd time.Time

49 LastUpdate time.Time // Policies are refreshed on use and periodically.

50 LastUse time.Time `bstore:"index"`

51 Backoff bool

52 RecordID string // As retrieved from DNS.

53 mtasts.Policy // As retrieved from the well-known HTTPS url.

55 // Text that make up the policy, as retrieved. We didn't store this in the past. If

56 // empty, policy can be reconstructed from Policy field. Needed by TLSRPT.

57 PolicyText string

58}

60var (

61 // No valid non-expired policy in database.

62 ErrNotFound = errors.New("mtastsdb: policy not found")

64 // Indicates an MTA-STS TXT record was fetched recently, but fetching the policy

65 // failed and should not yet be retried.

66 ErrBackoff = errors.New("mtastsdb: policy fetch failed recently")

67)

69var DBTypes = []any{PolicyRecord{}} // Types stored in DB.

70var DB *bstore.DB // Exported for backups.

72// Init opens the database and starts a goroutine that refreshes policies in

73// the database, and keeps doing so periodically.

74func Init(refresher bool) error {

75 log := mlog.New("mtastsdb", nil)

77 p := mox.DataDirPath("mtasts.db")

78 os.MkdirAll(filepath.Dir(p), 0770)

79 opts := bstore.Options{Timeout: 5 * time.Second, Perm: 0660, RegisterLogger: moxvar.RegisterLogger(p, log.Logger)}

80 var err error

81 DB, err = bstore.Open(mox.Shutdown, p, &opts, DBTypes...)

82 if err != nil {

83 return err

84 }

86 if refresher {

87 // todo: allow us to shut down cleanly?

88 go refresh()

89 }

91 return nil

92}

94// Close closes the database.

95func Close() error {

96 if err := DB.Close(); err != nil {

97 return fmt.Errorf("close db: %w", err)

98 }

99 DB = nil

100 return nil

101}

102

103// lookup looks up a policy for the domain in the database.

104//

105// Only non-expired records are returned.

106//

107// Returns ErrNotFound if record is not present.

108// Returns ErrBackoff if a recent attempt to fetch a record failed.

109func lookup(ctx context.Context, log mlog.Log, domain dns.Domain) (*PolicyRecord, error) {

110 if domain.IsZero() {

111 return nil, fmt.Errorf("empty domain")

112 }

113 now := timeNow()

114 q := bstore.QueryDB[PolicyRecord](ctx, DB)

115 q.FilterNonzero(PolicyRecord{Domain: domain.Name()})

116 q.FilterGreater("ValidEnd", now)

117 pr, err := q.Get()

118 if err == bstore.ErrAbsent {

119 return nil, ErrNotFound

120 } else if err != nil {

121 return nil, err

122 }

123

124 pr.LastUse = now

125 if err := DB.Update(ctx, &pr); err != nil {

126 log.Errorx("marking cached mta-sts policy as used in database", err)

127 }

128 if pr.Backoff {

129 return nil, ErrBackoff

130 }

131 return &pr, nil

132}

133

134// Upsert adds the policy to the database, overwriting an existing policy for the domain.

135// Policy can be nil, indicating a failure to fetch the policy.

136func Upsert(ctx context.Context, domain dns.Domain, recordID string, policy *mtasts.Policy, policyText string) error {

137 return DB.Write(ctx, func(tx *bstore.Tx) error {

138 pr := PolicyRecord{Domain: domain.Name()}

139 err := tx.Get(&pr)

140 if err != nil && err != bstore.ErrAbsent {

141 return err

142 }

143

144 now := timeNow()

145

146 var p mtasts.Policy

147 if policy != nil {

148 p = *policy

149 } else {

150 // ../rfc/8461:552

151 p.Mode = mtasts.ModeNone

152 p.MaxAgeSeconds = 5 * 60

153 }

154 backoff := policy == nil

155 validEnd := now.Add(time.Duration(p.MaxAgeSeconds) * time.Second)

156

157 if err == bstore.ErrAbsent {

158 pr = PolicyRecord{domain.Name(), now, validEnd, now, now, backoff, recordID, p, policyText}

159 return tx.Insert(&pr)

160 }

161

162 pr.ValidEnd = validEnd

163 pr.LastUpdate = now

164 pr.LastUse = now

165 pr.Backoff = backoff

166 pr.RecordID = recordID

167 pr.Policy = p

168 pr.PolicyText = policyText

169 return tx.Update(&pr)

170 })

171}

172

173// PolicyRecords returns all policies in the database, sorted descending by last

174// use, domain.

175func PolicyRecords(ctx context.Context) ([]PolicyRecord, error) {

176 return bstore.QueryDB[PolicyRecord](ctx, DB).SortDesc("LastUse", "Domain").List()

177}

178

179// Get retrieves an MTA-STS policy for domain and whether it is fresh.

180//

181// If an error is returned, it should be considered a transient error, e.g. a

182// temporary DNS lookup failure.

183//

184// The returned policy can be nil also when there is no error. In this case, the

185// domain does not implement MTA-STS.

186//

187// If a policy is present in the local database, it is refreshed if needed. If no

188// policy is present for the domain, an attempt is made to fetch the policy and

189// store it in the local database.

190//

191// Some errors are logged but not otherwise returned, e.g. if a new policy is

192// supposedly published but could not be retrieved.

193//

194// Get returns an "sts" or "no-policy-found" in reportResult in most cases (when

195// not a local/internal error). It may add an "sts" result without policy contents

196// ("policy-string") in case of errors while fetching the policy.

197func Get(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, domain dns.Domain) (policy *mtasts.Policy, reportResult tlsrpt.Result, fresh bool, err error) {

198 log := mlog.New("mtastsdb", elog)

199 defer func() {

200 result := "ok"

201 if err != nil && errors.Is(err, ErrBackoff) {

202 result = "backoff"

203 } else if err != nil && errors.Is(err, ErrNotFound) {

204 result = "notfound"

205 } else if err != nil {

206 result = "error"

207 }

208 metricGet.WithLabelValues(result).Inc()

209 log.Debugx("mtastsdb get result", err, slog.Any("domain", domain), slog.Bool("fresh", fresh))

210 }()

211

212 cachedPolicy, err := lookup(ctx, log, domain)

213 if err != nil && errors.Is(err, ErrNotFound) {

214 // We don't have a policy for this domain, not even a record that we tried recently

215 // and should backoff. So attempt to fetch policy.

216 nctx, cancel := context.WithTimeout(ctx, time.Minute)

217 defer cancel()

218 record, p, ptext, err := mtasts.Get(nctx, log.Logger, resolver, domain)

219 if err != nil {

220 switch {

221 case errors.Is(err, mtasts.ErrNoRecord) || errors.Is(err, mtasts.ErrMultipleRecords) || errors.Is(err, mtasts.ErrRecordSyntax) || errors.Is(err, mtasts.ErrNoPolicy) || errors.Is(err, mtasts.ErrPolicyFetch) || errors.Is(err, mtasts.ErrPolicySyntax):

222 // Remote is not doing MTA-STS, continue below. ../rfc/8461:333 ../rfc/8461:574

223 log.Debugx("interpreting mtasts error to mean remote is not doing mta-sts", err)

224

225 if errors.Is(err, mtasts.ErrNoRecord) {

226 reportResult = tlsrpt.MakeResult(tlsrpt.NoPolicyFound, domain)

227 } else {

228 fd := policyFetchFailureDetails(err)

229 reportResult = tlsrpt.MakeResult(tlsrpt.STS, domain, fd)

230 }

231

232 default:

233 // Interpret as temporary error, e.g. mtasts.ErrDNS, try again later.

234

235 // Temporary DNS error could be an operational issue on our side, but we can still

236 // report it.

237 // Result: ../rfc/8460:594

238 fd := tlsrpt.Details(tlsrpt.ResultSTSPolicyFetch, mtasts.TLSReportFailureReason(err))

239 reportResult = tlsrpt.MakeResult(tlsrpt.STS, domain, fd)

240

241 return nil, reportResult, false, fmt.Errorf("lookup up mta-sts policy: %w", err)

242 }

243 } else if p.Mode == mtasts.ModeNone {

244 reportResult = tlsrpt.MakeResult(tlsrpt.NoPolicyFound, domain)

245 } else {

246 reportResult = tlsrpt.Result{Policy: tlsrptPolicy(p, ptext, domain)}

247 }

248

249 // Insert policy into database. If we could not fetch the policy itself, we back

250 // off for 5 minutes. ../rfc/8461:555

251 if err == nil || errors.Is(err, mtasts.ErrNoPolicy) || errors.Is(err, mtasts.ErrPolicyFetch) || errors.Is(err, mtasts.ErrPolicySyntax) {

252 var recordID string

253 if record != nil {

254 recordID = record.ID

255 }

256 if err := Upsert(ctx, domain, recordID, p, ptext); err != nil {

257 log.Errorx("inserting policy into cache, continuing", err)

258 }

259 }

260

261 return p, reportResult, true, nil

262 } else if err != nil && errors.Is(err, ErrBackoff) {

263 // ../rfc/8461:552

264 // We recently failed to fetch a policy, act as if MTA-STS is not implemented.

265 // Result: ../rfc/8460:594

266 fd := tlsrpt.Details(tlsrpt.ResultSTSPolicyFetch, "back-off-after-recent-fetch-error")

267 reportResult = tlsrpt.MakeResult(tlsrpt.STS, domain, fd)

268 return nil, reportResult, false, nil

269 } else if err != nil {

270 // We don't add the result to the report, this is an internal error.

271 return nil, reportResult, false, fmt.Errorf("looking up mta-sts policy in cache: %w", err)

272 }

273

274 // Policy was found in database. Check in DNS it is still fresh.

275 policy = &cachedPolicy.Policy

276 nctx, cancel := context.WithTimeout(ctx, 30*time.Second)

277 defer cancel()

278 record, _, err := mtasts.LookupRecord(nctx, log.Logger, resolver, domain)

279 if err != nil {

280 if errors.Is(err, mtasts.ErrNoRecord) {

281 if policy.Mode != mtasts.ModeNone {

282 log.Errorx("no mtasts dns record while checking non-none policy for freshness, either domain owner removed mta-sts without phasing out policy with a none-policy for period of previous max-age, or this could be an attempt to downgrade to connection without mtasts, continuing with previous policy", err)

283 }

284 // else, policy will be removed by periodic refresher in the near future.

285 } else {

286 // Could be a temporary DNS or configuration error.

287 log.Errorx("checking for freshness of cached mta-sts dns txt record for domain, continuing with previously cached policy", err)

288 }

289

290 // Result: ../rfc/8460:594

291 fd := tlsrpt.Details(tlsrpt.ResultSTSPolicyFetch, mtasts.TLSReportFailureReason(err))

292 if policy.Mode != mtasts.ModeNone {

293 fd.FailureReasonCode += "+fallback-to-cached-policy"

294 }