1// Package mtastsdb stores MTA-STS policies for later use.

2//

3// An MTA-STS policy can specify how long it may be cached. By storing a

4// policy, it does not have to be fetched again during email delivery, which

5// makes it harder for attackers to intervene.

6package mtastsdb

8import (

9 "context"

10 "crypto/tls"

11 "errors"

12 "fmt"

13 "os"

14 "path/filepath"

15 "strings"

16 "sync"

17 "time"

19 "golang.org/x/exp/slog"

21 "github.com/prometheus/client_golang/prometheus"

22 "github.com/prometheus/client_golang/prometheus/promauto"

24 "github.com/mjl-/bstore"

26 "github.com/mjl-/mox/dns"

27 "github.com/mjl-/mox/mlog"

28 "github.com/mjl-/mox/mox-"

29 "github.com/mjl-/mox/mtasts"

30 "github.com/mjl-/mox/tlsrpt"

31)

33var (

34 metricGet = promauto.NewCounterVec(

35 prometheus.CounterOpts{

36 Name: "mox_mtastsdb_get_total",

37 Help: "Number of Get by result.",

38 },

39 []string{"result"},

40 )

41)

43var timeNow = time.Now // Tests override this.

45// PolicyRecord is a cached policy or absence of a policy.

46type PolicyRecord struct {

47 Domain string // Domain name, with unicode characters.

48 Inserted time.Time `bstore:"default now"`

49 ValidEnd time.Time

50 LastUpdate time.Time // Policies are refreshed on use and periodically.

51 LastUse time.Time `bstore:"index"`

52 Backoff bool

53 RecordID string // As retrieved from DNS.

54 mtasts.Policy // As retrieved from the well-known HTTPS url.

56 // Text that make up the policy, as retrieved. We didn't store this in the past. If

57 // empty, policy can be reconstructed from Policy field. Needed by TLSRPT.

58 PolicyText string

59}

61var (

62 // No valid non-expired policy in database.

63 ErrNotFound = errors.New("mtastsdb: policy not found")

65 // Indicates an MTA-STS TXT record was fetched recently, but fetching the policy

66 // failed and should not yet be retried.

67 ErrBackoff = errors.New("mtastsdb: policy fetch failed recently")

68)

70var DBTypes = []any{PolicyRecord{}} // Types stored in DB.

71var DB *bstore.DB // Exported for backups.

72var mutex sync.Mutex

74func database(ctx context.Context) (rdb *bstore.DB, rerr error) {

75 mutex.Lock()

76 defer mutex.Unlock()

77 if DB == nil {

78 p := mox.DataDirPath("mtasts.db")

79 os.MkdirAll(filepath.Dir(p), 0770)

80 db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)

81 if err != nil {

82 return nil, err

83 }

84 DB = db

85 }

86 return DB, nil

87}

89// Init opens the database and starts a goroutine that refreshes policies in

90// the database, and keeps doing so periodically.

91func Init(refresher bool) error {

92 _, err := database(mox.Shutdown)

93 if err != nil {

94 return err

95 }

97 if refresher {

98 // todo: allow us to shut down cleanly?

99 go refresh()

100 }

101

102 return nil

103}

104

105// Close closes the database.

106func Close() {

107 mutex.Lock()

108 defer mutex.Unlock()

109 if DB != nil {

110 err := DB.Close()

111 mlog.New("mtastsdb", nil).Check(err, "closing database")

112 DB = nil

113 }

114}

115

116// lookup looks up a policy for the domain in the database.

117//

118// Only non-expired records are returned.

119//

120// Returns ErrNotFound if record is not present.

121// Returns ErrBackoff if a recent attempt to fetch a record failed.

122func lookup(ctx context.Context, log mlog.Log, domain dns.Domain) (*PolicyRecord, error) {

123 db, err := database(ctx)

124 if err != nil {

125 return nil, err

126 }

127

128 if domain.IsZero() {

129 return nil, fmt.Errorf("empty domain")

130 }

131 now := timeNow()

132 q := bstore.QueryDB[PolicyRecord](ctx, db)

133 q.FilterNonzero(PolicyRecord{Domain: domain.Name()})

134 q.FilterGreater("ValidEnd", now)

135 pr, err := q.Get()

136 if err == bstore.ErrAbsent {

137 return nil, ErrNotFound

138 } else if err != nil {

139 return nil, err

140 }

141

142 pr.LastUse = now

143 if err := db.Update(ctx, &pr); err != nil {

144 log.Errorx("marking cached mta-sts policy as used in database", err)

145 }

146 if pr.Backoff {

147 return nil, ErrBackoff

148 }

149 return &pr, nil

150}

151

152// Upsert adds the policy to the database, overwriting an existing policy for the domain.

153// Policy can be nil, indicating a failure to fetch the policy.

154func Upsert(ctx context.Context, domain dns.Domain, recordID string, policy *mtasts.Policy, policyText string) error {

155 db, err := database(ctx)

156 if err != nil {

157 return err

158 }

159

160 return db.Write(ctx, func(tx *bstore.Tx) error {

161 pr := PolicyRecord{Domain: domain.Name()}

162 err := tx.Get(&pr)

163 if err != nil && err != bstore.ErrAbsent {

164 return err

165 }

166

167 now := timeNow()

168

169 var p mtasts.Policy

170 if policy != nil {

171 p = *policy

172 } else {

173 // ../rfc/8461:552

174 p.Mode = mtasts.ModeNone

175 p.MaxAgeSeconds = 5 * 60

176 }

177 backoff := policy == nil

178 validEnd := now.Add(time.Duration(p.MaxAgeSeconds) * time.Second)

179

180 if err == bstore.ErrAbsent {

181 pr = PolicyRecord{domain.Name(), now, validEnd, now, now, backoff, recordID, p, policyText}

182 return tx.Insert(&pr)

183 }

184

185 pr.ValidEnd = validEnd

186 pr.LastUpdate = now

187 pr.LastUse = now

188 pr.Backoff = backoff

189 pr.RecordID = recordID

190 pr.Policy = p

191 pr.PolicyText = policyText

192 return tx.Update(&pr)

193 })

194}

195

196// PolicyRecords returns all policies in the database, sorted descending by last

197// use, domain.

198func PolicyRecords(ctx context.Context) ([]PolicyRecord, error) {

199 db, err := database(ctx)

200 if err != nil {

201 return nil, err

202 }

203 return bstore.QueryDB[PolicyRecord](ctx, db).SortDesc("LastUse", "Domain").List()

204}

205

206// Get retrieves an MTA-STS policy for domain and whether it is fresh.

207//

208// If an error is returned, it should be considered a transient error, e.g. a

209// temporary DNS lookup failure.

210//

211// The returned policy can be nil also when there is no error. In this case, the

212// domain does not implement MTA-STS.

213//

214// If a policy is present in the local database, it is refreshed if needed. If no

215// policy is present for the domain, an attempt is made to fetch the policy and

216// store it in the local database.

217//

218// Some errors are logged but not otherwise returned, e.g. if a new policy is

219// supposedly published but could not be retrieved.

220//

221// Get returns an "sts" or "no-policy-found" in reportResult in most cases (when

222// not a local/internal error). It may add an "sts" result without policy contents

223// ("policy-string") in case of errors while fetching the policy.

224func Get(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, domain dns.Domain) (policy *mtasts.Policy, reportResult tlsrpt.Result, fresh bool, err error) {

225 log := mlog.New("mtastsdb", elog)

226 defer func() {

227 result := "ok"

228 if err != nil && errors.Is(err, ErrBackoff) {

229 result = "backoff"

230 } else if err != nil && errors.Is(err, ErrNotFound) {

231 result = "notfound"

232 } else if err != nil {

233 result = "error"

234 }

235 metricGet.WithLabelValues(result).Inc()

236 log.Debugx("mtastsdb get result", err, slog.Any("domain", domain), slog.Bool("fresh", fresh))

237 }()

238

239 cachedPolicy, err := lookup(ctx, log, domain)

240 if err != nil && errors.Is(err, ErrNotFound) {

241 // We don't have a policy for this domain, not even a record that we tried recently

242 // and should backoff. So attempt to fetch policy.

243 nctx, cancel := context.WithTimeout(ctx, time.Minute)

244 defer cancel()

245 record, p, ptext, err := mtasts.Get(nctx, log.Logger, resolver, domain)

246 if err != nil {

247 switch {

248 case errors.Is(err, mtasts.ErrNoRecord) || errors.Is(err, mtasts.ErrMultipleRecords) || errors.Is(err, mtasts.ErrRecordSyntax) || errors.Is(err, mtasts.ErrNoPolicy) || errors.Is(err, mtasts.ErrPolicyFetch) || errors.Is(err, mtasts.ErrPolicySyntax):

249 // Remote is not doing MTA-STS, continue below. ../rfc/8461:333 ../rfc/8461:574

250 log.Debugx("interpreting mtasts error to mean remote is not doing mta-sts", err)

251

252 if errors.Is(err, mtasts.ErrNoRecord) {

253 reportResult = tlsrpt.MakeResult(tlsrpt.NoPolicyFound, domain)

254 } else {

255 fd := policyFetchFailureDetails(err)

256 reportResult = tlsrpt.MakeResult(tlsrpt.STS, domain, fd)

257 }

258

259 default:

260 // Interpret as temporary error, e.g. mtasts.ErrDNS, try again later.

261

262 // Temporary DNS error could be an operational issue on our side, but we can still

263 // report it.

264 // Result: ../rfc/8460:594

265 fd := tlsrpt.Details(tlsrpt.ResultSTSPolicyFetch, mtasts.TLSReportFailureReason(err))

266 reportResult = tlsrpt.MakeResult(tlsrpt.STS, domain, fd)

267

268 return nil, reportResult, false, fmt.Errorf("lookup up mta-sts policy: %w", err)

269 }

270 } else if p.Mode == mtasts.ModeNone {

271 reportResult = tlsrpt.MakeResult(tlsrpt.NoPolicyFound, domain)

272 } else {

273 reportResult = tlsrpt.Result{Policy: tlsrptPolicy(p, ptext, domain)}

274 }

275

276 // Insert policy into database. If we could not fetch the policy itself, we back

277 // off for 5 minutes. ../rfc/8461:555

278 if err == nil || errors.Is(err, mtasts.ErrNoPolicy) || errors.Is(err, mtasts.ErrPolicyFetch) || errors.Is(err, mtasts.ErrPolicySyntax) {

279 var recordID string

280 if record != nil {

281 recordID = record.ID

282 }

283 if err := Upsert(ctx, domain, recordID, p, ptext); err != nil {

284 log.Errorx("inserting policy into cache, continuing", err)

285 }

286 }

287

288 return p, reportResult, true, nil

289 } else if err != nil && errors.Is(err, ErrBackoff) {

290 // ../rfc/8461:552

291 // We recently failed to fetch a policy, act as if MTA-STS is not implemented.

292 // Result: ../rfc/8460:594

293 fd := tlsrpt.Details(tlsrpt.ResultSTSPolicyFetch, "back-off-after-recent-fetch-error")

294 reportResult = tlsrpt.MakeResult(tlsrpt.STS, domain, fd)

295 return nil, reportResult, false, nil

296 } else if err != nil {

297 // We don't add the result to the report, this is an internal error.

298 return nil, reportResult, false, fmt.Errorf("looking up mta-sts policy in cache: %w", err)

299 }

300

301 // Policy was found in database. Check in DNS it is still fresh.

302 policy = &cachedPolicy.Policy

303 nctx, cancel := context.WithTimeout(ctx, 30*time.Second)

304 defer cancel()

305 record, _, err := mtasts.LookupRecord(nctx, log.Logger, resolver, domain)

306 if err != nil {

307 if errors.Is(err, mtasts.ErrNoRecord) {

308 if policy.Mode != mtasts.ModeNone {

309 log.Errorx("no mtasts dns record while checking non-none policy for freshness, either domain owner removed mta-sts without phasing out policy with a none-policy for period of previous max-age, or this could be an attempt to downgrade to connection without mtasts, continuing with previous policy", err)

310 }

311 // else, policy will be removed by periodic refresher in the near future.

312 } else {

313 // Could be a temporary DNS or configuration error.

314 log.Errorx("checking for freshness of cached mta-sts dns txt record for domain, continuing with previously cached policy", err)

315 }

316

317 // Result: ../rfc/8460:594

318 fd := tlsrpt.Details(tlsrpt.ResultSTSPolicyFetch, mtasts.TLSReportFailureReason(err))

319 if policy.Mode != mtasts.ModeNone {

320 fd.FailureReasonCode += "+fallback-to-cached-policy"

321 }