1package main

2

3import (

4 "context"

5 "fmt"

6 "log"

7 "os"

8 "path/filepath"

9 "strings"

10 "time"

11

12 "github.com/mjl-/bstore"

13 "github.com/mjl-/sconf"

14

15 "github.com/mjl-/mox/config"

16 "github.com/mjl-/mox/dmarcdb"

17 "github.com/mjl-/mox/dmarcrpt"

18 "github.com/mjl-/mox/dns"

19 "github.com/mjl-/mox/mlog"

20 "github.com/mjl-/mox/mox-"

21 "github.com/mjl-/mox/moxvar"

22 "github.com/mjl-/mox/mtasts"

23 "github.com/mjl-/mox/mtastsdb"

24 "github.com/mjl-/mox/queue"

25 "github.com/mjl-/mox/smtp"

26 "github.com/mjl-/mox/store"

27 "github.com/mjl-/mox/tlsrpt"

28 "github.com/mjl-/mox/tlsrptdb"

29)

30

31func cmdGentestdata(c *cmd) {

32 c.unlisted = true

33 c.params = "destdir"

34 c.help = `Generate a data directory populated, for testing upgrades.`

35 args := c.Parse()

36 if len(args) != 1 {

37 c.Usage()

38 }

39

40 destDataDir, err := filepath.Abs(args[0])

41 xcheckf(err, "making destination directory an absolute path")

42

43 if _, err := os.Stat(destDataDir); err == nil {

44 log.Fatalf("destination directory already exists, refusing to generate test data")

45 }

46 err = os.MkdirAll(destDataDir, 0770)

47 xcheckf(err, "creating destination data directory")

48 err = os.MkdirAll(filepath.Join(destDataDir, "tmp"), 0770)

49 xcheckf(err, "creating tmp directory")

50

51 tempfile := func() *os.File {

52 f, err := os.CreateTemp(filepath.Join(destDataDir, "tmp"), "temp")

53 xcheckf(err, "creating temp file")

54 return f

55 }

56

57 ctxbg := context.Background()

58 mox.Conf.Log[""] = mlog.LevelInfo

59 mlog.SetConfig(mox.Conf.Log)

60

61 const domainsConf = `

62Domains:

63 mox.example: nil

64 ☺.example: nil

65Accounts:

66 test0:

67 Domain: mox.example

68 Destinations:

69 test0@mox.example: nil

70 test1:

71 Domain: mox.example

72 Destinations:

73 test1@mox.example: nil

74 test2:

75 Domain: ☺.example

76 Destinations:

77 ☹@☺.example: nil

78 JunkFilter:

79 Threshold: 0.95

80 Params:

81 Twograms: true

82 MaxPower: 0.1

83 TopWords: 10

84 IgnoreWords: 0.1

85`

86

87 mox.ConfigStaticPath = filepath.FromSlash("/tmp/mox-bogus/mox.conf")

88 mox.ConfigDynamicPath = filepath.FromSlash("/tmp/mox-bogus/domains.conf")

89 mox.Conf.DynamicLastCheck = time.Now() // Should prevent warning.

90 mox.Conf.Static = config.Static{

91 DataDir: destDataDir,

92 }

93 err = sconf.Parse(strings.NewReader(domainsConf), &mox.Conf.Dynamic)

94 xcheckf(err, "parsing domains config")

95

96 const dmarcReport = `<?xml version="1.0" encoding="UTF-8" ?>

97<feedback>

98 <report_metadata>

99 <org_name>google.com</org_name>

100 <email>noreply-dmarc-support@google.com</email>

101 <extra_contact_info>https://support.google.com/a/answer/2466580</extra_contact_info>

102 <report_id>10051505501689795560</report_id>

103 <date_range>

104 <begin>1596412800</begin>

105 <end>1596499199</end>

106 </date_range>

107 </report_metadata>

108 <policy_published>

109 <domain>mox.example</domain>

110 <adkim>r</adkim>

111 <aspf>r</aspf>

112 <p>reject</p>

113 <sp>reject</sp>

114 <pct>100</pct>

115 </policy_published>

116 <record>

117 <row>

118 <source_ip>127.0.0.1</source_ip>

119 <count>1</count>

120 <policy_evaluated>

121 <disposition>none</disposition>

122 <dkim>pass</dkim>

123 <spf>pass</spf>

124 </policy_evaluated>

125 </row>

126 <identifiers>

127 <header_from>example.org</header_from>

128 </identifiers>

129 <auth_results>

130 <dkim>

131 <domain>example.org</domain>

132 <result>pass</result>

133 <selector>example</selector>

134 </dkim>

135 <spf>

136 <domain>example.org</domain>

137 <result>pass</result>

138 </spf>

139 </auth_results>

140 </record>

141</feedback>

142`

143

144 const tlsReport = `{

145 "organization-name": "Company-X",

146 "date-range": {

147 "start-datetime": "2016-04-01T00:00:00Z",

148 "end-datetime": "2016-04-01T23:59:59Z"

149 },

150 "contact-info": "sts-reporting@company-x.example",

151 "report-id": "5065427c-23d3-47ca-b6e0-946ea0e8c4be",

152 "policies": [{

153 "policy": {

154 "policy-type": "sts",

155 "policy-string": ["version: STSv1","mode: testing",

156 "mx: *.mail.company-y.example","max_age: 86400"],

157 "policy-domain": "mox.example",

158 "mx-host": ["*.mail.company-y.example"]

159 },

160 "summary": {

161 "total-successful-session-count": 5326,

162 "total-failure-session-count": 303

163 },

164 "failure-details": [{

165 "result-type": "certificate-expired",

166 "sending-mta-ip": "2001:db8:abcd:0012::1",

167 "receiving-mx-hostname": "mx1.mail.company-y.example",

168 "failed-session-count": 100

169 }, {

170 "result-type": "starttls-not-supported",

171 "sending-mta-ip": "2001:db8:abcd:0013::1",

172 "receiving-mx-hostname": "mx2.mail.company-y.example",

173 "receiving-ip": "203.0.113.56",

174 "failed-session-count": 200,

175 "additional-information": "https://reports.company-x.example/report_info ? id = 5065427 c - 23 d3# StarttlsNotSupported "

176 }, {

177 "result-type": "validation-failure",

178 "sending-mta-ip": "198.51.100.62",

179 "receiving-ip": "203.0.113.58",

180 "receiving-mx-hostname": "mx-backup.mail.company-y.example",

181 "failed-session-count": 3,

182 "failure-reason-code": "X509_V_ERR_PROXY_PATH_LENGTH_EXCEEDED"

183 }]

184 }]

185 }`

186

187 err = os.WriteFile(filepath.Join(destDataDir, "moxversion"), []byte(moxvar.Version), 0660)

188 xcheckf(err, "writing moxversion")

189

190 // Populate auth.db

191 err = store.Init(ctxbg)

192 xcheckf(err, "store init")

193 err = store.TLSPublicKeyAdd(ctxbg, &store.TLSPublicKey{Name: "testkey", Fingerprint: "...", Type: "ecdsa-p256", CertDER: []byte("..."), Account: "test0", LoginAddress: "test0@mox.example"})

194 xcheckf(err, "adding tlspubkey")

195

196 // Populate dmarc.db.

197 err = dmarcdb.Init()

198 xcheckf(err, "dmarcdb init")

199 report, err := dmarcrpt.ParseReport(strings.NewReader(dmarcReport))

200 xcheckf(err, "parsing dmarc aggregate report")

201 err = dmarcdb.AddReport(ctxbg, report, dns.Domain{ASCII: "mox.example"})

202 xcheckf(err, "adding dmarc aggregate report")

203

204 // Populate mtasts.db.

205 err = mtastsdb.Init(false)

206 xcheckf(err, "mtastsdb init")

207 mtastsPolicy := mtasts.Policy{

208 Version: "STSv1",

209 Mode: mtasts.ModeTesting,

210 MX: []mtasts.MX{

211 {Domain: dns.Domain{ASCII: "mx1.example.com"}},

212 {Domain: dns.Domain{ASCII: "mx2.example.com"}},

213 {Domain: dns.Domain{ASCII: "backup-example.com"}, Wildcard: true},

214 },

215 MaxAgeSeconds: 1296000,

216 }

217 err = mtastsdb.Upsert(ctxbg, dns.Domain{ASCII: "mox.example"}, "123", &mtastsPolicy, mtastsPolicy.String())

218 xcheckf(err, "adding mtastsdb report")

219

220 // Populate tlsrpt.db.

221 err = tlsrptdb.Init()

222 xcheckf(err, "tlsrptdb init")

223 tlsreportJSON, err := tlsrpt.Parse(strings.NewReader(tlsReport))

224 xcheckf(err, "parsing tls report")

225 tlsr := tlsreportJSON.Convert()

226 err = tlsrptdb.AddReport(ctxbg, c.log, dns.Domain{ASCII: "mox.example"}, "tlsrpt@mox.example", false, &tlsr)

227 xcheckf(err, "adding tls report")

228

229 // Populate queue, with a message.

230 err = queue.Init()

231 xcheckf(err, "queue init")

232 mailfrom := smtp.Path{Localpart: "other", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "other.example"}}}

233 rcptto := smtp.Path{Localpart: "test0", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "mox.example"}}}

234 prefix := []byte{}

235 mf := tempfile()

236 xcheckf(err, "temp file for queue message")

237 defer store.CloseRemoveTempFile(c.log, mf, "test message")

238 const qmsg = "From: <test0@mox.example>\r\nTo: <other@remote.example>\r\nSubject: test\r\n\r\nthe message...\r\n"

239 _, err = fmt.Fprint(mf, qmsg)

240 xcheckf(err, "writing message")

241 qm := queue.MakeMsg(mailfrom, rcptto, false, false, int64(len(qmsg)), "<test@localhost>", prefix, nil, time.Now(), "test")

242 err = queue.Add(ctxbg, c.log, "test0", mf, qm)

243 xcheckf(err, "enqueue message")

244

245 // Create three accounts.

246 // First account without messages.

247 accTest0, err := store.OpenAccount(c.log, "test0", false)

248 xcheckf(err, "open account test0")

249 err = accTest0.ThreadingWait(c.log)

250 xcheckf(err, "wait for threading to finish")

251 err = accTest0.Close()

252 xcheckf(err, "close account")

253

254 // Second account with one message.

255 accTest1, err := store.OpenAccount(c.log, "test1", false)

256 xcheckf(err, "open account test1")

257 err = accTest1.ThreadingWait(c.log)

258 xcheckf(err, "wait for threading to finish")

259 err = accTest1.DB.Write(ctxbg, func(tx *bstore.Tx) error {

260 inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()

261 xcheckf(err, "looking up inbox")

262 const msg = "From: <other@remote.example>\r\nTo: <test1@mox.example>\r\nSubject: test\r\n\r\nthe message...\r\n"

263 m := store.Message{

264 MailboxID: inbox.ID,

265 MailboxOrigID: inbox.ID,

266 RemoteIP: "1.2.3.4",

267 RemoteIPMasked1: "1.2.3.4",

268 RemoteIPMasked2: "1.2.3.0",

269 RemoteIPMasked3: "1.2.0.0",

270 EHLODomain: "other.example",

271 MailFrom: "other@remote.example",

272 MailFromLocalpart: smtp.Localpart("other"),

273 MailFromDomain: "remote.example",

274 RcptToLocalpart: "test1",

275 RcptToDomain: "mox.example",

276 MsgFromLocalpart: "other",

277 MsgFromDomain: "remote.example",

278 MsgFromOrgDomain: "remote.example",

279 EHLOValidated: true,

280 MailFromValidated: true,

281 MsgFromValidated: true,

282 EHLOValidation: store.ValidationStrict,

283 MailFromValidation: store.ValidationPass,

284 MsgFromValidation: store.ValidationStrict,

285 DKIMDomains: []string{"other.example"},

286 Size: int64(len(msg)),

287 }

288 mf := tempfile()

289 xcheckf(err, "creating temp file for delivery")

290 defer store.CloseRemoveTempFile(c.log, mf, "test message")

291 _, err = fmt.Fprint(mf, msg)

292 xcheckf(err, "writing deliver message to file")

293

294 err = accTest1.MessageAdd(c.log, tx, &inbox, &m, mf, store.AddOpts{})

295 xcheckf(err, "deliver message")

296

297 err = tx.Update(&inbox)

298 xcheckf(err, "update inbox")

299

300 return nil

301 })

302 xcheckf(err, "write transaction with new message")

303 err = accTest1.Close()

304 xcheckf(err, "close account")

305

306 // Third account with two messages and junkfilter.

307 accTest2, err := store.OpenAccount(c.log, "test2", false)

308 xcheckf(err, "open account test2")

309 err = accTest2.ThreadingWait(c.log)

310 xcheckf(err, "wait for threading to finish")

311 err = accTest2.DB.Write(ctxbg, func(tx *bstore.Tx) error {

312 inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()

313 xcheckf(err, "looking up inbox")

314 const msg0 = "From: <other@remote.example>\r\nTo: <☹@xn--74h.example>\r\nSubject: test\r\n\r\nthe message...\r\n"

315 m0 := store.Message{

316 MailboxID: inbox.ID,

317 MailboxOrigID: inbox.ID,

318 RemoteIP: "::1",

319 RemoteIPMasked1: "::",

320 RemoteIPMasked2: "::",

321 RemoteIPMasked3: "::",

322 EHLODomain: "other.example",

323 MailFrom: "other@remote.example",

324 MailFromLocalpart: smtp.Localpart("other"),

325 MailFromDomain: "remote.example",

326 RcptToLocalpart: "☹",

327 RcptToDomain: "☺.example",

328 MsgFromLocalpart: "other",

329 MsgFromDomain: "remote.example",

330 MsgFromOrgDomain: "remote.example",

331 EHLOValidated: true,

332 MailFromValidated: true,

333 MsgFromValidated: true,

334 EHLOValidation: store.ValidationStrict,

335 MailFromValidation: store.ValidationPass,

336 MsgFromValidation: store.ValidationStrict,

337 DKIMDomains: []string{"other.example"},

338 Size: int64(len(msg0)),

339 }

340 mf0 := tempfile()

341 xcheckf(err, "creating temp file for delivery")

342 defer store.CloseRemoveTempFile(c.log, mf0, "test message")

343 _, err = fmt.Fprint(mf0, msg0)

344 xcheckf(err, "writing deliver message to file")

345 err = accTest2.MessageAdd(c.log, tx, &inbox, &m0, mf0, store.AddOpts{})

346 xcheckf(err, "add message to account test2")

347 err = tx.Update(&inbox)

348 xcheckf(err, "update inbox")

349

350 sent, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Sent"}).Get()

351 xcheckf(err, "looking up inbox")

352 const prefix1 = "Extra: test\r\n"

353 const msg1 = "From: <other@remote.example>\r\nTo: <☹@xn--74h.example>\r\nSubject: test\r\n\r\nthe message...\r\n"

354 m1 := store.Message{

355 MailboxID: sent.ID,

356 MailboxOrigID: sent.ID,

357 Flags: store.Flags{Seen: true, Junk: true},

358 Size: int64(len(prefix1) + len(msg1)),

359 MsgPrefix: []byte(prefix1),

360 }

361 mf1 := tempfile()

362 xcheckf(err, "creating temp file for delivery")

363 defer store.CloseRemoveTempFile(c.log, mf1, "test message")

364 _, err = fmt.Fprint(mf1, msg1)

365 xcheckf(err, "writing deliver message to file")

366 err = accTest2.MessageAdd(c.log, tx, &sent, &m1, mf1, store.AddOpts{})

367 xcheckf(err, "add message to account test2")

368 err = tx.Update(&sent)

369 xcheckf(err, "update sent")

370

371 return nil

372 })

373 xcheckf(err, "write transaction with new message")

374 err = accTest2.Close()

375 xcheckf(err, "close account")

376}

377