1package main
2
3import (
4 "context"
5 "fmt"
6 "log"
7 "os"
8 "path/filepath"
9 "strings"
10 "time"
11
12 "github.com/mjl-/bstore"
13 "github.com/mjl-/sconf"
14
15 "github.com/mjl-/mox/config"
16 "github.com/mjl-/mox/dmarcdb"
17 "github.com/mjl-/mox/dmarcrpt"
18 "github.com/mjl-/mox/dns"
19 "github.com/mjl-/mox/mlog"
20 "github.com/mjl-/mox/mox-"
21 "github.com/mjl-/mox/moxvar"
22 "github.com/mjl-/mox/mtasts"
23 "github.com/mjl-/mox/mtastsdb"
24 "github.com/mjl-/mox/queue"
25 "github.com/mjl-/mox/smtp"
26 "github.com/mjl-/mox/store"
27 "github.com/mjl-/mox/tlsrpt"
28 "github.com/mjl-/mox/tlsrptdb"
29)
30
31func cmdGentestdata(c *cmd) {
32 c.unlisted = true
33 c.params = "dest-dir"
34 c.help = `Generate a data directory populated, for testing upgrades.`
35 args := c.Parse()
36 if len(args) != 1 {
37 c.Usage()
38 }
39
40 destDataDir, err := filepath.Abs(args[0])
41 xcheckf(err, "making destination directory an absolute path")
42
43 if _, err := os.Stat(destDataDir); err == nil {
44 log.Fatalf("destination directory already exists, refusing to generate test data")
45 }
46 err = os.MkdirAll(destDataDir, 0770)
47 xcheckf(err, "creating destination data directory")
48 err = os.MkdirAll(filepath.Join(destDataDir, "tmp"), 0770)
49 xcheckf(err, "creating tmp directory")
50
51 tempfile := func() *os.File {
52 f, err := os.CreateTemp(filepath.Join(destDataDir, "tmp"), "temp")
53 xcheckf(err, "creating temp file")
54 return f
55 }
56
57 ctxbg := context.Background()
58 mox.Conf.Log[""] = mlog.LevelInfo
59 mlog.SetConfig(mox.Conf.Log)
60
61 const domainsConf = `
62Domains:
63 mox.example: nil
64 ☺.example: nil
65Accounts:
66 test0:
67 Domain: mox.example
68 Destinations:
69 test0@mox.example: nil
70 test1:
71 Domain: mox.example
72 Destinations:
73 test1@mox.example: nil
74 test2:
75 Domain: ☺.example
76 Destinations:
77 ☹@☺.example: nil
78 JunkFilter:
79 Threshold: 0.95
80 Params:
81 Twograms: true
82 MaxPower: 0.1
83 TopWords: 10
84 IgnoreWords: 0.1
85`
86
87 mox.ConfigStaticPath = filepath.FromSlash("/tmp/mox-bogus/mox.conf")
88 mox.ConfigDynamicPath = filepath.FromSlash("/tmp/mox-bogus/domains.conf")
89 mox.Conf.DynamicLastCheck = time.Now() // Should prevent warning.
90 mox.Conf.Static = config.Static{
91 DataDir: destDataDir,
92 }
93 err = sconf.Parse(strings.NewReader(domainsConf), &mox.Conf.Dynamic)
94 xcheckf(err, "parsing domains config")
95
96 const dmarcReport = `<?xml version="1.0" encoding="UTF-8" ?>
97<feedback>
98 <report_metadata>
99 <org_name>google.com</org_name>
100 <email>noreply-dmarc-support@google.com</email>
101 <extra_contact_info>https://support.google.com/a/answer/2466580</extra_contact_info>
102 <report_id>10051505501689795560</report_id>
103 <date_range>
104 <begin>1596412800</begin>
105 <end>1596499199</end>
106 </date_range>
107 </report_metadata>
108 <policy_published>
109 <domain>mox.example</domain>
110 <adkim>r</adkim>
111 <aspf>r</aspf>
112 <p>reject</p>
113 <sp>reject</sp>
114 <pct>100</pct>
115 </policy_published>
116 <record>
117 <row>
118 <source_ip>127.0.0.1</source_ip>
119 <count>1</count>
120 <policy_evaluated>
121 <disposition>none</disposition>
122 <dkim>pass</dkim>
123 <spf>pass</spf>
124 </policy_evaluated>
125 </row>
126 <identifiers>
127 <header_from>example.org</header_from>
128 </identifiers>
129 <auth_results>
130 <dkim>
131 <domain>example.org</domain>
132 <result>pass</result>
133 <selector>example</selector>
134 </dkim>
135 <spf>
136 <domain>example.org</domain>
137 <result>pass</result>
138 </spf>
139 </auth_results>
140 </record>
141</feedback>
142`
143
144 const tlsReport = `{
145 "organization-name": "Company-X",
146 "date-range": {
147 "start-datetime": "2016-04-01T00:00:00Z",
148 "end-datetime": "2016-04-01T23:59:59Z"
149 },
150 "contact-info": "sts-reporting@company-x.example",
151 "report-id": "5065427c-23d3-47ca-b6e0-946ea0e8c4be",
152 "policies": [{
153 "policy": {
154 "policy-type": "sts",
155 "policy-string": ["version: STSv1","mode: testing",
156 "mx: *.mail.company-y.example","max_age: 86400"],
157 "policy-domain": "mox.example",
158 "mx-host": ["*.mail.company-y.example"]
159 },
160 "summary": {
161 "total-successful-session-count": 5326,
162 "total-failure-session-count": 303
163 },
164 "failure-details": [{
165 "result-type": "certificate-expired",
166 "sending-mta-ip": "2001:db8:abcd:0012::1",
167 "receiving-mx-hostname": "mx1.mail.company-y.example",
168 "failed-session-count": 100
169 }, {
170 "result-type": "starttls-not-supported",
171 "sending-mta-ip": "2001:db8:abcd:0013::1",
172 "receiving-mx-hostname": "mx2.mail.company-y.example",
173 "receiving-ip": "203.0.113.56",
174 "failed-session-count": 200,
175 "additional-information": "https://reports.company-x.example/report_info ? id = 5065427 c - 23 d3# StarttlsNotSupported "
176 }, {
177 "result-type": "validation-failure",
178 "sending-mta-ip": "198.51.100.62",
179 "receiving-ip": "203.0.113.58",
180 "receiving-mx-hostname": "mx-backup.mail.company-y.example",
181 "failed-session-count": 3,
182 "failure-reason-code": "X509_V_ERR_PROXY_PATH_LENGTH_EXCEEDED"
183 }]
184 }]
185 }`
186
187 err = os.WriteFile(filepath.Join(destDataDir, "moxversion"), []byte(moxvar.Version), 0660)
188 xcheckf(err, "writing moxversion")
189
190 // Populate auth.db
191 err = store.Init(ctxbg)
192 xcheckf(err, "store init")
193 err = store.TLSPublicKeyAdd(ctxbg, &store.TLSPublicKey{Fingerprint: "...", Type: "ecdsa-p256", CertDER: []byte("..."), Account: "test0", LoginAddress: "test0@mox.example"})
194 xcheckf(err, "adding tlspubkey")
195
196 // Populate dmarc.db.
197 err = dmarcdb.Init()
198 xcheckf(err, "dmarcdb init")
199 report, err := dmarcrpt.ParseReport(strings.NewReader(dmarcReport))
200 xcheckf(err, "parsing dmarc aggregate report")
201 err = dmarcdb.AddReport(ctxbg, report, dns.Domain{ASCII: "mox.example"})
202 xcheckf(err, "adding dmarc aggregate report")
203
204 // Populate mtasts.db.
205 err = mtastsdb.Init(false)
206 xcheckf(err, "mtastsdb init")
207 mtastsPolicy := mtasts.Policy{
208 Version: "STSv1",
209 Mode: mtasts.ModeTesting,
210 MX: []mtasts.MX{
211 {Domain: dns.Domain{ASCII: "mx1.example.com"}},
212 {Domain: dns.Domain{ASCII: "mx2.example.com"}},
213 {Domain: dns.Domain{ASCII: "backup-example.com"}, Wildcard: true},
214 },
215 MaxAgeSeconds: 1296000,
216 }
217 err = mtastsdb.Upsert(ctxbg, dns.Domain{ASCII: "mox.example"}, "123", &mtastsPolicy, mtastsPolicy.String())
218 xcheckf(err, "adding mtastsdb report")
219
220 // Populate tlsrpt.db.
221 err = tlsrptdb.Init()
222 xcheckf(err, "tlsrptdb init")
223 tlsreportJSON, err := tlsrpt.Parse(strings.NewReader(tlsReport))
224 xcheckf(err, "parsing tls report")
225 tlsr := tlsreportJSON.Convert()
226 err = tlsrptdb.AddReport(ctxbg, c.log, dns.Domain{ASCII: "mox.example"}, "tlsrpt@mox.example", false, &tlsr)
227 xcheckf(err, "adding tls report")
228
229 // Populate queue, with a message.
230 err = queue.Init()
231 xcheckf(err, "queue init")
232 mailfrom := smtp.Path{Localpart: "other", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "other.example"}}}
233 rcptto := smtp.Path{Localpart: "test0", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "mox.example"}}}
234 prefix := []byte{}
235 mf := tempfile()
236 xcheckf(err, "temp file for queue message")
237 defer os.Remove(mf.Name())
238 defer mf.Close()
239 const qmsg = "From: <test0@mox.example>\r\nTo: <other@remote.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
240 _, err = fmt.Fprint(mf, qmsg)
241 xcheckf(err, "writing message")
242 qm := queue.MakeMsg(mailfrom, rcptto, false, false, int64(len(qmsg)), "<test@localhost>", prefix, nil, time.Now(), "test")
243 err = queue.Add(ctxbg, c.log, "test0", mf, qm)
244 xcheckf(err, "enqueue message")
245
246 // Create three accounts.
247 // First account without messages.
248 accTest0, err := store.OpenAccount(c.log, "test0")
249 xcheckf(err, "open account test0")
250 err = accTest0.ThreadingWait(c.log)
251 xcheckf(err, "wait for threading to finish")
252 err = accTest0.Close()
253 xcheckf(err, "close account")
254
255 // Second account with one message.
256 accTest1, err := store.OpenAccount(c.log, "test1")
257 xcheckf(err, "open account test1")
258 err = accTest1.ThreadingWait(c.log)
259 xcheckf(err, "wait for threading to finish")
260 err = accTest1.DB.Write(ctxbg, func(tx *bstore.Tx) error {
261 inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()
262 xcheckf(err, "looking up inbox")
263 const msg = "From: <other@remote.example>\r\nTo: <test1@mox.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
264 m := store.Message{
265 MailboxID: inbox.ID,
266 MailboxOrigID: inbox.ID,
267 MailboxDestinedID: inbox.ID,
268 RemoteIP: "1.2.3.4",
269 RemoteIPMasked1: "1.2.3.4",
270 RemoteIPMasked2: "1.2.3.0",
271 RemoteIPMasked3: "1.2.0.0",
272 EHLODomain: "other.example",
273 MailFrom: "other@remote.example",
274 MailFromLocalpart: smtp.Localpart("other"),
275 MailFromDomain: "remote.example",
276 RcptToLocalpart: "test1",
277 RcptToDomain: "mox.example",
278 MsgFromLocalpart: "other",
279 MsgFromDomain: "remote.example",
280 MsgFromOrgDomain: "remote.example",
281 EHLOValidated: true,
282 MailFromValidated: true,
283 MsgFromValidated: true,
284 EHLOValidation: store.ValidationStrict,
285 MailFromValidation: store.ValidationPass,
286 MsgFromValidation: store.ValidationStrict,
287 DKIMDomains: []string{"other.example"},
288 Size: int64(len(msg)),
289 }
290 mf := tempfile()
291 xcheckf(err, "creating temp file for delivery")
292 _, err = fmt.Fprint(mf, msg)
293 xcheckf(err, "writing deliver message to file")
294 err = accTest1.DeliverMessage(c.log, tx, &m, mf, false, true, false, true)
295
296 mfname := mf.Name()
297 xcheckf(err, "add message to account test1")
298 err = mf.Close()
299 xcheckf(err, "closing file")
300 err = os.Remove(mfname)
301 xcheckf(err, "removing temp message file")
302
303 err = tx.Get(&inbox)
304 xcheckf(err, "get inbox")
305 inbox.Add(m.MailboxCounts())
306 err = tx.Update(&inbox)
307 xcheckf(err, "update inbox")
308
309 return nil
310 })
311 xcheckf(err, "write transaction with new message")
312 err = accTest1.Close()
313 xcheckf(err, "close account")
314
315 // Third account with two messages and junkfilter.
316 accTest2, err := store.OpenAccount(c.log, "test2")
317 xcheckf(err, "open account test2")
318 err = accTest2.ThreadingWait(c.log)
319 xcheckf(err, "wait for threading to finish")
320 err = accTest2.DB.Write(ctxbg, func(tx *bstore.Tx) error {
321 inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()
322 xcheckf(err, "looking up inbox")
323 const msg0 = "From: <other@remote.example>\r\nTo: <☹@xn--74h.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
324 m0 := store.Message{
325 MailboxID: inbox.ID,
326 MailboxOrigID: inbox.ID,
327 MailboxDestinedID: inbox.ID,
328 RemoteIP: "::1",
329 RemoteIPMasked1: "::",
330 RemoteIPMasked2: "::",
331 RemoteIPMasked3: "::",
332 EHLODomain: "other.example",
333 MailFrom: "other@remote.example",
334 MailFromLocalpart: smtp.Localpart("other"),
335 MailFromDomain: "remote.example",
336 RcptToLocalpart: "☹",
337 RcptToDomain: "☺.example",
338 MsgFromLocalpart: "other",
339 MsgFromDomain: "remote.example",
340 MsgFromOrgDomain: "remote.example",
341 EHLOValidated: true,
342 MailFromValidated: true,
343 MsgFromValidated: true,
344 EHLOValidation: store.ValidationStrict,
345 MailFromValidation: store.ValidationPass,
346 MsgFromValidation: store.ValidationStrict,
347 DKIMDomains: []string{"other.example"},
348 Size: int64(len(msg0)),
349 }
350 mf0 := tempfile()
351 xcheckf(err, "creating temp file for delivery")
352 _, err = fmt.Fprint(mf0, msg0)
353 xcheckf(err, "writing deliver message to file")
354 err = accTest2.DeliverMessage(c.log, tx, &m0, mf0, false, false, false, true)
355 xcheckf(err, "add message to account test2")
356
357 mf0name := mf0.Name()
358 err = mf0.Close()
359 xcheckf(err, "closing file")
360 err = os.Remove(mf0name)
361 xcheckf(err, "removing temp message file")
362
363 err = tx.Get(&inbox)
364 xcheckf(err, "get inbox")
365 inbox.Add(m0.MailboxCounts())
366 err = tx.Update(&inbox)
367 xcheckf(err, "update inbox")
368
369 sent, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Sent"}).Get()
370 xcheckf(err, "looking up inbox")
371 const prefix1 = "Extra: test\r\n"
372 const msg1 = "From: <other@remote.example>\r\nTo: <☹@xn--74h.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
373 m1 := store.Message{
374 MailboxID: sent.ID,
375 MailboxOrigID: sent.ID,
376 MailboxDestinedID: sent.ID,
377 Flags: store.Flags{Seen: true, Junk: true},
378 Size: int64(len(prefix1) + len(msg1)),
379 MsgPrefix: []byte(prefix1),
380 }
381 mf1 := tempfile()
382 xcheckf(err, "creating temp file for delivery")
383 _, err = fmt.Fprint(mf1, msg1)
384 xcheckf(err, "writing deliver message to file")
385 err = accTest2.DeliverMessage(c.log, tx, &m1, mf1, false, false, false, true)
386 xcheckf(err, "add message to account test2")
387
388 mf1name := mf1.Name()
389 err = mf1.Close()
390 xcheckf(err, "closing file")
391 err = os.Remove(mf1name)
392 xcheckf(err, "removing temp message file")
393
394 err = tx.Get(&sent)
395 xcheckf(err, "get sent")
396 sent.Add(m1.MailboxCounts())
397 err = tx.Update(&sent)
398 xcheckf(err, "update sent")
399
400 return nil
401 })
402 xcheckf(err, "write transaction with new message")
403 err = accTest2.Close()
404 xcheckf(err, "close account")
405}
406