1package main
2
3import (
4 "context"
5 "fmt"
6 "log"
7 "os"
8 "path/filepath"
9 "strings"
10 "time"
11
12 "github.com/mjl-/bstore"
13 "github.com/mjl-/sconf"
14
15 "github.com/mjl-/mox/config"
16 "github.com/mjl-/mox/dmarcdb"
17 "github.com/mjl-/mox/dmarcrpt"
18 "github.com/mjl-/mox/dns"
19 "github.com/mjl-/mox/mlog"
20 "github.com/mjl-/mox/mox-"
21 "github.com/mjl-/mox/moxvar"
22 "github.com/mjl-/mox/mtasts"
23 "github.com/mjl-/mox/mtastsdb"
24 "github.com/mjl-/mox/queue"
25 "github.com/mjl-/mox/smtp"
26 "github.com/mjl-/mox/store"
27 "github.com/mjl-/mox/tlsrpt"
28 "github.com/mjl-/mox/tlsrptdb"
29)
30
31func cmdGentestdata(c *cmd) {
32 c.unlisted = true
33 c.params = "dest-dir"
34 c.help = `Generate a data directory populated, for testing upgrades.`
35 args := c.Parse()
36 if len(args) != 1 {
37 c.Usage()
38 }
39
40 destDataDir, err := filepath.Abs(args[0])
41 xcheckf(err, "making destination directory an absolute path")
42
43 if _, err := os.Stat(destDataDir); err == nil {
44 log.Fatalf("destination directory already exists, refusing to generate test data")
45 }
46 err = os.MkdirAll(destDataDir, 0770)
47 xcheckf(err, "creating destination data directory")
48 err = os.MkdirAll(filepath.Join(destDataDir, "tmp"), 0770)
49 xcheckf(err, "creating tmp directory")
50
51 tempfile := func() *os.File {
52 f, err := os.CreateTemp(filepath.Join(destDataDir, "tmp"), "temp")
53 xcheckf(err, "creating temp file")
54 return f
55 }
56
57 ctxbg := context.Background()
58 mox.Conf.Log[""] = mlog.LevelInfo
59 mlog.SetConfig(mox.Conf.Log)
60
61 const domainsConf = `
62Domains:
63 mox.example: nil
64 ☺.example: nil
65Accounts:
66 test0:
67 Domain: mox.example
68 Destinations:
69 test0@mox.example: nil
70 test1:
71 Domain: mox.example
72 Destinations:
73 test1@mox.example: nil
74 test2:
75 Domain: ☺.example
76 Destinations:
77 ☹@☺.example: nil
78 JunkFilter:
79 Threshold: 0.95
80 Params:
81 Twograms: true
82 MaxPower: 0.1
83 TopWords: 10
84 IgnoreWords: 0.1
85`
86
87 mox.ConfigStaticPath = filepath.FromSlash("/tmp/mox-bogus/mox.conf")
88 mox.ConfigDynamicPath = filepath.FromSlash("/tmp/mox-bogus/domains.conf")
89 mox.Conf.DynamicLastCheck = time.Now() // Should prevent warning.
90 mox.Conf.Static = config.Static{
91 DataDir: destDataDir,
92 }
93 err = sconf.Parse(strings.NewReader(domainsConf), &mox.Conf.Dynamic)
94 xcheckf(err, "parsing domains config")
95
96 const dmarcReport = `<?xml version="1.0" encoding="UTF-8" ?>
97<feedback>
98 <report_metadata>
99 <org_name>google.com</org_name>
100 <email>noreply-dmarc-support@google.com</email>
101 <extra_contact_info>https://support.google.com/a/answer/2466580</extra_contact_info>
102 <report_id>10051505501689795560</report_id>
103 <date_range>
104 <begin>1596412800</begin>
105 <end>1596499199</end>
106 </date_range>
107 </report_metadata>
108 <policy_published>
109 <domain>mox.example</domain>
110 <adkim>r</adkim>
111 <aspf>r</aspf>
112 <p>reject</p>
113 <sp>reject</sp>
114 <pct>100</pct>
115 </policy_published>
116 <record>
117 <row>
118 <source_ip>127.0.0.1</source_ip>
119 <count>1</count>
120 <policy_evaluated>
121 <disposition>none</disposition>
122 <dkim>pass</dkim>
123 <spf>pass</spf>
124 </policy_evaluated>
125 </row>
126 <identifiers>
127 <header_from>example.org</header_from>
128 </identifiers>
129 <auth_results>
130 <dkim>
131 <domain>example.org</domain>
132 <result>pass</result>
133 <selector>example</selector>
134 </dkim>
135 <spf>
136 <domain>example.org</domain>
137 <result>pass</result>
138 </spf>
139 </auth_results>
140 </record>
141</feedback>
142`
143
144 const tlsReport = `{
145 "organization-name": "Company-X",
146 "date-range": {
147 "start-datetime": "2016-04-01T00:00:00Z",
148 "end-datetime": "2016-04-01T23:59:59Z"
149 },
150 "contact-info": "sts-reporting@company-x.example",
151 "report-id": "5065427c-23d3-47ca-b6e0-946ea0e8c4be",
152 "policies": [{
153 "policy": {
154 "policy-type": "sts",
155 "policy-string": ["version: STSv1","mode: testing",
156 "mx: *.mail.company-y.example","max_age: 86400"],
157 "policy-domain": "mox.example",
158 "mx-host": ["*.mail.company-y.example"]
159 },
160 "summary": {
161 "total-successful-session-count": 5326,
162 "total-failure-session-count": 303
163 },
164 "failure-details": [{
165 "result-type": "certificate-expired",
166 "sending-mta-ip": "2001:db8:abcd:0012::1",
167 "receiving-mx-hostname": "mx1.mail.company-y.example",
168 "failed-session-count": 100
169 }, {
170 "result-type": "starttls-not-supported",
171 "sending-mta-ip": "2001:db8:abcd:0013::1",
172 "receiving-mx-hostname": "mx2.mail.company-y.example",
173 "receiving-ip": "203.0.113.56",
174 "failed-session-count": 200,
175 "additional-information": "https://reports.company-x.example/report_info ? id = 5065427 c - 23 d3# StarttlsNotSupported "
176 }, {
177 "result-type": "validation-failure",
178 "sending-mta-ip": "198.51.100.62",
179 "receiving-ip": "203.0.113.58",
180 "receiving-mx-hostname": "mx-backup.mail.company-y.example",
181 "failed-session-count": 3,
182 "failure-reason-code": "X509_V_ERR_PROXY_PATH_LENGTH_EXCEEDED"
183 }]
184 }]
185 }`
186
187 err = os.WriteFile(filepath.Join(destDataDir, "moxversion"), []byte(moxvar.Version), 0660)
188 xcheckf(err, "writing moxversion")
189
190 // Populate dmarc.db.
191 err = dmarcdb.Init()
192 xcheckf(err, "dmarcdb init")
193 report, err := dmarcrpt.ParseReport(strings.NewReader(dmarcReport))
194 xcheckf(err, "parsing dmarc aggregate report")
195 err = dmarcdb.AddReport(ctxbg, report, dns.Domain{ASCII: "mox.example"})
196 xcheckf(err, "adding dmarc aggregate report")
197
198 // Populate mtasts.db.
199 err = mtastsdb.Init(false)
200 xcheckf(err, "mtastsdb init")
201 mtastsPolicy := mtasts.Policy{
202 Version: "STSv1",
203 Mode: mtasts.ModeTesting,
204 MX: []mtasts.MX{
205 {Domain: dns.Domain{ASCII: "mx1.example.com"}},
206 {Domain: dns.Domain{ASCII: "mx2.example.com"}},
207 {Domain: dns.Domain{ASCII: "backup-example.com"}, Wildcard: true},
208 },
209 MaxAgeSeconds: 1296000,
210 }
211 err = mtastsdb.Upsert(ctxbg, dns.Domain{ASCII: "mox.example"}, "123", &mtastsPolicy, mtastsPolicy.String())
212 xcheckf(err, "adding mtastsdb report")
213
214 // Populate tlsrpt.db.
215 err = tlsrptdb.Init()
216 xcheckf(err, "tlsrptdb init")
217 tlsreportJSON, err := tlsrpt.Parse(strings.NewReader(tlsReport))
218 xcheckf(err, "parsing tls report")
219 tlsr := tlsreportJSON.Convert()
220 err = tlsrptdb.AddReport(ctxbg, c.log, dns.Domain{ASCII: "mox.example"}, "tlsrpt@mox.example", false, &tlsr)
221 xcheckf(err, "adding tls report")
222
223 // Populate queue, with a message.
224 err = queue.Init()
225 xcheckf(err, "queue init")
226 mailfrom := smtp.Path{Localpart: "other", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "other.example"}}}
227 rcptto := smtp.Path{Localpart: "test0", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "mox.example"}}}
228 prefix := []byte{}
229 mf := tempfile()
230 xcheckf(err, "temp file for queue message")
231 defer os.Remove(mf.Name())
232 defer mf.Close()
233 const qmsg = "From: <test0@mox.example>\r\nTo: <other@remote.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
234 _, err = fmt.Fprint(mf, qmsg)
235 xcheckf(err, "writing message")
236 qm := queue.MakeMsg(mailfrom, rcptto, false, false, int64(len(qmsg)), "<test@localhost>", prefix, nil, time.Now(), "test")
237 err = queue.Add(ctxbg, c.log, "test0", mf, qm)
238 xcheckf(err, "enqueue message")
239
240 // Create three accounts.
241 // First account without messages.
242 accTest0, err := store.OpenAccount(c.log, "test0")
243 xcheckf(err, "open account test0")
244 err = accTest0.ThreadingWait(c.log)
245 xcheckf(err, "wait for threading to finish")
246 err = accTest0.Close()
247 xcheckf(err, "close account")
248
249 // Second account with one message.
250 accTest1, err := store.OpenAccount(c.log, "test1")
251 xcheckf(err, "open account test1")
252 err = accTest1.ThreadingWait(c.log)
253 xcheckf(err, "wait for threading to finish")
254 err = accTest1.DB.Write(ctxbg, func(tx *bstore.Tx) error {
255 inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()
256 xcheckf(err, "looking up inbox")
257 const msg = "From: <other@remote.example>\r\nTo: <test1@mox.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
258 m := store.Message{
259 MailboxID: inbox.ID,
260 MailboxOrigID: inbox.ID,
261 MailboxDestinedID: inbox.ID,
262 RemoteIP: "1.2.3.4",
263 RemoteIPMasked1: "1.2.3.4",
264 RemoteIPMasked2: "1.2.3.0",
265 RemoteIPMasked3: "1.2.0.0",
266 EHLODomain: "other.example",
267 MailFrom: "other@remote.example",
268 MailFromLocalpart: smtp.Localpart("other"),
269 MailFromDomain: "remote.example",
270 RcptToLocalpart: "test1",
271 RcptToDomain: "mox.example",
272 MsgFromLocalpart: "other",
273 MsgFromDomain: "remote.example",
274 MsgFromOrgDomain: "remote.example",
275 EHLOValidated: true,
276 MailFromValidated: true,
277 MsgFromValidated: true,
278 EHLOValidation: store.ValidationStrict,
279 MailFromValidation: store.ValidationPass,
280 MsgFromValidation: store.ValidationStrict,
281 DKIMDomains: []string{"other.example"},
282 Size: int64(len(msg)),
283 }
284 mf := tempfile()
285 xcheckf(err, "creating temp file for delivery")
286 _, err = fmt.Fprint(mf, msg)
287 xcheckf(err, "writing deliver message to file")
288 err = accTest1.DeliverMessage(c.log, tx, &m, mf, false, true, false, true)
289
290 mfname := mf.Name()
291 xcheckf(err, "add message to account test1")
292 err = mf.Close()
293 xcheckf(err, "closing file")
294 err = os.Remove(mfname)
295 xcheckf(err, "removing temp message file")
296
297 err = tx.Get(&inbox)
298 xcheckf(err, "get inbox")
299 inbox.Add(m.MailboxCounts())
300 err = tx.Update(&inbox)
301 xcheckf(err, "update inbox")
302
303 return nil
304 })
305 xcheckf(err, "write transaction with new message")
306 err = accTest1.Close()
307 xcheckf(err, "close account")
308
309 // Third account with two messages and junkfilter.
310 accTest2, err := store.OpenAccount(c.log, "test2")
311 xcheckf(err, "open account test2")
312 err = accTest2.ThreadingWait(c.log)
313 xcheckf(err, "wait for threading to finish")
314 err = accTest2.DB.Write(ctxbg, func(tx *bstore.Tx) error {
315 inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()
316 xcheckf(err, "looking up inbox")
317 const msg0 = "From: <other@remote.example>\r\nTo: <☹@xn--74h.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
318 m0 := store.Message{
319 MailboxID: inbox.ID,
320 MailboxOrigID: inbox.ID,
321 MailboxDestinedID: inbox.ID,
322 RemoteIP: "::1",
323 RemoteIPMasked1: "::",
324 RemoteIPMasked2: "::",
325 RemoteIPMasked3: "::",
326 EHLODomain: "other.example",
327 MailFrom: "other@remote.example",
328 MailFromLocalpart: smtp.Localpart("other"),
329 MailFromDomain: "remote.example",
330 RcptToLocalpart: "☹",
331 RcptToDomain: "☺.example",
332 MsgFromLocalpart: "other",
333 MsgFromDomain: "remote.example",
334 MsgFromOrgDomain: "remote.example",
335 EHLOValidated: true,
336 MailFromValidated: true,
337 MsgFromValidated: true,
338 EHLOValidation: store.ValidationStrict,
339 MailFromValidation: store.ValidationPass,
340 MsgFromValidation: store.ValidationStrict,
341 DKIMDomains: []string{"other.example"},
342 Size: int64(len(msg0)),
343 }
344 mf0 := tempfile()
345 xcheckf(err, "creating temp file for delivery")
346 _, err = fmt.Fprint(mf0, msg0)
347 xcheckf(err, "writing deliver message to file")
348 err = accTest2.DeliverMessage(c.log, tx, &m0, mf0, false, false, false, true)
349 xcheckf(err, "add message to account test2")
350
351 mf0name := mf0.Name()
352 err = mf0.Close()
353 xcheckf(err, "closing file")
354 err = os.Remove(mf0name)
355 xcheckf(err, "removing temp message file")
356
357 err = tx.Get(&inbox)
358 xcheckf(err, "get inbox")
359 inbox.Add(m0.MailboxCounts())
360 err = tx.Update(&inbox)
361 xcheckf(err, "update inbox")
362
363 sent, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Sent"}).Get()
364 xcheckf(err, "looking up inbox")
365 const prefix1 = "Extra: test\r\n"
366 const msg1 = "From: <other@remote.example>\r\nTo: <☹@xn--74h.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
367 m1 := store.Message{
368 MailboxID: sent.ID,
369 MailboxOrigID: sent.ID,
370 MailboxDestinedID: sent.ID,
371 Flags: store.Flags{Seen: true, Junk: true},
372 Size: int64(len(prefix1) + len(msg1)),
373 MsgPrefix: []byte(prefix1),
374 }
375 mf1 := tempfile()
376 xcheckf(err, "creating temp file for delivery")
377 _, err = fmt.Fprint(mf1, msg1)
378 xcheckf(err, "writing deliver message to file")
379 err = accTest2.DeliverMessage(c.log, tx, &m1, mf1, false, false, false, true)
380 xcheckf(err, "add message to account test2")
381
382 mf1name := mf1.Name()
383 err = mf1.Close()
384 xcheckf(err, "closing file")
385 err = os.Remove(mf1name)
386 xcheckf(err, "removing temp message file")
387
388 err = tx.Get(&sent)
389 xcheckf(err, "get sent")
390 sent.Add(m1.MailboxCounts())
391 err = tx.Update(&sent)
392 xcheckf(err, "update sent")
393
394 return nil
395 })
396 xcheckf(err, "write transaction with new message")
397 err = accTest2.Close()
398 xcheckf(err, "close account")
399}
400