1// Package webops implements shared functionality between webapisrv and webmail.

2package webops

3

4import (

5 "context"

6 "errors"

7 "fmt"

8 "io"

9 "os"

10 "sort"

11

12 "golang.org/x/exp/maps"

13

14 "github.com/mjl-/bstore"

15

16 "github.com/mjl-/mox/message"

17 "github.com/mjl-/mox/mlog"

18 "github.com/mjl-/mox/store"

19)

20

21var ErrMessageNotFound = errors.New("no such message")

22

23type XOps struct {

24 DBWrite func(ctx context.Context, acc *store.Account, fn func(tx *bstore.Tx))

25 Checkf func(ctx context.Context, err error, format string, args ...any)

26 Checkuserf func(ctx context.Context, err error, format string, args ...any)

27}

28

29func (x XOps) mailboxID(ctx context.Context, tx *bstore.Tx, mailboxID int64) store.Mailbox {

30 if mailboxID == 0 {

31 x.Checkuserf(ctx, errors.New("invalid zero mailbox ID"), "getting mailbox")

32 }

33 mb := store.Mailbox{ID: mailboxID}

34 err := tx.Get(&mb)

35 if err == bstore.ErrAbsent {

36 x.Checkuserf(ctx, err, "getting mailbox")

37 }

38 x.Checkf(ctx, err, "getting mailbox")

39 return mb

40}

41

42// messageID returns a non-expunged message or panics with a sherpa error.

43func (x XOps) messageID(ctx context.Context, tx *bstore.Tx, messageID int64) store.Message {

44 if messageID == 0 {

45 x.Checkuserf(ctx, errors.New("invalid zero message id"), "getting message")

46 }

47 m := store.Message{ID: messageID}

48 err := tx.Get(&m)

49 if err == bstore.ErrAbsent {

50 x.Checkuserf(ctx, ErrMessageNotFound, "getting message")

51 } else if err == nil && m.Expunged {

52 x.Checkuserf(ctx, errors.New("message was removed"), "getting message")

53 }

54 x.Checkf(ctx, err, "getting message")

55 return m

56}

57

58func (x XOps) MessageDelete(ctx context.Context, log mlog.Log, acc *store.Account, messageIDs []int64) {

59 acc.WithWLock(func() {

60 var changes []store.Change

61

62 x.DBWrite(ctx, acc, func(tx *bstore.Tx) {

63 _, changes = x.MessageDeleteTx(ctx, log, tx, acc, messageIDs, 0)

64 })

65

66 store.BroadcastChanges(acc, changes)

67 })

68

69 for _, mID := range messageIDs {

70 p := acc.MessagePath(mID)

71 err := os.Remove(p)

72 log.Check(err, "removing message file for expunge")

73 }

74}

75

76func (x XOps) MessageDeleteTx(ctx context.Context, log mlog.Log, tx *bstore.Tx, acc *store.Account, messageIDs []int64, modseq store.ModSeq) (store.ModSeq, []store.Change) {

77 removeChanges := map[int64]store.ChangeRemoveUIDs{}

78 changes := make([]store.Change, 0, len(messageIDs)+1) // n remove, 1 mailbox counts

79

80 var mb store.Mailbox

81 remove := make([]store.Message, 0, len(messageIDs))

82

83 var totalSize int64

84 for _, mid := range messageIDs {

85 m := x.messageID(ctx, tx, mid)

86 totalSize += m.Size

87

88 if m.MailboxID != mb.ID {

89 if mb.ID != 0 {

90 err := tx.Update(&mb)

91 x.Checkf(ctx, err, "updating mailbox counts")

92 changes = append(changes, mb.ChangeCounts())

93 }

94 mb = x.mailboxID(ctx, tx, m.MailboxID)

95 }

96

97 qmr := bstore.QueryTx[store.Recipient](tx)

98 qmr.FilterEqual("MessageID", m.ID)

99 _, err := qmr.Delete()

100 x.Checkf(ctx, err, "removing message recipients")

101

102 mb.Sub(m.MailboxCounts())

103

104 if modseq == 0 {

105 modseq, err = acc.NextModSeq(tx)

106 x.Checkf(ctx, err, "assigning next modseq")

107 }

108 m.Expunged = true

109 m.ModSeq = modseq

110 err = tx.Update(&m)

111 x.Checkf(ctx, err, "marking message as expunged")

112

113 ch := removeChanges[m.MailboxID]

114 ch.UIDs = append(ch.UIDs, m.UID)

115 ch.MailboxID = m.MailboxID

116 ch.ModSeq = modseq

117 removeChanges[m.MailboxID] = ch

118 remove = append(remove, m)

119 }

120

121 if mb.ID != 0 {

122 err := tx.Update(&mb)

123 x.Checkf(ctx, err, "updating count in mailbox")

124 changes = append(changes, mb.ChangeCounts())

125 }

126

127 err := acc.AddMessageSize(log, tx, -totalSize)

128 x.Checkf(ctx, err, "updating disk usage")

129

130 // Mark removed messages as not needing training, then retrain them, so if they

131 // were trained, they get untrained.

132 for i := range remove {

133 remove[i].Junk = false

134 remove[i].Notjunk = false

135 }

136 err = acc.RetrainMessages(ctx, log, tx, remove, true)

137 x.Checkf(ctx, err, "untraining deleted messages")

138

139 for _, ch := range removeChanges {

140 sort.Slice(ch.UIDs, func(i, j int) bool {

141 return ch.UIDs[i] < ch.UIDs[j]

142 })

143 changes = append(changes, ch)

144 }

145

146 return modseq, changes

147}

148

149func (x XOps) MessageFlagsAdd(ctx context.Context, log mlog.Log, acc *store.Account, messageIDs []int64, flaglist []string) {

150 flags, keywords, err := store.ParseFlagsKeywords(flaglist)

151 x.Checkuserf(ctx, err, "parsing flags")

152

153 acc.WithRLock(func() {

154 var changes []store.Change

155

156 x.DBWrite(ctx, acc, func(tx *bstore.Tx) {

157 var modseq store.ModSeq

158 var retrain []store.Message

159 var mb, origmb store.Mailbox

160

161 for _, mid := range messageIDs {

162 m := x.messageID(ctx, tx, mid)

163

164 if mb.ID != m.MailboxID {

165 if mb.ID != 0 {

166 err := tx.Update(&mb)

167 x.Checkf(ctx, err, "updating mailbox")

168 if mb.MailboxCounts != origmb.MailboxCounts {

169 changes = append(changes, mb.ChangeCounts())

170 }

171 if mb.KeywordsChanged(origmb) {

172 changes = append(changes, mb.ChangeKeywords())

173 }

174 }

175 mb = x.mailboxID(ctx, tx, m.MailboxID)

176 origmb = mb

177 }

178 mb.Keywords, _ = store.MergeKeywords(mb.Keywords, keywords)

179

180 mb.Sub(m.MailboxCounts())

181 oflags := m.Flags

182 m.Flags = m.Flags.Set(flags, flags)

183 var kwChanged bool

184 m.Keywords, kwChanged = store.MergeKeywords(m.Keywords, keywords)

185 mb.Add(m.MailboxCounts())

186

187 if m.Flags == oflags && !kwChanged {

188 continue

189 }

190

191 if modseq == 0 {

192 modseq, err = acc.NextModSeq(tx)

193 x.Checkf(ctx, err, "assigning next modseq")

194 }

195 m.ModSeq = modseq

196 err = tx.Update(&m)

197 x.Checkf(ctx, err, "updating message")

198

199 changes = append(changes, m.ChangeFlags(oflags))

200 retrain = append(retrain, m)

201 }

202

203 if mb.ID != 0 {

204 err := tx.Update(&mb)

205 x.Checkf(ctx, err, "updating mailbox")

206 if mb.MailboxCounts != origmb.MailboxCounts {

207 changes = append(changes, mb.ChangeCounts())

208 }

209 if mb.KeywordsChanged(origmb) {

210 changes = append(changes, mb.ChangeKeywords())

211 }

212 }

213

214 err = acc.RetrainMessages(ctx, log, tx, retrain, false)

215 x.Checkf(ctx, err, "retraining messages")

216 })

217

218 store.BroadcastChanges(acc, changes)

219 })

220}

221

222func (x XOps) MessageFlagsClear(ctx context.Context, log mlog.Log, acc *store.Account, messageIDs []int64, flaglist []string) {

223 flags, keywords, err := store.ParseFlagsKeywords(flaglist)

224 x.Checkuserf(ctx, err, "parsing flags")

225

226 acc.WithRLock(func() {

227 var retrain []store.Message

228 var changes []store.Change

229

230 x.DBWrite(ctx, acc, func(tx *bstore.Tx) {

231 var modseq store.ModSeq

232 var mb, origmb store.Mailbox

233

234 for _, mid := range messageIDs {

235 m := x.messageID(ctx, tx, mid)

236

237 if mb.ID != m.MailboxID {

238 if mb.ID != 0 {

239 err := tx.Update(&mb)

240 x.Checkf(ctx, err, "updating counts for mailbox")

241 if mb.MailboxCounts != origmb.MailboxCounts {

242 changes = append(changes, mb.ChangeCounts())

243 }

244 // note: cannot remove keywords from mailbox by removing keywords from message.

245 }

246 mb = x.mailboxID(ctx, tx, m.MailboxID)

247 origmb = mb

248 }

249

250 oflags := m.Flags

251 mb.Sub(m.MailboxCounts())

252 m.Flags = m.Flags.Set(flags, store.Flags{})

253 var changed bool

254 m.Keywords, changed = store.RemoveKeywords(m.Keywords, keywords)

255 mb.Add(m.MailboxCounts())

256

257 if m.Flags == oflags && !changed {

258 continue

259 }

260

261 if modseq == 0 {

262 modseq, err = acc.NextModSeq(tx)

263 x.Checkf(ctx, err, "assigning next modseq")

264 }

265 m.ModSeq = modseq

266 err = tx.Update(&m)

267 x.Checkf(ctx, err, "updating message")

268

269 changes = append(changes, m.ChangeFlags(oflags))

270 retrain = append(retrain, m)

271 }

272

273 if mb.ID != 0 {

274 err := tx.Update(&mb)

275 x.Checkf(ctx, err, "updating keywords in mailbox")

276 if mb.MailboxCounts != origmb.MailboxCounts {

277 changes = append(changes, mb.ChangeCounts())

278 }

279 // note: cannot remove keywords from mailbox by removing keywords from message.

280 }

281

282 err = acc.RetrainMessages(ctx, log, tx, retrain, false)

283 x.Checkf(ctx, err, "retraining messages")

284 })

285

286 store.BroadcastChanges(acc, changes)

287 })

288}

289

290// MessageMove moves messages to the mailbox represented by mailboxName, or to mailboxID if mailboxName is empty.

291func (x XOps) MessageMove(ctx context.Context, log mlog.Log, acc *store.Account, messageIDs []int64, mailboxName string, mailboxID int64) {

292 acc.WithRLock(func() {

293 var changes []store.Change

294

295 x.DBWrite(ctx, acc, func(tx *bstore.Tx) {

296 if mailboxName != "" {

297 mb, err := acc.MailboxFind(tx, mailboxName)

298 x.Checkf(ctx, err, "looking up mailbox name")

299 if mb == nil {

300 x.Checkuserf(ctx, errors.New("not found"), "looking up mailbox name")

301 } else {

302 mailboxID = mb.ID

303 }

304 }

305

306 mbDst := x.mailboxID(ctx, tx, mailboxID)

307

308 if len(messageIDs) == 0 {

309 return

310 }

311

312 _, changes = x.MessageMoveTx(ctx, log, acc, tx, messageIDs, mbDst, 0)

313 })

314

315 store.BroadcastChanges(acc, changes)

316 })

317}

318

319func (x XOps) MessageMoveTx(ctx context.Context, log mlog.Log, acc *store.Account, tx *bstore.Tx, messageIDs []int64, mbDst store.Mailbox, modseq store.ModSeq) (store.ModSeq, []store.Change) {

320 retrain := make([]store.Message, 0, len(messageIDs))

321 removeChanges := map[int64]store.ChangeRemoveUIDs{}

322 // n adds, 1 remove, 2 mailboxcounts, optimistic and at least for a single message.

323 changes := make([]store.Change, 0, len(messageIDs)+3)

324

325 var mbSrc store.Mailbox

326

327 keywords := map[string]struct{}{}

328

329 for _, mid := range messageIDs {

330 m := x.messageID(ctx, tx, mid)

331

332 // We may have loaded this mailbox in the previous iteration of this loop.

333 if m.MailboxID != mbSrc.ID {

334 if mbSrc.ID != 0 {

335 err := tx.Update(&mbSrc)

336 x.Checkf(ctx, err, "updating source mailbox counts")

337 changes = append(changes, mbSrc.ChangeCounts())

338 }

339 mbSrc = x.mailboxID(ctx, tx, m.MailboxID)

340 }

341

342 if mbSrc.ID == mbDst.ID {

343 // Client should filter out messages that are already in mailbox.

344 x.Checkuserf(ctx, errors.New("already in destination mailbox"), "moving message")

345 }

346

347 var err error

348 if modseq == 0 {

349 modseq, err = acc.NextModSeq(tx)

350 x.Checkf(ctx, err, "assigning next modseq")

351 }

352

353 ch := removeChanges[m.MailboxID]

354 ch.UIDs = append(ch.UIDs, m.UID)

355 ch.ModSeq = modseq

356 ch.MailboxID = m.MailboxID

357 removeChanges[m.MailboxID] = ch

358

359 // Copy of message record that we'll insert when UID is freed up.

360 om := m

361 om.PrepareExpunge()

362 om.ID = 0 // Assign new ID.

363 om.ModSeq = modseq

364

365 mbSrc.Sub(m.MailboxCounts())

366

367 if mbDst.Trash {

368 m.Seen = true

369 }

370 conf, _ := acc.Conf()

371 m.MailboxID = mbDst.ID

372 if m.IsReject && m.MailboxDestinedID != 0 {

373 // Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message

374 // is used for reputation calculation during future deliveries.

375 m.MailboxOrigID = m.MailboxDestinedID

376 m.IsReject = false

377 m.Seen = false

378 }

379 m.UID = mbDst.UIDNext

380 m.ModSeq = modseq

381 mbDst.UIDNext++

382 m.JunkFlagsForMailbox(mbDst, conf)

383 err = tx.Update(&m)

384 x.Checkf(ctx, err, "updating moved message in database")

385

386 // Now that UID is unused, we can insert the old record again.

387 err = tx.Insert(&om)

388 x.Checkf(ctx, err, "inserting record for expunge after moving message")

389

390 mbDst.Add(m.MailboxCounts())

391

392 changes = append(changes, m.ChangeAddUID())

393 retrain = append(retrain, m)

394

395 for _, kw := range m.Keywords {

396 keywords[kw] = struct{}{}

397 }

398 }

399

400 err := tx.Update(&mbSrc)

401 x.Checkf(ctx, err, "updating source mailbox counts")

402

403 changes = append(changes, mbSrc.ChangeCounts(), mbDst.ChangeCounts())

404

405 // Ensure destination mailbox has keywords of the moved messages.

406 var mbKwChanged bool

407 mbDst.Keywords, mbKwChanged = store.MergeKeywords(mbDst.Keywords, maps.Keys(keywords))

408 if mbKwChanged {

409 changes = append(changes, mbDst.ChangeKeywords())

410 }

411

412 err = tx.Update(&mbDst)

413 x.Checkf(ctx, err, "updating mailbox with uidnext")

414

415 err = acc.RetrainMessages(ctx, log, tx, retrain, false)

416 x.Checkf(ctx, err, "retraining messages after move")

417

418 // Ensure UIDs of the removed message are in increasing order. It is quite common

419 // for all messages to be from a single source mailbox, meaning this is just one

420 // change, for which we preallocated space.

421 for _, ch := range removeChanges {

422 sort.Slice(ch.UIDs, func(i, j int) bool {

423 return ch.UIDs[i] < ch.UIDs[j]

424 })

425 changes = append(changes, ch)

426 }

427

428 return modseq, changes

429}

430

431func isText(p message.Part) bool {

432 return p.MediaType == "" && p.MediaSubType == "" || p.MediaType == "TEXT" && p.MediaSubType == "PLAIN"

433}

434

435func isHTML(p message.Part) bool {

436 return p.MediaType == "" && p.MediaSubType == "" || p.MediaType == "TEXT" && p.MediaSubType == "HTML"

437}

438

439func isAlternative(p message.Part) bool {

440 return p.MediaType == "MULTIPART" && p.MediaSubType == "ALTERNATIVE"

441}

442

443func readPart(p message.Part, maxSize int64) (string, error) {

444 buf, err := io.ReadAll(io.LimitReader(p.ReaderUTF8OrBinary(), maxSize))

445 if err != nil {

446 return "", fmt.Errorf("reading part contents: %v", err)

447 }

448 return string(buf), nil

449}

450

451// ReadableParts returns the contents of the first text and/or html parts,

452// descending into multiparts, truncated to maxSize bytes if longer.

453func ReadableParts(p message.Part, maxSize int64) (text string, html string, found bool, err error) {

454 // todo: may want to merge this logic with webmail's message parsing.

455

456 // For non-multipart messages, top-level part.

457 if isText(p) {

458 data, err := readPart(p, maxSize)

459 return data, "", true, err

460 } else if isHTML(p) {

461 data, err := readPart(p, maxSize)

462 return "", data, true, err

463 }

464

465 // Look in sub-parts. Stop when we have a readable part, don't continue with other

466 // subparts unless we have a multipart/alternative.

467 // todo: we may have to look at disposition "inline".

468 var haveText, haveHTML bool

469 for _, pp := range p.Parts {

470 if isText(pp) {

471 haveText = true

472 text, err = readPart(pp, maxSize)

473 if !isAlternative(p) {

474 break

475 }

476 } else if isHTML(pp) {

477 haveHTML = true

478 html, err = readPart(pp, maxSize)

479 if !isAlternative(p) {

480 break

481 }

482 }

483 }

484 if haveText || haveHTML {

485 return text, html, true, err

486 }

487

488 // Descend into the subparts.

489 for _, pp := range p.Parts {

490 text, html, found, err = ReadableParts(pp, maxSize)

491 if found {

492 break

493 }

494 }

495 return

496}

497