Replace reply with a forked version to fix the cut-off of the incoming mail text (#3747)

replace reply with forgejos forked version

If plain text is selected as the message format in e.g. Apple Mail, the inline attachments are no longer at the end of the mail, but instead directly where they are in the mail. When parsing the mail, these inline attachments are replaced by "--". The new reply version no longer cuts the text at the first "--".

Tests for this are present in reply (7dc5750c6d).

Fixes https://codeberg.org/forgejo/forgejo/issues/3496#issuecomment-1798416

---

Additionally, I reduced the allocations for the inline attachments.

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/3747
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
Co-authored-by: Beowulf <beowulf@beocode.eu>
Co-committed-by: Beowulf <beowulf@beocode.eu>

go.mod

@@ -3,6 +3,7 @@ module code.gitea.io/gitea
 go 1.22.3
 
 require (
+	code.forgejo.org/forgejo/reply v1.0.1
 	code.gitea.io/actions-proto-go v0.4.0
 	code.gitea.io/gitea-vet v0.2.3
 	code.gitea.io/sdk/gitea v0.17.1
@@ -23,7 +24,6 @@ require (
 	github.com/buildkite/terminal-to-html/v3 v3.10.1
 	github.com/caddyserver/certmagic v0.21.0
 	github.com/chi-middleware/proxy v1.1.1
-	github.com/dimiro1/reply v0.0.0-20200315094148-d0136a4c9e21
 	github.com/djherbis/buffer v1.2.0
 	github.com/djherbis/nio/v3 v3.0.1
 	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5

go.sum

@@ -35,6 +35,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+code.forgejo.org/forgejo/reply v1.0.1 h1:usZi5yx7/g0D+xtGPJEM6mCvoDNdWvmtJu5J9/B/KBI=
+code.forgejo.org/forgejo/reply v1.0.1/go.mod h1:RyZUfzQLc+fuLIGjTSQWDAJWPiL4WtKXB/FifT5fM7U=
 code.gitea.io/actions-proto-go v0.4.0 h1:OsPBPhodXuQnsspG1sQ4eRE1PeoZyofd7+i73zCwnsU=
 code.gitea.io/actions-proto-go v0.4.0/go.mod h1:mn7Wkqz6JbnTOHQpot3yDeHx+O5C9EGhMEE+htvHBas=
 code.gitea.io/gitea-vet v0.2.3 h1:gdFmm6WOTM65rE8FUBTRzeQZYzXePKSSB1+r574hWwI=
@@ -226,8 +228,6 @@ github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+
 github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
-github.com/dimiro1/reply v0.0.0-20200315094148-d0136a4c9e21 h1:PdsjTl0Cg+ZJgOx/CFV5NNgO1ThTreqdgKYiDCMHJwA=
-github.com/dimiro1/reply v0.0.0-20200315094148-d0136a4c9e21/go.mod h1:xJvkyD6Y2rZapGvPJLYo9dyx1s5dxBEDPa8T3YTuOk0=
 github.com/djherbis/buffer v1.1.0/go.mod h1:VwN8VdFkMY0DCALdY8o00d3IZ6Amz/UNVMWcSaJT44o=
 github.com/djherbis/buffer v1.2.0 h1:PH5Dd2ss0C7CRRhQCZ2u7MssF+No9ide8Ye71nPHcrQ=
 github.com/djherbis/buffer v1.2.0/go.mod h1:fjnebbZjCUpPinBRD+TDwXSOeNQ7fPQWLfGQqiAiUyE=

services/mailer/incoming/incoming.go

@@ -17,7 +17,7 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/services/mailer/token"
 
-	"github.com/dimiro1/reply"
+	"code.forgejo.org/forgejo/reply"
 	"github.com/emersion/go-imap"
 	"github.com/emersion/go-imap/client"
 	"github.com/jhillyerd/enmime"
@@ -377,9 +377,10 @@ func getContentFromMailReader(env *enmime.Envelope) *MailContent {
 			Content: attachment.Content,
 		})
 	}
+	inlineAttachments := make([]*Attachment, 0, len(env.Inlines))
 	for _, inline := range env.Inlines {
-		if inline.FileName != "" {
+		if inline.FileName != "" && inline.ContentType != "text/plain" {
-			attachments = append(attachments, &Attachment{
+			inlineAttachments = append(inlineAttachments, &Attachment{
 				Name:    inline.FileName,
 				Content: inline.Content,
 			})
@@ -388,6 +389,6 @@ func getContentFromMailReader(env *enmime.Envelope) *MailContent {
 
 	return &MailContent{
 		Content:     reply.FromText(env.Text),
-		Attachments: attachments,
+		Attachments: append(attachments, inlineAttachments...),
 	}
 }

services/mailer/incoming/incoming_test.go

@@ -125,15 +125,27 @@ func TestGetContentFromMailReader(t *testing.T) {
 		"Content-Disposition: inline; filename=attachment.txt\r\n" +
 		"\r\n" +
 		"attachment content\r\n" +
+		"--message-boundary\r\n" +
+		"Content-Type: text/html\r\n" +
+		"Content-Disposition: inline; filename=attachment.html\r\n" +
+		"\r\n" +
+		"<p>html attachment content</p>\r\n" +
+		"--message-boundary\r\n" +
+		"Content-Type: image/png\r\n" +
+		"Content-Disposition: inline; filename=attachment.png\r\n" +
+		"Content-Transfer-Encoding: base64\r\n" +
+		"\r\n" +
+		"iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII\r\n" +
 		"--message-boundary--\r\n"
 
 	env, err = enmime.ReadEnvelope(strings.NewReader(mailString))
 	assert.NoError(t, err)
 	content = getContentFromMailReader(env)
-	assert.Equal(t, "mail content", content.Content)
+	assert.Equal(t, "mail content\n--\nattachment content", content.Content)
-	assert.Len(t, content.Attachments, 1)
+	assert.Len(t, content.Attachments, 2)
-	assert.Equal(t, "attachment.txt", content.Attachments[0].Name)
+	assert.Equal(t, "attachment.html", content.Attachments[0].Name)
-	assert.Equal(t, []byte("attachment content"), content.Attachments[0].Content)
+	assert.Equal(t, []byte("<p>html attachment content</p>"), content.Attachments[0].Content)
+	assert.Equal(t, "attachment.png", content.Attachments[1].Name)
 
 	mailString = "Content-Type: multipart/mixed; boundary=message-boundary\r\n" +
 		"\r\n" +
@@ -164,7 +176,7 @@ func TestGetContentFromMailReader(t *testing.T) {
 		"Content-Disposition: inline\r\n" +
 		"\r\n" +
 		"mail content without signature\r\n" +
-		"--\r\n" +
+		"----\r\n" +
 		"signature\r\n" +
 		"--text-boundary--\r\n" +
 		"--message-boundary--\r\n"