2014-03-19 07:27:27 -05:00
|
|
|
// Copyright 2014 The Gogs Authors. All rights reserved.
|
2017-06-06 20:14:31 -05:00
|
|
|
// Copyright 2017 The Gitea Authors. All rights reserved.
|
2022-11-27 13:20:29 -05:00
|
|
|
// SPDX-License-Identifier: MIT
|
2014-03-19 07:27:27 -05:00
|
|
|
|
|
|
|
|
package mailer
|
|
|
|
|
|
|
|
|
|
import (
|
2019-02-02 21:06:52 -05:00
|
|
|
"bytes"
|
2022-08-28 04:43:25 -05:00
|
|
|
"context"
|
2014-10-09 17:08:07 -05:00
|
|
|
"crypto/tls"
|
2014-03-19 07:27:27 -05:00
|
|
|
"fmt"
|
2021-12-08 02:34:23 -05:00
|
|
|
"hash/fnv"
|
2015-09-17 00:54:12 -05:00
|
|
|
"io"
|
2014-10-09 17:08:07 -05:00
|
|
|
"net"
|
2014-03-19 07:27:27 -05:00
|
|
|
"net/smtp"
|
2015-02-19 02:47:05 -05:00
|
|
|
"os"
|
2016-12-25 08:55:22 -05:00
|
|
|
"os/exec"
|
2014-03-19 07:27:27 -05:00
|
|
|
"strings"
|
2015-09-17 00:54:12 -05:00
|
|
|
"time"
|
|
|
|
|
|
2017-09-11 01:33:28 -05:00
|
|
|
"code.gitea.io/gitea/modules/base"
|
2020-01-16 12:55:36 -05:00
|
|
|
"code.gitea.io/gitea/modules/graceful"
|
2016-11-10 11:24:48 -05:00
|
|
|
"code.gitea.io/gitea/modules/log"
|
2020-05-02 18:04:31 -05:00
|
|
|
"code.gitea.io/gitea/modules/process"
|
2020-01-16 12:55:36 -05:00
|
|
|
"code.gitea.io/gitea/modules/queue"
|
2016-11-10 11:24:48 -05:00
|
|
|
"code.gitea.io/gitea/modules/setting"
|
2022-08-08 13:04:28 -05:00
|
|
|
"code.gitea.io/gitea/modules/templates"
|
2023-09-05 13:37:47 -05:00
|
|
|
notify_service "code.gitea.io/gitea/services/notify"
|
2017-06-06 20:14:31 -05:00
|
|
|
|
2023-05-02 16:40:46 -05:00
|
|
|
ntlmssp "github.com/Azure/go-ntlmssp"
|
2017-06-06 20:14:31 -05:00
|
|
|
"github.com/jaytaylor/html2text"
|
|
|
|
|
"gopkg.in/gomail.v2"
|
2014-03-19 07:27:27 -05:00
|
|
|
)
|
|
|
|
|
|
2016-11-24 20:44:04 -05:00
|
|
|
// Message mail body and log info
|
2015-09-17 00:54:12 -05:00
|
|
|
type Message struct {
|
2020-01-16 12:55:36 -05:00
|
|
|
Info string // Message information for log purpose.
|
|
|
|
|
FromAddress string
|
|
|
|
|
FromDisplayName string
|
2023-01-22 09:23:52 -05:00
|
|
|
To string // Use only one recipient to prevent leaking of addresses
|
2023-01-14 10:57:10 -05:00
|
|
|
ReplyTo string
|
2020-01-16 12:55:36 -05:00
|
|
|
Subject string
|
|
|
|
|
Date time.Time
|
|
|
|
|
Body string
|
|
|
|
|
Headers map[string][]string
|
2015-09-17 00:54:12 -05:00
|
|
|
}
|
|
|
|
|
|
2020-01-16 12:55:36 -05:00
|
|
|
// ToMessage converts a Message to gomail.Message
|
|
|
|
|
func (m *Message) ToMessage() *gomail.Message {
|
2015-09-17 00:54:12 -05:00
|
|
|
msg := gomail.NewMessage()
|
2020-01-16 12:55:36 -05:00
|
|
|
msg.SetAddressHeader("From", m.FromAddress, m.FromDisplayName)
|
2023-01-22 09:23:52 -05:00
|
|
|
msg.SetHeader("To", m.To)
|
2023-01-14 10:57:10 -05:00
|
|
|
if m.ReplyTo != "" {
|
|
|
|
|
msg.SetHeader("Reply-To", m.ReplyTo)
|
|
|
|
|
}
|
2020-01-16 12:55:36 -05:00
|
|
|
for header := range m.Headers {
|
|
|
|
|
msg.SetHeader(header, m.Headers[header]...)
|
|
|
|
|
}
|
|
|
|
|
|
2019-04-16 23:56:40 -05:00
|
|
|
if len(setting.MailService.SubjectPrefix) > 0 {
|
2020-01-16 12:55:36 -05:00
|
|
|
msg.SetHeader("Subject", setting.MailService.SubjectPrefix+" "+m.Subject)
|
2019-04-16 23:56:40 -05:00
|
|
|
} else {
|
2020-01-16 12:55:36 -05:00
|
|
|
msg.SetHeader("Subject", m.Subject)
|
2019-04-16 23:56:40 -05:00
|
|
|
}
|
2020-01-16 12:55:36 -05:00
|
|
|
msg.SetDateHeader("Date", m.Date)
|
2019-04-02 10:45:54 -05:00
|
|
|
msg.SetHeader("X-Auto-Response-Suppress", "All")
|
2016-05-30 03:32:01 -05:00
|
|
|
|
2020-01-16 12:55:36 -05:00
|
|
|
plainBody, err := html2text.FromString(m.Body)
|
2017-06-06 20:14:31 -05:00
|
|
|
if err != nil || setting.MailService.SendAsPlainText {
|
2020-01-16 12:55:36 -05:00
|
|
|
if strings.Contains(base.TruncateString(m.Body, 100), "<html>") {
|
2017-06-06 20:14:31 -05:00
|
|
|
log.Warn("Mail contains HTML but configured to send as plain text.")
|
2016-05-30 03:50:20 -05:00
|
|
|
}
|
2017-06-06 20:14:31 -05:00
|
|
|
msg.SetBody("text/plain", plainBody)
|
|
|
|
|
} else {
|
|
|
|
|
msg.SetBody("text/plain", plainBody)
|
2020-01-16 12:55:36 -05:00
|
|
|
msg.AddAlternative("text/html", m.Body)
|
2016-05-30 03:18:49 -05:00
|
|
|
}
|
2021-12-08 02:34:23 -05:00
|
|
|
|
|
|
|
|
if len(msg.GetHeader("Message-ID")) == 0 {
|
|
|
|
|
msg.SetHeader("Message-ID", m.generateAutoMessageID())
|
|
|
|
|
}
|
2020-01-16 12:55:36 -05:00
|
|
|
return msg
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// SetHeader adds additional headers to a message
|
|
|
|
|
func (m *Message) SetHeader(field string, value ...string) {
|
|
|
|
|
m.Headers[field] = value
|
|
|
|
|
}
|
|
|
|
|
|
2021-12-08 02:34:23 -05:00
|
|
|
func (m *Message) generateAutoMessageID() string {
|
|
|
|
|
dateMs := m.Date.UnixNano() / 1e6
|
|
|
|
|
h := fnv.New64()
|
|
|
|
|
if len(m.To) > 0 {
|
2023-01-22 09:23:52 -05:00
|
|
|
_, _ = h.Write([]byte(m.To))
|
2021-12-08 02:34:23 -05:00
|
|
|
}
|
|
|
|
|
_, _ = h.Write([]byte(m.Subject))
|
|
|
|
|
_, _ = h.Write([]byte(m.Body))
|
|
|
|
|
return fmt.Sprintf("<autogen-%d-%016x@%s>", dateMs, h.Sum64(), setting.Domain)
|
|
|
|
|
}
|
|
|
|
|
|
2020-01-16 12:55:36 -05:00
|
|
|
// NewMessageFrom creates new mail message object with custom From header.
|
2023-01-22 09:23:52 -05:00
|
|
|
func NewMessageFrom(to, fromDisplayName, fromAddress, subject, body string) *Message {
|
2020-01-16 12:55:36 -05:00
|
|
|
log.Trace("NewMessageFrom (body):\n%s", body)
|
2015-09-17 00:54:12 -05:00
|
|
|
|
|
|
|
|
return &Message{
|
2020-01-16 12:55:36 -05:00
|
|
|
FromAddress: fromAddress,
|
|
|
|
|
FromDisplayName: fromDisplayName,
|
|
|
|
|
To: to,
|
|
|
|
|
Subject: subject,
|
|
|
|
|
Date: time.Now(),
|
|
|
|
|
Body: body,
|
|
|
|
|
Headers: map[string][]string{},
|
2015-09-17 00:54:12 -05:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// NewMessage creates new mail message object with default From header.
|
2023-01-22 09:23:52 -05:00
|
|
|
func NewMessage(to, subject, body string) *Message {
|
2017-09-20 23:29:45 -05:00
|
|
|
return NewMessageFrom(to, setting.MailService.FromName, setting.MailService.FromEmail, subject, body)
|
2015-09-17 00:54:12 -05:00
|
|
|
}
|
|
|
|
|
|
2015-08-20 00:56:25 -05:00
|
|
|
type loginAuth struct {
|
2015-08-20 06:12:55 -05:00
|
|
|
username, password string
|
2015-08-20 00:56:25 -05:00
|
|
|
}
|
|
|
|
|
|
2016-11-24 20:44:04 -05:00
|
|
|
// LoginAuth SMTP AUTH LOGIN Auth Handler
|
2015-08-20 00:56:25 -05:00
|
|
|
func LoginAuth(username, password string) smtp.Auth {
|
|
|
|
|
return &loginAuth{username, password}
|
|
|
|
|
}
|
|
|
|
|
|
2016-11-24 20:44:04 -05:00
|
|
|
// Start start SMTP login auth
|
2015-08-20 00:56:25 -05:00
|
|
|
func (a *loginAuth) Start(server *smtp.ServerInfo) (string, []byte, error) {
|
|
|
|
|
return "LOGIN", []byte{}, nil
|
|
|
|
|
}
|
|
|
|
|
|
2016-11-24 20:44:04 -05:00
|
|
|
// Next next step of SMTP login auth
|
2015-08-20 00:56:25 -05:00
|
|
|
func (a *loginAuth) Next(fromServer []byte, more bool) ([]byte, error) {
|
|
|
|
|
if more {
|
|
|
|
|
switch string(fromServer) {
|
|
|
|
|
case "Username:":
|
|
|
|
|
return []byte(a.username), nil
|
|
|
|
|
case "Password:":
|
|
|
|
|
return []byte(a.password), nil
|
|
|
|
|
default:
|
2017-06-17 19:30:04 -05:00
|
|
|
return nil, fmt.Errorf("unknown fromServer: %s", string(fromServer))
|
2015-08-20 00:56:25 -05:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return nil, nil
|
|
|
|
|
}
|
|
|
|
|
|
2023-05-02 16:40:46 -05:00
|
|
|
type ntlmAuth struct {
|
|
|
|
|
username, password, domain string
|
|
|
|
|
domainNeeded bool
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// NtlmAuth SMTP AUTH NTLM Auth Handler
|
|
|
|
|
func NtlmAuth(username, password string) smtp.Auth {
|
|
|
|
|
user, domain, domainNeeded := ntlmssp.GetDomain(username)
|
|
|
|
|
return &ntlmAuth{user, password, domain, domainNeeded}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Start starts SMTP NTLM Auth
|
|
|
|
|
func (a *ntlmAuth) Start(server *smtp.ServerInfo) (string, []byte, error) {
|
|
|
|
|
negotiateMessage, err := ntlmssp.NewNegotiateMessage(a.domain, "")
|
|
|
|
|
return "NTLM", negotiateMessage, err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Next next step of SMTP ntlm auth
|
|
|
|
|
func (a *ntlmAuth) Next(fromServer []byte, more bool) ([]byte, error) {
|
|
|
|
|
if more {
|
|
|
|
|
if len(fromServer) == 0 {
|
|
|
|
|
return nil, fmt.Errorf("ntlm ChallengeMessage is empty")
|
|
|
|
|
}
|
|
|
|
|
authenticateMessage, err := ntlmssp.ProcessChallenge(fromServer, a.username, a.password, a.domainNeeded)
|
|
|
|
|
return authenticateMessage, err
|
|
|
|
|
}
|
|
|
|
|
return nil, nil
|
|
|
|
|
}
|
|
|
|
|
|
2016-12-25 08:55:22 -05:00
|
|
|
// Sender SMTP mail sender
|
2022-01-20 12:46:10 -05:00
|
|
|
type smtpSender struct{}
|
2014-03-19 07:27:27 -05:00
|
|
|
|
2016-11-24 20:44:04 -05:00
|
|
|
// Send send email
|
2016-12-25 08:55:22 -05:00
|
|
|
func (s *smtpSender) Send(from string, to []string, msg io.WriterTo) error {
|
2015-09-17 00:54:12 -05:00
|
|
|
opts := setting.MailService
|
2014-03-19 20:05:48 -05:00
|
|
|
|
Rework mailer settings (#18982)
* `PROTOCOL`: can be smtp, smtps, smtp+startls, smtp+unix, sendmail, dummy
* `SMTP_ADDR`: domain for SMTP, or path to unix socket
* `SMTP_PORT`: port for SMTP; defaults to 25 for `smtp`, 465 for `smtps`, and 587 for `smtp+startls`
* `ENABLE_HELO`, `HELO_HOSTNAME`: reverse `DISABLE_HELO` to `ENABLE_HELO`; default to false + system hostname
* `FORCE_TRUST_SERVER_CERT`: replace the unclear `SKIP_VERIFY`
* `CLIENT_CERT_FILE`, `CLIENT_KEY_FILE`, `USE_CLIENT_CERT`: clarify client certificates here
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2022-08-02 00:24:18 -05:00
|
|
|
var network string
|
|
|
|
|
var address string
|
|
|
|
|
if opts.Protocol == "smtp+unix" {
|
|
|
|
|
network = "unix"
|
|
|
|
|
address = opts.SMTPAddr
|
|
|
|
|
} else {
|
|
|
|
|
network = "tcp"
|
|
|
|
|
address = net.JoinHostPort(opts.SMTPAddr, opts.SMTPPort)
|
2014-10-09 17:08:07 -05:00
|
|
|
}
|
|
|
|
|
|
Rework mailer settings (#18982)
* `PROTOCOL`: can be smtp, smtps, smtp+startls, smtp+unix, sendmail, dummy
* `SMTP_ADDR`: domain for SMTP, or path to unix socket
* `SMTP_PORT`: port for SMTP; defaults to 25 for `smtp`, 465 for `smtps`, and 587 for `smtp+startls`
* `ENABLE_HELO`, `HELO_HOSTNAME`: reverse `DISABLE_HELO` to `ENABLE_HELO`; default to false + system hostname
* `FORCE_TRUST_SERVER_CERT`: replace the unclear `SKIP_VERIFY`
* `CLIENT_CERT_FILE`, `CLIENT_KEY_FILE`, `USE_CLIENT_CERT`: clarify client certificates here
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2022-08-02 00:24:18 -05:00
|
|
|
conn, err := net.Dial(network, address)
|
|
|
|
|
if err != nil {
|
2022-10-24 14:29:17 -05:00
|
|
|
return fmt.Errorf("failed to establish network connection to SMTP server: %w", err)
|
2014-10-09 17:08:07 -05:00
|
|
|
}
|
Rework mailer settings (#18982)
* `PROTOCOL`: can be smtp, smtps, smtp+startls, smtp+unix, sendmail, dummy
* `SMTP_ADDR`: domain for SMTP, or path to unix socket
* `SMTP_PORT`: port for SMTP; defaults to 25 for `smtp`, 465 for `smtps`, and 587 for `smtp+startls`
* `ENABLE_HELO`, `HELO_HOSTNAME`: reverse `DISABLE_HELO` to `ENABLE_HELO`; default to false + system hostname
* `FORCE_TRUST_SERVER_CERT`: replace the unclear `SKIP_VERIFY`
* `CLIENT_CERT_FILE`, `CLIENT_KEY_FILE`, `USE_CLIENT_CERT`: clarify client certificates here
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2022-08-02 00:24:18 -05:00
|
|
|
defer conn.Close()
|
2014-12-18 06:34:30 -05:00
|
|
|
|
Rework mailer settings (#18982)
* `PROTOCOL`: can be smtp, smtps, smtp+startls, smtp+unix, sendmail, dummy
* `SMTP_ADDR`: domain for SMTP, or path to unix socket
* `SMTP_PORT`: port for SMTP; defaults to 25 for `smtp`, 465 for `smtps`, and 587 for `smtp+startls`
* `ENABLE_HELO`, `HELO_HOSTNAME`: reverse `DISABLE_HELO` to `ENABLE_HELO`; default to false + system hostname
* `FORCE_TRUST_SERVER_CERT`: replace the unclear `SKIP_VERIFY`
* `CLIENT_CERT_FILE`, `CLIENT_KEY_FILE`, `USE_CLIENT_CERT`: clarify client certificates here
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2022-08-02 00:24:18 -05:00
|
|
|
var tlsconfig *tls.Config
|
2022-11-27 05:08:40 -05:00
|
|
|
if opts.Protocol == "smtps" || opts.Protocol == "smtp+starttls" {
|
Rework mailer settings (#18982)
* `PROTOCOL`: can be smtp, smtps, smtp+startls, smtp+unix, sendmail, dummy
* `SMTP_ADDR`: domain for SMTP, or path to unix socket
* `SMTP_PORT`: port for SMTP; defaults to 25 for `smtp`, 465 for `smtps`, and 587 for `smtp+startls`
* `ENABLE_HELO`, `HELO_HOSTNAME`: reverse `DISABLE_HELO` to `ENABLE_HELO`; default to false + system hostname
* `FORCE_TRUST_SERVER_CERT`: replace the unclear `SKIP_VERIFY`
* `CLIENT_CERT_FILE`, `CLIENT_KEY_FILE`, `USE_CLIENT_CERT`: clarify client certificates here
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2022-08-02 00:24:18 -05:00
|
|
|
tlsconfig = &tls.Config{
|
|
|
|
|
InsecureSkipVerify: opts.ForceTrustServerCert,
|
|
|
|
|
ServerName: opts.SMTPAddr,
|
2015-02-13 02:33:55 -05:00
|
|
|
}
|
2014-12-18 06:34:30 -05:00
|
|
|
|
Rework mailer settings (#18982)
* `PROTOCOL`: can be smtp, smtps, smtp+startls, smtp+unix, sendmail, dummy
* `SMTP_ADDR`: domain for SMTP, or path to unix socket
* `SMTP_PORT`: port for SMTP; defaults to 25 for `smtp`, 465 for `smtps`, and 587 for `smtp+startls`
* `ENABLE_HELO`, `HELO_HOSTNAME`: reverse `DISABLE_HELO` to `ENABLE_HELO`; default to false + system hostname
* `FORCE_TRUST_SERVER_CERT`: replace the unclear `SKIP_VERIFY`
* `CLIENT_CERT_FILE`, `CLIENT_KEY_FILE`, `USE_CLIENT_CERT`: clarify client certificates here
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2022-08-02 00:24:18 -05:00
|
|
|
if opts.UseClientCert {
|
|
|
|
|
cert, err := tls.LoadX509KeyPair(opts.ClientCertFile, opts.ClientKeyFile)
|
|
|
|
|
if err != nil {
|
2022-10-24 14:29:17 -05:00
|
|
|
return fmt.Errorf("could not load SMTP client certificate: %w", err)
|
Rework mailer settings (#18982)
* `PROTOCOL`: can be smtp, smtps, smtp+startls, smtp+unix, sendmail, dummy
* `SMTP_ADDR`: domain for SMTP, or path to unix socket
* `SMTP_PORT`: port for SMTP; defaults to 25 for `smtp`, 465 for `smtps`, and 587 for `smtp+startls`
* `ENABLE_HELO`, `HELO_HOSTNAME`: reverse `DISABLE_HELO` to `ENABLE_HELO`; default to false + system hostname
* `FORCE_TRUST_SERVER_CERT`: replace the unclear `SKIP_VERIFY`
* `CLIENT_CERT_FILE`, `CLIENT_KEY_FILE`, `USE_CLIENT_CERT`: clarify client certificates here
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2022-08-02 00:24:18 -05:00
|
|
|
}
|
|
|
|
|
tlsconfig.Certificates = []tls.Certificate{cert}
|
|
|
|
|
}
|
2014-12-18 06:34:30 -05:00
|
|
|
}
|
|
|
|
|
|
Rework mailer settings (#18982)
* `PROTOCOL`: can be smtp, smtps, smtp+startls, smtp+unix, sendmail, dummy
* `SMTP_ADDR`: domain for SMTP, or path to unix socket
* `SMTP_PORT`: port for SMTP; defaults to 25 for `smtp`, 465 for `smtps`, and 587 for `smtp+startls`
* `ENABLE_HELO`, `HELO_HOSTNAME`: reverse `DISABLE_HELO` to `ENABLE_HELO`; default to false + system hostname
* `FORCE_TRUST_SERVER_CERT`: replace the unclear `SKIP_VERIFY`
* `CLIENT_CERT_FILE`, `CLIENT_KEY_FILE`, `USE_CLIENT_CERT`: clarify client certificates here
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2022-08-02 00:24:18 -05:00
|
|
|
if opts.Protocol == "smtps" {
|
2014-12-18 06:34:30 -05:00
|
|
|
conn = tls.Client(conn, tlsconfig)
|
|
|
|
|
}
|
|
|
|
|
|
Rework mailer settings (#18982)
* `PROTOCOL`: can be smtp, smtps, smtp+startls, smtp+unix, sendmail, dummy
* `SMTP_ADDR`: domain for SMTP, or path to unix socket
* `SMTP_PORT`: port for SMTP; defaults to 25 for `smtp`, 465 for `smtps`, and 587 for `smtp+startls`
* `ENABLE_HELO`, `HELO_HOSTNAME`: reverse `DISABLE_HELO` to `ENABLE_HELO`; default to false + system hostname
* `FORCE_TRUST_SERVER_CERT`: replace the unclear `SKIP_VERIFY`
* `CLIENT_CERT_FILE`, `CLIENT_KEY_FILE`, `USE_CLIENT_CERT`: clarify client certificates here
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2022-08-02 00:24:18 -05:00
|
|
|
host := "localhost"
|
|
|
|
|
if opts.Protocol == "smtp+unix" {
|
|
|
|
|
host = opts.SMTPAddr
|
|
|
|
|
}
|
2014-12-19 00:24:17 -05:00
|
|
|
client, err := smtp.NewClient(conn, host)
|
|
|
|
|
if err != nil {
|
2022-10-24 14:29:17 -05:00
|
|
|
return fmt.Errorf("could not initiate SMTP session: %w", err)
|
2014-10-09 17:08:07 -05:00
|
|
|
}
|
|
|
|
|
|
Rework mailer settings (#18982)
* `PROTOCOL`: can be smtp, smtps, smtp+startls, smtp+unix, sendmail, dummy
* `SMTP_ADDR`: domain for SMTP, or path to unix socket
* `SMTP_PORT`: port for SMTP; defaults to 25 for `smtp`, 465 for `smtps`, and 587 for `smtp+startls`
* `ENABLE_HELO`, `HELO_HOSTNAME`: reverse `DISABLE_HELO` to `ENABLE_HELO`; default to false + system hostname
* `FORCE_TRUST_SERVER_CERT`: replace the unclear `SKIP_VERIFY`
* `CLIENT_CERT_FILE`, `CLIENT_KEY_FILE`, `USE_CLIENT_CERT`: clarify client certificates here
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2022-08-02 00:24:18 -05:00
|
|
|
if opts.EnableHelo {
|
2016-08-29 02:10:21 -05:00
|
|
|
hostname := opts.HeloHostname
|
2015-07-03 01:08:18 -05:00
|
|
|
if len(hostname) == 0 {
|
|
|
|
|
hostname, err = os.Hostname()
|
|
|
|
|
if err != nil {
|
2022-10-24 14:29:17 -05:00
|
|
|
return fmt.Errorf("could not retrieve system hostname: %w", err)
|
2015-07-03 01:08:18 -05:00
|
|
|
}
|
|
|
|
|
}
|
2015-02-20 02:12:27 -05:00
|
|
|
|
2015-07-03 01:08:18 -05:00
|
|
|
if err = client.Hello(hostname); err != nil {
|
2022-10-24 14:29:17 -05:00
|
|
|
return fmt.Errorf("failed to issue HELO command: %w", err)
|
2015-07-03 01:08:18 -05:00
|
|
|
}
|
2015-02-20 02:12:27 -05:00
|
|
|
}
|
2015-02-19 02:47:05 -05:00
|
|
|
|
2022-11-27 05:08:40 -05:00
|
|
|
if opts.Protocol == "smtp+starttls" {
|
Rework mailer settings (#18982)
* `PROTOCOL`: can be smtp, smtps, smtp+startls, smtp+unix, sendmail, dummy
* `SMTP_ADDR`: domain for SMTP, or path to unix socket
* `SMTP_PORT`: port for SMTP; defaults to 25 for `smtp`, 465 for `smtps`, and 587 for `smtp+startls`
* `ENABLE_HELO`, `HELO_HOSTNAME`: reverse `DISABLE_HELO` to `ENABLE_HELO`; default to false + system hostname
* `FORCE_TRUST_SERVER_CERT`: replace the unclear `SKIP_VERIFY`
* `CLIENT_CERT_FILE`, `CLIENT_KEY_FILE`, `USE_CLIENT_CERT`: clarify client certificates here
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2022-08-02 00:24:18 -05:00
|
|
|
hasStartTLS, _ := client.Extension("STARTTLS")
|
|
|
|
|
if hasStartTLS {
|
|
|
|
|
if err = client.StartTLS(tlsconfig); err != nil {
|
2022-10-24 14:29:17 -05:00
|
|
|
return fmt.Errorf("failed to start TLS connection: %w", err)
|
Rework mailer settings (#18982)
* `PROTOCOL`: can be smtp, smtps, smtp+startls, smtp+unix, sendmail, dummy
* `SMTP_ADDR`: domain for SMTP, or path to unix socket
* `SMTP_PORT`: port for SMTP; defaults to 25 for `smtp`, 465 for `smtps`, and 587 for `smtp+startls`
* `ENABLE_HELO`, `HELO_HOSTNAME`: reverse `DISABLE_HELO` to `ENABLE_HELO`; default to false + system hostname
* `FORCE_TRUST_SERVER_CERT`: replace the unclear `SKIP_VERIFY`
* `CLIENT_CERT_FILE`, `CLIENT_KEY_FILE`, `USE_CLIENT_CERT`: clarify client certificates here
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2022-08-02 00:24:18 -05:00
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
log.Warn("StartTLS requested, but SMTP server does not support it; falling back to regular SMTP")
|
2014-12-18 06:34:30 -05:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-12-19 00:24:17 -05:00
|
|
|
canAuth, options := client.Extension("AUTH")
|
Rework mailer settings (#18982)
* `PROTOCOL`: can be smtp, smtps, smtp+startls, smtp+unix, sendmail, dummy
* `SMTP_ADDR`: domain for SMTP, or path to unix socket
* `SMTP_PORT`: port for SMTP; defaults to 25 for `smtp`, 465 for `smtps`, and 587 for `smtp+startls`
* `ENABLE_HELO`, `HELO_HOSTNAME`: reverse `DISABLE_HELO` to `ENABLE_HELO`; default to false + system hostname
* `FORCE_TRUST_SERVER_CERT`: replace the unclear `SKIP_VERIFY`
* `CLIENT_CERT_FILE`, `CLIENT_KEY_FILE`, `USE_CLIENT_CERT`: clarify client certificates here
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2022-08-02 00:24:18 -05:00
|
|
|
if len(opts.User) > 0 {
|
|
|
|
|
if !canAuth {
|
|
|
|
|
return fmt.Errorf("SMTP server does not support AUTH, but credentials provided")
|
|
|
|
|
}
|
|
|
|
|
|
2014-12-18 06:58:18 -05:00
|
|
|
var auth smtp.Auth
|
|
|
|
|
|
2014-12-18 07:15:13 -05:00
|
|
|
if strings.Contains(options, "CRAM-MD5") {
|
2015-09-17 00:54:12 -05:00
|
|
|
auth = smtp.CRAMMD5Auth(opts.User, opts.Passwd)
|
2014-12-18 07:15:13 -05:00
|
|
|
} else if strings.Contains(options, "PLAIN") {
|
2015-09-17 00:54:12 -05:00
|
|
|
auth = smtp.PlainAuth("", opts.User, opts.Passwd, host)
|
2015-08-20 00:56:25 -05:00
|
|
|
} else if strings.Contains(options, "LOGIN") {
|
2015-08-20 06:12:55 -05:00
|
|
|
// Patch for AUTH LOGIN
|
2015-09-17 00:54:12 -05:00
|
|
|
auth = LoginAuth(opts.User, opts.Passwd)
|
2023-05-02 16:40:46 -05:00
|
|
|
} else if strings.Contains(options, "NTLM") {
|
|
|
|
|
auth = NtlmAuth(opts.User, opts.Passwd)
|
2014-12-18 06:58:18 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if auth != nil {
|
|
|
|
|
if err = client.Auth(auth); err != nil {
|
2022-10-24 14:29:17 -05:00
|
|
|
return fmt.Errorf("failed to authenticate SMTP: %w", err)
|
2014-12-18 06:58:18 -05:00
|
|
|
}
|
2014-10-09 17:08:07 -05:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2021-11-19 10:35:20 -05:00
|
|
|
if opts.OverrideEnvelopeFrom {
|
|
|
|
|
if err = client.Mail(opts.EnvelopeFrom); err != nil {
|
2022-10-24 14:29:17 -05:00
|
|
|
return fmt.Errorf("failed to issue MAIL command: %w", err)
|
2021-11-19 10:35:20 -05:00
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
if err = client.Mail(from); err != nil {
|
2022-10-24 14:29:17 -05:00
|
|
|
return fmt.Errorf("failed to issue MAIL command: %w", err)
|
2021-11-19 10:35:20 -05:00
|
|
|
}
|
2014-10-09 17:08:07 -05:00
|
|
|
}
|
|
|
|
|
|
2015-09-17 00:54:12 -05:00
|
|
|
for _, rec := range to {
|
2014-10-09 17:08:07 -05:00
|
|
|
if err = client.Rcpt(rec); err != nil {
|
2022-10-24 14:29:17 -05:00
|
|
|
return fmt.Errorf("failed to issue RCPT command: %w", err)
|
2014-10-09 17:08:07 -05:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
w, err := client.Data()
|
|
|
|
|
if err != nil {
|
2022-10-24 14:29:17 -05:00
|
|
|
return fmt.Errorf("failed to issue DATA command: %w", err)
|
2015-09-17 00:54:12 -05:00
|
|
|
} else if _, err = msg.WriteTo(w); err != nil {
|
2022-10-24 14:29:17 -05:00
|
|
|
return fmt.Errorf("SMTP write failed: %w", err)
|
2015-09-17 00:54:12 -05:00
|
|
|
} else if err = w.Close(); err != nil {
|
2022-10-24 14:29:17 -05:00
|
|
|
return fmt.Errorf("SMTP close failed: %w", err)
|
2014-10-09 17:08:07 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return client.Quit()
|
|
|
|
|
}
|
|
|
|
|
|
2016-12-25 08:55:22 -05:00
|
|
|
// Sender sendmail mail sender
|
2022-01-20 12:46:10 -05:00
|
|
|
type sendmailSender struct{}
|
2016-12-25 08:55:22 -05:00
|
|
|
|
|
|
|
|
// Send send email
|
|
|
|
|
func (s *sendmailSender) Send(from string, to []string, msg io.WriterTo) error {
|
|
|
|
|
var err error
|
|
|
|
|
var closeError error
|
|
|
|
|
var waitError error
|
|
|
|
|
|
2021-11-19 10:35:20 -05:00
|
|
|
envelopeFrom := from
|
|
|
|
|
if setting.MailService.OverrideEnvelopeFrom {
|
|
|
|
|
envelopeFrom = setting.MailService.EnvelopeFrom
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
args := []string{"-f", envelopeFrom, "-i"}
|
2017-10-25 14:27:25 -05:00
|
|
|
args = append(args, setting.MailService.SendmailArgs...)
|
2016-12-25 08:55:22 -05:00
|
|
|
args = append(args, to...)
|
|
|
|
|
log.Trace("Sending with: %s %v", setting.MailService.SendmailPath, args)
|
2020-05-02 18:04:31 -05:00
|
|
|
|
|
|
|
|
desc := fmt.Sprintf("SendMail: %s %v", setting.MailService.SendmailPath, args)
|
|
|
|
|
|
2021-11-30 15:06:32 -05:00
|
|
|
ctx, _, finished := process.GetManager().AddContextTimeout(graceful.GetManager().HammerContext(), setting.MailService.SendmailTimeout, desc)
|
|
|
|
|
defer finished()
|
2020-05-02 18:04:31 -05:00
|
|
|
|
|
|
|
|
cmd := exec.CommandContext(ctx, setting.MailService.SendmailPath, args...)
|
2016-12-25 08:55:22 -05:00
|
|
|
pipe, err := cmd.StdinPipe()
|
|
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
2022-06-03 09:36:18 -05:00
|
|
|
process.SetSysProcAttribute(cmd)
|
2016-12-25 08:55:22 -05:00
|
|
|
|
|
|
|
|
if err = cmd.Start(); err != nil {
|
2021-11-30 15:06:32 -05:00
|
|
|
_ = pipe.Close()
|
2016-12-25 08:55:22 -05:00
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
|
2022-01-05 19:43:45 -05:00
|
|
|
if setting.MailService.SendmailConvertCRLF {
|
|
|
|
|
buf := &strings.Builder{}
|
|
|
|
|
_, err = msg.WriteTo(buf)
|
|
|
|
|
if err == nil {
|
|
|
|
|
_, err = strings.NewReplacer("\r\n", "\n").WriteString(pipe, buf.String())
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
_, err = msg.WriteTo(pipe)
|
|
|
|
|
}
|
2014-03-19 07:27:27 -05:00
|
|
|
|
2016-12-25 08:55:22 -05:00
|
|
|
// we MUST close the pipe or sendmail will hang waiting for more of the message
|
|
|
|
|
// Also we should wait on our sendmail command even if something fails
|
|
|
|
|
closeError = pipe.Close()
|
2016-12-30 02:26:05 -05:00
|
|
|
waitError = cmd.Wait()
|
2016-12-25 08:55:22 -05:00
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
} else if closeError != nil {
|
|
|
|
|
return closeError
|
|
|
|
|
} else {
|
|
|
|
|
return waitError
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2019-02-02 21:06:52 -05:00
|
|
|
// Sender sendmail mail sender
|
2022-01-20 12:46:10 -05:00
|
|
|
type dummySender struct{}
|
2019-02-02 21:06:52 -05:00
|
|
|
|
|
|
|
|
// Send send email
|
|
|
|
|
func (s *dummySender) Send(from string, to []string, msg io.WriterTo) error {
|
|
|
|
|
buf := bytes.Buffer{}
|
|
|
|
|
if _, err := msg.WriteTo(&buf); err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
log.Info("Mail From: %s To: %v Body: %s", from, to, buf.String())
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
|
Rewrite queue (#24505)
# ⚠️ Breaking
Many deprecated queue config options are removed (actually, they should
have been removed in 1.18/1.19).
If you see the fatal message when starting Gitea: "Please update your
app.ini to remove deprecated config options", please follow the error
messages to remove these options from your app.ini.
Example:
```
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options
```
Many options in `[queue]` are are dropped, including:
`WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`,
`BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed
from app.ini.
# The problem
The old queue package has some legacy problems:
* complexity: I doubt few people could tell how it works.
* maintainability: Too many channels and mutex/cond are mixed together,
too many different structs/interfaces depends each other.
* stability: due to the complexity & maintainability, sometimes there
are strange bugs and difficult to debug, and some code doesn't have test
(indeed some code is difficult to test because a lot of things are mixed
together).
* general applicability: although it is called "queue", its behavior is
not a well-known queue.
* scalability: it doesn't seem easy to make it work with a cluster
without breaking its behaviors.
It came from some very old code to "avoid breaking", however, its
technical debt is too heavy now. It's a good time to introduce a better
"queue" package.
# The new queue package
It keeps using old config and concept as much as possible.
* It only contains two major kinds of concepts:
* The "base queue": channel, levelqueue, redis
* They have the same abstraction, the same interface, and they are
tested by the same testing code.
* The "WokerPoolQueue", it uses the "base queue" to provide "worker
pool" function, calls the "handler" to process the data in the base
queue.
* The new code doesn't do "PushBack"
* Think about a queue with many workers, the "PushBack" can't guarantee
the order for re-queued unhandled items, so in new code it just does
"normal push"
* The new code doesn't do "pause/resume"
* The "pause/resume" was designed to handle some handler's failure: eg:
document indexer (elasticsearch) is down
* If a queue is paused for long time, either the producers blocks or the
new items are dropped.
* The new code doesn't do such "pause/resume" trick, it's not a common
queue's behavior and it doesn't help much.
* If there are unhandled items, the "push" function just blocks for a
few seconds and then re-queue them and retry.
* The new code doesn't do "worker booster"
* Gitea's queue's handlers are light functions, the cost is only the
go-routine, so it doesn't make sense to "boost" them.
* The new code only use "max worker number" to limit the concurrent
workers.
* The new "Push" never blocks forever
* Instead of creating more and more blocking goroutines, return an error
is more friendly to the server and to the end user.
There are more details in code comments: eg: the "Flush" problem, the
strange "code.index" hanging problem, the "immediate" queue problem.
Almost ready for review.
TODO:
* [x] add some necessary comments during review
* [x] add some more tests if necessary
* [x] update documents and config options
* [x] test max worker / active worker
* [x] re-run the CI tasks to see whether any test is flaky
* [x] improve the `handleOldLengthConfiguration` to provide more
friendly messages
* [x] fine tune default config values (eg: length?)
## Code coverage:
2023-05-08 06:49:59 -05:00
|
|
|
var mailQueue *queue.WorkerPoolQueue[*Message]
|
2015-09-17 00:54:12 -05:00
|
|
|
|
2016-12-25 08:55:22 -05:00
|
|
|
// Sender sender for sending mail synchronously
|
|
|
|
|
var Sender gomail.Sender
|
|
|
|
|
|
2016-11-24 20:44:04 -05:00
|
|
|
// NewContext start mail queue service
|
2022-08-28 04:43:25 -05:00
|
|
|
func NewContext(ctx context.Context) {
|
2016-02-20 17:32:34 -05:00
|
|
|
// Need to check if mailQueue is nil because in during reinstall (user had installed
|
2021-07-08 06:38:13 -05:00
|
|
|
// before but switched install lock off), this function will be called again
|
2016-02-20 17:32:34 -05:00
|
|
|
// while mail queue is already processing tasks, and produces a race condition.
|
|
|
|
|
if setting.MailService == nil || mailQueue != nil {
|
2015-09-17 00:54:12 -05:00
|
|
|
return
|
2014-03-19 07:27:27 -05:00
|
|
|
}
|
2015-09-17 00:54:12 -05:00
|
|
|
|
2023-09-05 04:26:59 -05:00
|
|
|
if setting.Service.EnableNotifyMail {
|
2023-09-05 13:37:47 -05:00
|
|
|
notify_service.RegisterNotifier(NewNotifier())
|
2023-09-05 04:26:59 -05:00
|
|
|
}
|
|
|
|
|
|
Rework mailer settings (#18982)
* `PROTOCOL`: can be smtp, smtps, smtp+startls, smtp+unix, sendmail, dummy
* `SMTP_ADDR`: domain for SMTP, or path to unix socket
* `SMTP_PORT`: port for SMTP; defaults to 25 for `smtp`, 465 for `smtps`, and 587 for `smtp+startls`
* `ENABLE_HELO`, `HELO_HOSTNAME`: reverse `DISABLE_HELO` to `ENABLE_HELO`; default to false + system hostname
* `FORCE_TRUST_SERVER_CERT`: replace the unclear `SKIP_VERIFY`
* `CLIENT_CERT_FILE`, `CLIENT_KEY_FILE`, `USE_CLIENT_CERT`: clarify client certificates here
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2022-08-02 00:24:18 -05:00
|
|
|
switch setting.MailService.Protocol {
|
2019-02-02 21:06:52 -05:00
|
|
|
case "sendmail":
|
|
|
|
|
Sender = &sendmailSender{}
|
|
|
|
|
case "dummy":
|
|
|
|
|
Sender = &dummySender{}
|
Rework mailer settings (#18982)
* `PROTOCOL`: can be smtp, smtps, smtp+startls, smtp+unix, sendmail, dummy
* `SMTP_ADDR`: domain for SMTP, or path to unix socket
* `SMTP_PORT`: port for SMTP; defaults to 25 for `smtp`, 465 for `smtps`, and 587 for `smtp+startls`
* `ENABLE_HELO`, `HELO_HOSTNAME`: reverse `DISABLE_HELO` to `ENABLE_HELO`; default to false + system hostname
* `FORCE_TRUST_SERVER_CERT`: replace the unclear `SKIP_VERIFY`
* `CLIENT_CERT_FILE`, `CLIENT_KEY_FILE`, `USE_CLIENT_CERT`: clarify client certificates here
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2022-08-02 00:24:18 -05:00
|
|
|
default:
|
|
|
|
|
Sender = &smtpSender{}
|
2016-12-25 08:55:22 -05:00
|
|
|
}
|
|
|
|
|
|
Improve queue and logger context (#24924)
Before there was a "graceful function": RunWithShutdownFns, it's mainly
for some modules which doesn't support context.
The old queue system doesn't work well with context, so the old queues
need it.
After the queue refactoring, the new queue works with context well, so,
use Golang context as much as possible, the `RunWithShutdownFns` could
be removed (replaced by RunWithCancel for context cancel mechanism), the
related code could be simplified.
This PR also fixes some legacy queue-init problems, eg:
* typo : archiver: "unable to create codes indexer queue" => "unable to
create repo-archive queue"
* no nil check for failed queues, which causes unfriendly panic
After this PR, many goroutines could have better display name:
2023-05-26 02:31:55 -05:00
|
|
|
subjectTemplates, bodyTemplates = templates.Mailer(ctx)
|
|
|
|
|
|
|
|
|
|
mailQueue = queue.CreateSimpleQueue(graceful.GetManager().ShutdownContext(), "mail", func(items ...*Message) []*Message {
|
Rewrite queue (#24505)
# ⚠️ Breaking
Many deprecated queue config options are removed (actually, they should
have been removed in 1.18/1.19).
If you see the fatal message when starting Gitea: "Please update your
app.ini to remove deprecated config options", please follow the error
messages to remove these options from your app.ini.
Example:
```
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options
```
Many options in `[queue]` are are dropped, including:
`WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`,
`BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed
from app.ini.
# The problem
The old queue package has some legacy problems:
* complexity: I doubt few people could tell how it works.
* maintainability: Too many channels and mutex/cond are mixed together,
too many different structs/interfaces depends each other.
* stability: due to the complexity & maintainability, sometimes there
are strange bugs and difficult to debug, and some code doesn't have test
(indeed some code is difficult to test because a lot of things are mixed
together).
* general applicability: although it is called "queue", its behavior is
not a well-known queue.
* scalability: it doesn't seem easy to make it work with a cluster
without breaking its behaviors.
It came from some very old code to "avoid breaking", however, its
technical debt is too heavy now. It's a good time to introduce a better
"queue" package.
# The new queue package
It keeps using old config and concept as much as possible.
* It only contains two major kinds of concepts:
* The "base queue": channel, levelqueue, redis
* They have the same abstraction, the same interface, and they are
tested by the same testing code.
* The "WokerPoolQueue", it uses the "base queue" to provide "worker
pool" function, calls the "handler" to process the data in the base
queue.
* The new code doesn't do "PushBack"
* Think about a queue with many workers, the "PushBack" can't guarantee
the order for re-queued unhandled items, so in new code it just does
"normal push"
* The new code doesn't do "pause/resume"
* The "pause/resume" was designed to handle some handler's failure: eg:
document indexer (elasticsearch) is down
* If a queue is paused for long time, either the producers blocks or the
new items are dropped.
* The new code doesn't do such "pause/resume" trick, it's not a common
queue's behavior and it doesn't help much.
* If there are unhandled items, the "push" function just blocks for a
few seconds and then re-queue them and retry.
* The new code doesn't do "worker booster"
* Gitea's queue's handlers are light functions, the cost is only the
go-routine, so it doesn't make sense to "boost" them.
* The new code only use "max worker number" to limit the concurrent
workers.
* The new "Push" never blocks forever
* Instead of creating more and more blocking goroutines, return an error
is more friendly to the server and to the end user.
There are more details in code comments: eg: the "Flush" problem, the
strange "code.index" hanging problem, the "immediate" queue problem.
Almost ready for review.
TODO:
* [x] add some necessary comments during review
* [x] add some more tests if necessary
* [x] update documents and config options
* [x] test max worker / active worker
* [x] re-run the CI tasks to see whether any test is flaky
* [x] improve the `handleOldLengthConfiguration` to provide more
friendly messages
* [x] fine tune default config values (eg: length?)
## Code coverage:
2023-05-08 06:49:59 -05:00
|
|
|
for _, msg := range items {
|
2020-01-16 12:55:36 -05:00
|
|
|
gomailMsg := msg.ToMessage()
|
|
|
|
|
log.Trace("New e-mail sending request %s: %s", gomailMsg.GetHeader("To"), msg.Info)
|
|
|
|
|
if err := gomail.Send(Sender, gomailMsg); err != nil {
|
|
|
|
|
log.Error("Failed to send emails %s: %s - %v", gomailMsg.GetHeader("To"), msg.Info, err)
|
|
|
|
|
} else {
|
|
|
|
|
log.Trace("E-mails sent %s: %s", gomailMsg.GetHeader("To"), msg.Info)
|
|
|
|
|
}
|
|
|
|
|
}
|
2022-01-22 16:22:14 -05:00
|
|
|
return nil
|
Rewrite queue (#24505)
# ⚠️ Breaking
Many deprecated queue config options are removed (actually, they should
have been removed in 1.18/1.19).
If you see the fatal message when starting Gitea: "Please update your
app.ini to remove deprecated config options", please follow the error
messages to remove these options from your app.ini.
Example:
```
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options
```
Many options in `[queue]` are are dropped, including:
`WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`,
`BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed
from app.ini.
# The problem
The old queue package has some legacy problems:
* complexity: I doubt few people could tell how it works.
* maintainability: Too many channels and mutex/cond are mixed together,
too many different structs/interfaces depends each other.
* stability: due to the complexity & maintainability, sometimes there
are strange bugs and difficult to debug, and some code doesn't have test
(indeed some code is difficult to test because a lot of things are mixed
together).
* general applicability: although it is called "queue", its behavior is
not a well-known queue.
* scalability: it doesn't seem easy to make it work with a cluster
without breaking its behaviors.
It came from some very old code to "avoid breaking", however, its
technical debt is too heavy now. It's a good time to introduce a better
"queue" package.
# The new queue package
It keeps using old config and concept as much as possible.
* It only contains two major kinds of concepts:
* The "base queue": channel, levelqueue, redis
* They have the same abstraction, the same interface, and they are
tested by the same testing code.
* The "WokerPoolQueue", it uses the "base queue" to provide "worker
pool" function, calls the "handler" to process the data in the base
queue.
* The new code doesn't do "PushBack"
* Think about a queue with many workers, the "PushBack" can't guarantee
the order for re-queued unhandled items, so in new code it just does
"normal push"
* The new code doesn't do "pause/resume"
* The "pause/resume" was designed to handle some handler's failure: eg:
document indexer (elasticsearch) is down
* If a queue is paused for long time, either the producers blocks or the
new items are dropped.
* The new code doesn't do such "pause/resume" trick, it's not a common
queue's behavior and it doesn't help much.
* If there are unhandled items, the "push" function just blocks for a
few seconds and then re-queue them and retry.
* The new code doesn't do "worker booster"
* Gitea's queue's handlers are light functions, the cost is only the
go-routine, so it doesn't make sense to "boost" them.
* The new code only use "max worker number" to limit the concurrent
workers.
* The new "Push" never blocks forever
* Instead of creating more and more blocking goroutines, return an error
is more friendly to the server and to the end user.
There are more details in code comments: eg: the "Flush" problem, the
strange "code.index" hanging problem, the "immediate" queue problem.
Almost ready for review.
TODO:
* [x] add some necessary comments during review
* [x] add some more tests if necessary
* [x] update documents and config options
* [x] test max worker / active worker
* [x] re-run the CI tasks to see whether any test is flaky
* [x] improve the `handleOldLengthConfiguration` to provide more
friendly messages
* [x] fine tune default config values (eg: length?)
## Code coverage:
2023-05-08 06:49:59 -05:00
|
|
|
})
|
Improve queue and logger context (#24924)
Before there was a "graceful function": RunWithShutdownFns, it's mainly
for some modules which doesn't support context.
The old queue system doesn't work well with context, so the old queues
need it.
After the queue refactoring, the new queue works with context well, so,
use Golang context as much as possible, the `RunWithShutdownFns` could
be removed (replaced by RunWithCancel for context cancel mechanism), the
related code could be simplified.
This PR also fixes some legacy queue-init problems, eg:
* typo : archiver: "unable to create codes indexer queue" => "unable to
create repo-archive queue"
* no nil check for failed queues, which causes unfriendly panic
After this PR, many goroutines could have better display name:
2023-05-26 02:31:55 -05:00
|
|
|
if mailQueue == nil {
|
|
|
|
|
log.Fatal("Unable to create mail queue")
|
|
|
|
|
}
|
|
|
|
|
go graceful.GetManager().RunWithCancel(mailQueue)
|
2014-03-19 07:27:27 -05:00
|
|
|
}
|
|
|
|
|
|
2019-11-18 03:08:20 -05:00
|
|
|
// SendAsync send mail asynchronously
|
2014-03-19 20:05:48 -05:00
|
|
|
func SendAsync(msg *Message) {
|
2021-04-02 05:25:13 -05:00
|
|
|
SendAsyncs([]*Message{msg})
|
2014-03-19 07:27:27 -05:00
|
|
|
}
|
2019-11-18 03:08:20 -05:00
|
|
|
|
|
|
|
|
// SendAsyncs send mails asynchronously
|
|
|
|
|
func SendAsyncs(msgs []*Message) {
|
2021-04-02 05:25:13 -05:00
|
|
|
if setting.MailService == nil {
|
|
|
|
|
log.Error("Mailer: SendAsyncs is being invoked but mail service hasn't been initialized")
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
2019-11-18 03:08:20 -05:00
|
|
|
go func() {
|
|
|
|
|
for _, msg := range msgs {
|
2020-01-16 12:55:36 -05:00
|
|
|
_ = mailQueue.Push(msg)
|
2019-11-18 03:08:20 -05:00
|
|
|
}
|
|
|
|
|
}()
|
|
|
|
|
}
|
