0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2024-12-23 15:53:07 -05:00
forgejo/tests/integration/integration_test.go

565 lines
16 KiB
Go
Raw Normal View History

// Copyright 2017 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
Enable forbidigo linter (#24278) Enable [forbidigo](https://github.com/ashanbrown/forbidigo) linter which forbids print statements. Will check how to integrate this with the smallest impact possible, so a few `nolint` comments will likely be required. Plan is to just go through the issues and either: - Remove the print if it is nonsensical - Add a `//nolint` directive if it makes sense I don't plan on investigating the individual issues any further. <details> <summary>Initial Lint Results</summary> ``` modules/log/event.go:348:6: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Println(err) ^ modules/log/event.go:382:6: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Println(err) ^ modules/queue/unique_queue_disk_channel_test.go:20:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("TempDir %s\n", tmpDir) ^ contrib/backport/backport.go:168:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("* Backporting %s to %s as %s\n", pr, localReleaseBranch, backportBranch) ^ contrib/backport/backport.go:216:4: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("* Navigate to %s to open PR\n", url) ^ contrib/backport/backport.go:223:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("* `xdg-open %s`\n", url) ^ contrib/backport/backport.go:233:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("* `git push -u %s %s`\n", remote, backportBranch) ^ contrib/backport/backport.go:243:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("* Amending commit to prepend `Backport #%s` to body\n", pr) ^ contrib/backport/backport.go:272:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Println("* Attempting git cherry-pick --continue") ^ contrib/backport/backport.go:281:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("* Attempting git cherry-pick %s\n", sha) ^ contrib/backport/backport.go:297:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("* Current branch is %s\n", currentBranch) ^ contrib/backport/backport.go:299:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("* Current branch is %s - not checking out\n", currentBranch) ^ contrib/backport/backport.go:304:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("* Branch %s already exists. Checking it out...\n", backportBranch) ^ contrib/backport/backport.go:308:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("* `git checkout -b %s %s`\n", backportBranch, releaseBranch) ^ contrib/backport/backport.go:313:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("* `git fetch %s main`\n", remote) ^ contrib/backport/backport.go:316:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Println(string(out)) ^ contrib/backport/backport.go:319:2: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Println(string(out)) ^ contrib/backport/backport.go:321:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("* `git fetch %s %s`\n", remote, releaseBranch) ^ contrib/backport/backport.go:324:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Println(string(out)) ^ contrib/backport/backport.go:327:2: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Println(string(out)) ^ models/unittest/fixtures.go:50:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Println("Unsupported RDBMS for integration tests") ^ models/unittest/fixtures.go:89:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("LoadFixtures failed after retries: %v\n", err) ^ models/unittest/fixtures.go:110:4: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("Failed to generate sequence update: %v\n", err) ^ models/unittest/fixtures.go:117:6: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("Failed to update sequence: %s Error: %v\n", value, err) ^ models/migrations/base/tests.go:118:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Println("Environment variable $GITEA_ROOT not set") ^ models/migrations/base/tests.go:127:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("Could not find gitea binary at %s\n", setting.AppPath) ^ models/migrations/base/tests.go:134:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("Environment variable $GITEA_CONF not set - defaulting to %s\n", giteaConf) ^ models/migrations/base/tests.go:145:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("Unable to create temporary data path %v\n", err) ^ models/migrations/base/tests.go:154:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("Unable to InitFull: %v\n", err) ^ models/migrations/v1_11/v112.go:34:5: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("Error: %v", err) ^ contrib/fixtures/fixture_generation.go:36:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("CreateTestEngine: %+v", err) ^ contrib/fixtures/fixture_generation.go:40:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("PrepareTestDatabase: %+v\n", err) ^ contrib/fixtures/fixture_generation.go:46:5: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("generate '%s': %+v\n", r, err) ^ contrib/fixtures/fixture_generation.go:53:5: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("generate '%s': %+v\n", g.name, err) ^ contrib/fixtures/fixture_generation.go:71:4: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("%s created.\n", path) ^ services/gitdiff/gitdiff_test.go:543:2: use of `println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) println(result) ^ services/gitdiff/gitdiff_test.go:560:2: use of `println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) println(result) ^ services/gitdiff/gitdiff_test.go:577:2: use of `println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) println(result) ^ modules/web/routing/logger_manager.go:34:2: use of `print` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) print Printer ^ modules/doctor/paths.go:109:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("Warning: can't remove temporary file: '%s'\n", tmpFile.Name()) ^ tests/test_utils.go:33:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf(format+"\n", args...) ^ tests/test_utils.go:61:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("Environment variable $GITEA_CONF not set, use default: %s\n", giteaConf) ^ cmd/actions.go:54:9: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) _, _ = fmt.Printf("%s\n", respText) ^ cmd/admin_user_change_password.go:74:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("%s's password has been successfully updated!\n", user.Name) ^ cmd/admin_user_create.go:109:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("generated random password is '%s'\n", password) ^ cmd/admin_user_create.go:164:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("Access token was successfully created... %s\n", t.Token) ^ cmd/admin_user_create.go:167:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("New user '%s' has been successfully created!\n", username) ^ cmd/admin_user_generate_access_token.go:74:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("%s\n", t.Token) ^ cmd/admin_user_generate_access_token.go:76:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("Access token was successfully created: %s\n", t.Token) ^ cmd/admin_user_must_change_password.go:56:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("Updated %d users setting MustChangePassword to %t\n", n, mustChangePassword) ^ cmd/convert.go:44:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Println("Converted successfully, please confirm your database's character set is now utf8mb4") ^ cmd/convert.go:50:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Println("Converted successfully, please confirm your database's all columns character is NVARCHAR now") ^ cmd/convert.go:52:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Println("This command can only be used with a MySQL or MSSQL database") ^ cmd/doctor.go:104:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Println(err) ^ cmd/doctor.go:105:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Println("Check if you are using the right config file. You can use a --config directive to specify one.") ^ cmd/doctor.go:243:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Println(err) ^ cmd/embedded.go:154:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Println(a.path) ^ cmd/embedded.go:198:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Println("Using app.ini at", setting.CustomConf) ^ cmd/embedded.go:217:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("Extracting to %s:\n", destdir) ^ cmd/embedded.go:253:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("%s already exists; skipped.\n", dest) ^ cmd/embedded.go:275:2: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Println(dest) ^ cmd/generate.go:63:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("%s", internalToken) ^ cmd/generate.go:66:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("\n") ^ cmd/generate.go:78:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("%s", JWTSecretBase64) ^ cmd/generate.go:81:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("\n") ^ cmd/generate.go:93:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("%s", secretKey) ^ cmd/generate.go:96:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("\n") ^ cmd/keys.go:74:2: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Println(strings.TrimSpace(authorizedString)) ^ cmd/mailer.go:32:4: use of `fmt.Print` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Print("warning: Content is empty") ^ cmd/mailer.go:35:3: use of `fmt.Print` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Print("Proceed with sending email? [Y/n] ") ^ cmd/mailer.go:40:4: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Println("The mail was not sent") ^ cmd/mailer.go:49:9: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) _, _ = fmt.Printf("Sent %s email(s) to all users\n", respText) ^ cmd/serv.go:147:3: use of `println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) println("Gitea: SSH has been disabled") ^ cmd/serv.go:153:4: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("error showing subcommand help: %v\n", err) ^ cmd/serv.go:175:4: use of `println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) println("Hi there! You've successfully authenticated with the deploy key named " + key.Name + ", but Gitea does not provide shell access.") ^ cmd/serv.go:177:4: use of `println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) println("Hi there! You've successfully authenticated with the principal " + key.Content + ", but Gitea does not provide shell access.") ^ cmd/serv.go:179:4: use of `println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) println("Hi there, " + user.Name + "! You've successfully authenticated with the key named " + key.Name + ", but Gitea does not provide shell access.") ^ cmd/serv.go:181:3: use of `println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) println("If this is unexpected, please log in with password and setup Gitea under another user.") ^ cmd/serv.go:196:5: use of `fmt.Print` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Print(`{"type":"gitea","version":1}`) ^ tests/e2e/e2e_test.go:54:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("Error initializing test database: %v\n", err) ^ tests/e2e/e2e_test.go:63:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("util.RemoveAll: %v\n", err) ^ tests/e2e/e2e_test.go:67:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("Unable to remove repo indexer: %v\n", err) ^ tests/e2e/e2e_test.go:109:6: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("%v", stdout.String()) ^ tests/e2e/e2e_test.go:110:6: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("%v", stderr.String()) ^ tests/e2e/e2e_test.go:113:6: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("%v", stdout.String()) ^ tests/integration/integration_test.go:124:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("Error initializing test database: %v\n", err) ^ tests/integration/integration_test.go:135:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("util.RemoveAll: %v\n", err) ^ tests/integration/integration_test.go:139:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("Unable to remove repo indexer: %v\n", err) ^ tests/integration/repo_test.go:357:4: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo) fmt.Printf("%s", resp.Body) ^ ``` </details> --------- Co-authored-by: Giteabot <teabot@gitea.io>
2023-04-24 04:50:58 -05:00
//nolint:forbidigo
package integration
import (
"bytes"
"context"
"fmt"
"hash"
"hash/fnv"
"io"
"net/http"
"net/http/cookiejar"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
[SECURITY] Rework long-term authentication - This is a 'front-port' of the already existing patch on v1.21 and v1.20, but applied on top of what Gitea has done to rework the LTA mechanism. Forgejo will stick with the reworked mechanism by the Forgejo Security team for the time being. The removal of legacy code (AES-GCM) has been left out. - The current architecture is inherently insecure, because you can construct the 'secret' cookie value with values that are available in the database. Thus provides zero protection when a database is dumped/leaked. - This patch implements a new architecture that's inspired from: [Paragonie Initiative](https://paragonie.com/blog/2015/04/secure-authentication-php-with-long-term-persistence#secure-remember-me-cookies). - Integration testing is added to ensure the new mechanism works. - Removes a setting, because it's not used anymore. (cherry picked from commit e3d6622a63da9c33eed1e3d102cf28a92ff653d6) (cherry picked from commit fef1a6dac5e25579e42d40209c4cfc06879948b9) (cherry picked from commit b0c5165145fa52f2f7bbec1f50b308bdf1d20ef3) (cherry picked from commit 7ad51b9f8d0647eecacd258f6ee26155da3872e1) (cherry picked from commit 64f053f3834e764112cde26bb0d16c5e88d6b2af) (cherry picked from commit f5e78e4c204ce50b800645d614218b6b6096eecb) Conflicts: services/auth/auth_token_test.go https://codeberg.org/forgejo/forgejo/pulls/2069 (cherry picked from commit f69fc23d4bbadf388c7857040ee0774b824e418e) (cherry picked from commit d955ab3ab02cbb7f1245a8cddec426d64d3ac500) (cherry picked from commit 9220088f902a25c4690bcabf5a40a8d02e784182) (cherry picked from commit c73ac636962c41c71814c273510146f0533264ab) (cherry picked from commit 747a176048ea93085b406429db0e25bb21912eda) Conflicts: models/user/user.go routers/web/user/setting/account.go https://codeberg.org/forgejo/forgejo/pulls/2295
2023-11-22 11:26:21 -05:00
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
"code.gitea.io/gitea/models/auth"
[TESTS] createUser via the user model helper for integration tests (cherry picked from commit c1d14c5fffeb823385b2984cfcdb3e195bfb151d) (cherry picked from commit e0e8aabc985af153cf1fcb2064c17f68ec37f3a2) (cherry picked from commit 392a415070af7f4e872c7bb013f259e2dcc8b7b4) (cherry picked from commit c7cf1307ca227ff9f4935391582c058823419d7b) (cherry picked from commit 93b13d092b46c261fc3c0f70ecd9d4f45ee84029) [TESTS] createUser via the user model helper for integration tests (squash) (cherry picked from commit 6ff23839528db66ae91e88d6b687b63c769f8f43) (cherry picked from commit de2a6fe8c374965d50bcd6421d44c3a5d312a689) (cherry picked from commit 398a6ab072e8364db3c5236bf258ec6bb068dc29) (cherry picked from commit 16abc89780a228fdc288ed2ce44a95daa6368e36) (cherry picked from commit 312a3ec5d9d50864af9dbfb8672778789b2e11ca) (cherry picked from commit 85c6d8e29065641ccbaf57fcebe72b93141a14a4) (cherry picked from commit 79150d30a4d9a91e9e5272f3d46bd0ee836bcab1) (cherry picked from commit 436137962d3f150d4c28093461c30c9910f2d32d) (cherry picked from commit e4eb8d471e8e1c8e14a6a3208418bde0b121472c) (cherry picked from commit a7257052f49a480dd2ccfbef5c0573c9b5b73adb) (cherry picked from commit d5eba9a6dd93eb21350dc611edff7c9a13cb2552) (cherry picked from commit d89ef2ffa9a83249b498867a02996a3dcb17aa31) (cherry picked from commit f1d25aa30762ae97d7994048dd03c656028dedbe) (cherry picked from commit 60c7c0735376d7987717c0d375fa409ca9b90e8f) (cherry picked from commit cfeff3afdb80785019f53312369953e2d996de89) (cherry picked from commit 608ac6bd683d60fe295163ba5562aee5037b6e01) (cherry picked from commit c64e530a13fe4afc89ad3de481fff16fe190c3d1) (cherry picked from commit 513db02971fa98ff88f9b06bf25b03c31d0a857c) (cherry picked from commit 43eaaa5a61134928ff9a83e571552407450d0a09) (cherry picked from commit 678eb494408e3784336a3aeb5e5f54bd738d009e) (cherry picked from commit f7458dabda74dedecb431a75e9f049e545fc973e) (cherry picked from commit 32568e43b412bf53bc207f5b429ee4ef77693857) (cherry picked from commit 3fcbb2043d04fb701a7baa26cc599c98dfaa5601) (cherry picked from commit 95f0b8eefad590c647951f42bd0f467685901bd2) (cherry picked from commit f285b9469d0b644abc70ebbc79c4b14fe5a03e56) (cherry picked from commit 4b8a02ad088bc925dee1b2f0b759ea9ef77d96e8) (cherry picked from commit 70eb98025b6012f42c0161999f7675f069bde877) (cherry picked from commit dc2d448705404872c7f67887f7cf80658e42c3fa) (cherry picked from commit e1402ca319f549f8ed36a9f8b58f7aba09793415) (cherry picked from commit 807d587285c06b82d636fba371f946c68f0a22e3)
2023-06-05 04:43:31 -05:00
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/models/unittest"
[TESTS] createUser via the user model helper for integration tests (cherry picked from commit c1d14c5fffeb823385b2984cfcdb3e195bfb151d) (cherry picked from commit e0e8aabc985af153cf1fcb2064c17f68ec37f3a2) (cherry picked from commit 392a415070af7f4e872c7bb013f259e2dcc8b7b4) (cherry picked from commit c7cf1307ca227ff9f4935391582c058823419d7b) (cherry picked from commit 93b13d092b46c261fc3c0f70ecd9d4f45ee84029) [TESTS] createUser via the user model helper for integration tests (squash) (cherry picked from commit 6ff23839528db66ae91e88d6b687b63c769f8f43) (cherry picked from commit de2a6fe8c374965d50bcd6421d44c3a5d312a689) (cherry picked from commit 398a6ab072e8364db3c5236bf258ec6bb068dc29) (cherry picked from commit 16abc89780a228fdc288ed2ce44a95daa6368e36) (cherry picked from commit 312a3ec5d9d50864af9dbfb8672778789b2e11ca) (cherry picked from commit 85c6d8e29065641ccbaf57fcebe72b93141a14a4) (cherry picked from commit 79150d30a4d9a91e9e5272f3d46bd0ee836bcab1) (cherry picked from commit 436137962d3f150d4c28093461c30c9910f2d32d) (cherry picked from commit e4eb8d471e8e1c8e14a6a3208418bde0b121472c) (cherry picked from commit a7257052f49a480dd2ccfbef5c0573c9b5b73adb) (cherry picked from commit d5eba9a6dd93eb21350dc611edff7c9a13cb2552) (cherry picked from commit d89ef2ffa9a83249b498867a02996a3dcb17aa31) (cherry picked from commit f1d25aa30762ae97d7994048dd03c656028dedbe) (cherry picked from commit 60c7c0735376d7987717c0d375fa409ca9b90e8f) (cherry picked from commit cfeff3afdb80785019f53312369953e2d996de89) (cherry picked from commit 608ac6bd683d60fe295163ba5562aee5037b6e01) (cherry picked from commit c64e530a13fe4afc89ad3de481fff16fe190c3d1) (cherry picked from commit 513db02971fa98ff88f9b06bf25b03c31d0a857c) (cherry picked from commit 43eaaa5a61134928ff9a83e571552407450d0a09) (cherry picked from commit 678eb494408e3784336a3aeb5e5f54bd738d009e) (cherry picked from commit f7458dabda74dedecb431a75e9f049e545fc973e) (cherry picked from commit 32568e43b412bf53bc207f5b429ee4ef77693857) (cherry picked from commit 3fcbb2043d04fb701a7baa26cc599c98dfaa5601) (cherry picked from commit 95f0b8eefad590c647951f42bd0f467685901bd2) (cherry picked from commit f285b9469d0b644abc70ebbc79c4b14fe5a03e56) (cherry picked from commit 4b8a02ad088bc925dee1b2f0b759ea9ef77d96e8) (cherry picked from commit 70eb98025b6012f42c0161999f7675f069bde877) (cherry picked from commit dc2d448705404872c7f67887f7cf80658e42c3fa) (cherry picked from commit e1402ca319f549f8ed36a9f8b58f7aba09793415) (cherry picked from commit 807d587285c06b82d636fba371f946c68f0a22e3)
2023-06-05 04:43:31 -05:00
user_model "code.gitea.io/gitea/models/user"
gitea_context "code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
Rewrite queue (#24505) # ⚠️ Breaking Many deprecated queue config options are removed (actually, they should have been removed in 1.18/1.19). If you see the fatal message when starting Gitea: "Please update your app.ini to remove deprecated config options", please follow the error messages to remove these options from your app.ini. Example: ``` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options ``` Many options in `[queue]` are are dropped, including: `WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`, `BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed from app.ini. # The problem The old queue package has some legacy problems: * complexity: I doubt few people could tell how it works. * maintainability: Too many channels and mutex/cond are mixed together, too many different structs/interfaces depends each other. * stability: due to the complexity & maintainability, sometimes there are strange bugs and difficult to debug, and some code doesn't have test (indeed some code is difficult to test because a lot of things are mixed together). * general applicability: although it is called "queue", its behavior is not a well-known queue. * scalability: it doesn't seem easy to make it work with a cluster without breaking its behaviors. It came from some very old code to "avoid breaking", however, its technical debt is too heavy now. It's a good time to introduce a better "queue" package. # The new queue package It keeps using old config and concept as much as possible. * It only contains two major kinds of concepts: * The "base queue": channel, levelqueue, redis * They have the same abstraction, the same interface, and they are tested by the same testing code. * The "WokerPoolQueue", it uses the "base queue" to provide "worker pool" function, calls the "handler" to process the data in the base queue. * The new code doesn't do "PushBack" * Think about a queue with many workers, the "PushBack" can't guarantee the order for re-queued unhandled items, so in new code it just does "normal push" * The new code doesn't do "pause/resume" * The "pause/resume" was designed to handle some handler's failure: eg: document indexer (elasticsearch) is down * If a queue is paused for long time, either the producers blocks or the new items are dropped. * The new code doesn't do such "pause/resume" trick, it's not a common queue's behavior and it doesn't help much. * If there are unhandled items, the "push" function just blocks for a few seconds and then re-queue them and retry. * The new code doesn't do "worker booster" * Gitea's queue's handlers are light functions, the cost is only the go-routine, so it doesn't make sense to "boost" them. * The new code only use "max worker number" to limit the concurrent workers. * The new "Push" never blocks forever * Instead of creating more and more blocking goroutines, return an error is more friendly to the server and to the end user. There are more details in code comments: eg: the "Flush" problem, the strange "code.index" hanging problem, the "immediate" queue problem. Almost ready for review. TODO: * [x] add some necessary comments during review * [x] add some more tests if necessary * [x] update documents and config options * [x] test max worker / active worker * [x] re-run the CI tasks to see whether any test is flaky * [x] improve the `handleOldLengthConfiguration` to provide more friendly messages * [x] fine tune default config values (eg: length?) ## Code coverage: ![image](https://user-images.githubusercontent.com/2114189/236620635-55576955-f95d-4810-b12f-879026a3afdf.png)
2023-05-08 06:49:59 -05:00
"code.gitea.io/gitea/modules/testlogger"
"code.gitea.io/gitea/modules/util"
Move macaron to chi (#14293) Use [chi](https://github.com/go-chi/chi) instead of the forked [macaron](https://gitea.com/macaron/macaron). Since macaron and chi have conflicts with session share, this big PR becomes a have-to thing. According my previous idea, we can replace macaron step by step but I'm wrong. :( Below is a list of big changes on this PR. - [x] Define `context.ResponseWriter` interface with an implementation `context.Response`. - [x] Use chi instead of macaron, and also a customize `Route` to wrap chi so that the router usage is similar as before. - [x] Create different routers for `web`, `api`, `internal` and `install` so that the codes will be more clear and no magic . - [x] Use https://github.com/unrolled/render instead of macaron's internal render - [x] Use https://github.com/NYTimes/gziphandler instead of https://gitea.com/macaron/gzip - [x] Use https://gitea.com/go-chi/session which is a modified version of https://gitea.com/macaron/session and removed `nodb` support since it will not be maintained. **BREAK** - [x] Use https://gitea.com/go-chi/captcha which is a modified version of https://gitea.com/macaron/captcha - [x] Use https://gitea.com/go-chi/cache which is a modified version of https://gitea.com/macaron/cache - [x] Use https://gitea.com/go-chi/binding which is a modified version of https://gitea.com/macaron/binding - [x] Use https://github.com/go-chi/cors instead of https://gitea.com/macaron/cors - [x] Dropped https://gitea.com/macaron/i18n and make a new one in `code.gitea.io/gitea/modules/translation` - [x] Move validation form structs from `code.gitea.io/gitea/modules/auth` to `code.gitea.io/gitea/modules/forms` to avoid dependency cycle. - [x] Removed macaron log service because it's not need any more. **BREAK** - [x] All form structs have to be get by `web.GetForm(ctx)` in the route function but not as a function parameter on routes definition. - [x] Move Git HTTP protocol implementation to use routers directly. - [x] Fix the problem that chi routes don't support trailing slash but macaron did. - [x] `/api/v1/swagger` now will be redirect to `/api/swagger` but not render directly so that `APIContext` will not create a html render. Notices: - Chi router don't support request with trailing slash - Integration test `TestUserHeatmap` maybe mysql version related. It's failed on my macOS(mysql 5.7.29 installed via brew) but succeed on CI. Co-authored-by: 6543 <6543@obermui.de>
2021-01-26 10:36:53 -05:00
"code.gitea.io/gitea/modules/web"
"code.gitea.io/gitea/routers"
[TESTS] createUser via the user model helper for integration tests (cherry picked from commit c1d14c5fffeb823385b2984cfcdb3e195bfb151d) (cherry picked from commit e0e8aabc985af153cf1fcb2064c17f68ec37f3a2) (cherry picked from commit 392a415070af7f4e872c7bb013f259e2dcc8b7b4) (cherry picked from commit c7cf1307ca227ff9f4935391582c058823419d7b) (cherry picked from commit 93b13d092b46c261fc3c0f70ecd9d4f45ee84029) [TESTS] createUser via the user model helper for integration tests (squash) (cherry picked from commit 6ff23839528db66ae91e88d6b687b63c769f8f43) (cherry picked from commit de2a6fe8c374965d50bcd6421d44c3a5d312a689) (cherry picked from commit 398a6ab072e8364db3c5236bf258ec6bb068dc29) (cherry picked from commit 16abc89780a228fdc288ed2ce44a95daa6368e36) (cherry picked from commit 312a3ec5d9d50864af9dbfb8672778789b2e11ca) (cherry picked from commit 85c6d8e29065641ccbaf57fcebe72b93141a14a4) (cherry picked from commit 79150d30a4d9a91e9e5272f3d46bd0ee836bcab1) (cherry picked from commit 436137962d3f150d4c28093461c30c9910f2d32d) (cherry picked from commit e4eb8d471e8e1c8e14a6a3208418bde0b121472c) (cherry picked from commit a7257052f49a480dd2ccfbef5c0573c9b5b73adb) (cherry picked from commit d5eba9a6dd93eb21350dc611edff7c9a13cb2552) (cherry picked from commit d89ef2ffa9a83249b498867a02996a3dcb17aa31) (cherry picked from commit f1d25aa30762ae97d7994048dd03c656028dedbe) (cherry picked from commit 60c7c0735376d7987717c0d375fa409ca9b90e8f) (cherry picked from commit cfeff3afdb80785019f53312369953e2d996de89) (cherry picked from commit 608ac6bd683d60fe295163ba5562aee5037b6e01) (cherry picked from commit c64e530a13fe4afc89ad3de481fff16fe190c3d1) (cherry picked from commit 513db02971fa98ff88f9b06bf25b03c31d0a857c) (cherry picked from commit 43eaaa5a61134928ff9a83e571552407450d0a09) (cherry picked from commit 678eb494408e3784336a3aeb5e5f54bd738d009e) (cherry picked from commit f7458dabda74dedecb431a75e9f049e545fc973e) (cherry picked from commit 32568e43b412bf53bc207f5b429ee4ef77693857) (cherry picked from commit 3fcbb2043d04fb701a7baa26cc599c98dfaa5601) (cherry picked from commit 95f0b8eefad590c647951f42bd0f467685901bd2) (cherry picked from commit f285b9469d0b644abc70ebbc79c4b14fe5a03e56) (cherry picked from commit 4b8a02ad088bc925dee1b2f0b759ea9ef77d96e8) (cherry picked from commit 70eb98025b6012f42c0161999f7675f069bde877) (cherry picked from commit dc2d448705404872c7f67887f7cf80658e42c3fa) (cherry picked from commit e1402ca319f549f8ed36a9f8b58f7aba09793415) (cherry picked from commit 807d587285c06b82d636fba371f946c68f0a22e3)
2023-06-05 04:43:31 -05:00
user_service "code.gitea.io/gitea/services/user"
"code.gitea.io/gitea/tests"
"github.com/PuerkitoBio/goquery"
[TESTS] coverage for SignInOAuthCallback (cherry picked from commit f8e1619b993943eafb8ee12bf06f4cdb5862de70) (cherry picked from commit 46d8bc9bdf68b53767211dc103e6130f55bcdb64) (cherry picked from commit e0c7b7055f5f4eeca84f1d0b1260b7b9622d3aab) (cherry picked from commit faab747f8e7eb09262f755445462a77f8a6fb953) (cherry picked from commit 46acb6a9a79e7ce588b2863aa37bf26805afb2b1) (cherry picked from commit 22d964e74407c52ffcd3d3a84b0a66e2c186b0fa) (cherry picked from commit 4c8a6031acf760c2383d9e103c703ee5ececb8e8) (cherry picked from commit 032e8c7a9a357a13f41410063c2f7fb925dba5ac) (cherry picked from commit 7a17a3b0fb979e2923019de4b9a7318f578b73b8) (cherry picked from commit 8ea71c2a31ea7492f5f2e3de529c7fd0b232d3e3) (cherry picked from commit 4b027e2d37cb91c5951f1d10a018778b19590eb0) (cherry picked from commit d787089a5de09fa11f8e82a66ec43e4abdde1b2e) (cherry picked from commit 7b9999357a5d34861b5fd7390cc400f497896246) (cherry picked from commit 80eb531c380914c66d30a29159b81154e7adefeb) (cherry picked from commit 373b198bfbc29855c409294ee487639f83516a55) (cherry picked from commit 15781eedf755713ad4bbc83cf0b82e899e05d075) (cherry picked from commit 46bdb17a2fb25c23336ef493449ff3ff0eb05409) (cherry picked from commit 22ec6c11ee779cc06c2e6e6dca3213129033389e) (cherry picked from commit 3f94b9a11103458d6b4f44dfda8158b748a2e3ad) (cherry picked from commit a4194c29ffcca46f20d2ccc660f8c95cf527c7a4) (cherry picked from commit aa80ba2ed1e529a85eda01beeb25c6732d2bc9bf) (cherry picked from commit d349f3e80ec764f6f402ea6183e41511f73cd33f) (cherry picked from commit ccb073f71ac855b1d7c7dd1e71a29939a14a20c5) (cherry picked from commit d8a996a9c1052a7c4b7693cb75f10ee0cbce1534) (cherry picked from commit af12965737bf60bb74fed2ca5363b034eca15fe4) (cherry picked from commit 3867b17a485e441198b248be08cbe14bb8bd3946) (cherry picked from commit 0c48072b2e19f70530d76de459bddd9e7c539c0d) (cherry picked from commit 9c5d675ded22eb2777df5b4bbd24e4b1341b8b26) (cherry picked from commit 665119370f4e9103978853c53c6ae9258a415cbc) (cherry picked from commit 658417e7ad06e8a03ee562f3c8ef8b3c2abd158e)
2023-06-27 08:00:15 -05:00
"github.com/markbates/goth"
"github.com/markbates/goth/gothic"
[TESTS] oauth2: add integration test helpers (cherry picked from commit e11dcc60f291f1b882a993f60f8381fe4561d6d0) use backticks to avoid backslash (cherry picked from commit 34212791eef2031ef09ea118a2ee5b98082174dc) (cherry picked from commit bde9473c69eaf6306457b4218d9704af64cb6cc8) (cherry picked from commit d4deb43084eec4ce0de786a01acef52921a39b13) (cherry picked from commit 08e91649b0057258ea5d775447d84093c31ad523) (cherry picked from commit 2b988e5415b35e608726facb5d23a920334fda1c) [TESTS] auth LinkAccount test coverage (squash) (cherry picked from commit a2b2e3066bee46ca15ce66d0deb7ef3e89915248) (cherry picked from commit 841d1b50731a94b9330b6a623a40f8aa0a6befa8) (cherry picked from commit 35da630ad884a9ffff5bd873123687af169a6cac) (cherry picked from commit caf2dc4fa7c6fb45a19edc5a025579d42d8db455) (cherry picked from commit 6eb81e67ba69aeb9f1290f6717ec6c6a367752c3) (cherry picked from commit d59757239f4fd6353dafd88f2460145b88ef38a1) (cherry picked from commit 38a121b6880538f381799fb69666e13abf667502) (cherry picked from commit 20613874ee04286a5ecb28045ec80af0fd850582) (cherry picked from commit 6d2705e10858baf5e33df0ced047c544ed826fd3) (cherry picked from commit f177b728142911fed6709339dd0e686017b610b0) (cherry picked from commit 75e1fc4c8318b378f94065a268b079ac152657ef) (cherry picked from commit ba64fa9867b06fb0b390a799ef4c3f39f554bb0b) (cherry picked from commit 0b8ab0893ec6b6d689534b5e4ac50cdfe36c34e9) (cherry picked from commit 1419d11435b0cdf7c41cb7175dffaf521ecfacd7) (cherry picked from commit 38766847e0441f4b3841b05b34e3442f4e23af06) (cherry picked from commit 6f23426a6ab09df7bb5817d364301975715dc10b) (cherry picked from commit 9e0ff9ca54505723ad39a3fb221b94cbcef2da66) (cherry picked from commit 353f3601c318f77a07fba0976fc9e3d28b2fc818) (cherry picked from commit 6e4ae401d815bf32ca21e2fdada5aa1ac528c756) (cherry picked from commit 1a7afe41530378cf194ce7c302cfe6bf757a2838) (cherry picked from commit f9f3e0cc02fda87ef769ee8410e9d926963d2d97) (cherry picked from commit 22fd0337f3cc57e4365c783b80db553627022f6d) (cherry picked from commit ee57e138d1a89508f7613d1e6782a9909977b153) (cherry picked from commit 21f9b7e73ddf12948feb220ec5432e14b75e0baa) (cherry picked from commit 17c548c09298472af65526f1334fecffd1e72d1e) (cherry picked from commit 02d31865174d94273e993248aa152f482fa14802) (cherry picked from commit f02a040fa27afdbcf12d197894e9adc0a8a17734) (cherry picked from commit 3cf9f82b282fe62d2124e1d3c1d75ea5f92ddce0) (cherry picked from commit aa9d06dbac2a14cde066f0c1f896c3993a49aae0) (cherry picked from commit 689421315464c16462938b3dbd710978e1fd14f3)
2023-06-05 04:29:07 -05:00
goth_gitlab "github.com/markbates/goth/providers/gitlab"
"github.com/stretchr/testify/assert"
"github.com/xeipuuv/gojsonschema"
)
var testWebRoutes *web.Route
type NilResponseRecorder struct {
httptest.ResponseRecorder
Length int
}
func (n *NilResponseRecorder) Write(b []byte) (int, error) {
2019-06-12 14:41:28 -05:00
n.Length += len(b)
return len(b), nil
}
// NewRecorder returns an initialized ResponseRecorder.
func NewNilResponseRecorder() *NilResponseRecorder {
return &NilResponseRecorder{
ResponseRecorder: *httptest.NewRecorder(),
}
}
type NilResponseHashSumRecorder struct {
httptest.ResponseRecorder
Hash hash.Hash
Length int
}
func (n *NilResponseHashSumRecorder) Write(b []byte) (int, error) {
_, _ = n.Hash.Write(b)
n.Length += len(b)
return len(b), nil
}
// NewRecorder returns an initialized ResponseRecorder.
func NewNilResponseHashSumRecorder() *NilResponseHashSumRecorder {
return &NilResponseHashSumRecorder{
Hash: fnv.New32(),
ResponseRecorder: *httptest.NewRecorder(),
}
}
func TestMain(m *testing.M) {
Rewrite logger system (#24726) ## ⚠️ Breaking The `log.<mode>.<logger>` style config has been dropped. If you used it, please check the new config manual & app.example.ini to make your instance output logs as expected. Although many legacy options still work, it's encouraged to upgrade to the new options. The SMTP logger is deleted because SMTP is not suitable to collect logs. If you have manually configured Gitea log options, please confirm the logger system works as expected after upgrading. ## Description Close #12082 and maybe more log-related issues, resolve some related FIXMEs in old code (which seems unfixable before) Just like rewriting queue #24505 : make code maintainable, clear legacy bugs, and add the ability to support more writers (eg: JSON, structured log) There is a new document (with examples): `logging-config.en-us.md` This PR is safer than the queue rewriting, because it's just for logging, it won't break other logic. ## The old problems The logging system is quite old and difficult to maintain: * Unclear concepts: Logger, NamedLogger, MultiChannelledLogger, SubLogger, EventLogger, WriterLogger etc * Some code is diffuclt to konw whether it is right: `log.DelNamedLogger("console")` vs `log.DelNamedLogger(log.DEFAULT)` vs `log.DelLogger("console")` * The old system heavily depends on ini config system, it's difficult to create new logger for different purpose, and it's very fragile. * The "color" trick is difficult to use and read, many colors are unnecessary, and in the future structured log could help * It's difficult to add other log formats, eg: JSON format * The log outputer doesn't have full control of its goroutine, it's difficult to make outputer have advanced behaviors * The logs could be lost in some cases: eg: no Fatal error when using CLI. * Config options are passed by JSON, which is quite fragile. * INI package makes the KEY in `[log]` section visible in `[log.sub1]` and `[log.sub1.subA]`, this behavior is quite fragile and would cause more unclear problems, and there is no strong requirement to support `log.<mode>.<logger>` syntax. ## The new design See `logger.go` for documents. ## Screenshot <details> ![image](https://github.com/go-gitea/gitea/assets/2114189/4462d713-ba39-41f5-bb08-de912e67e1ff) ![image](https://github.com/go-gitea/gitea/assets/2114189/b188035e-f691-428b-8b2d-ff7b2199b2f9) ![image](https://github.com/go-gitea/gitea/assets/2114189/132e9745-1c3b-4e00-9e0d-15eaea495dee) </details> ## TODO * [x] add some new tests * [x] fix some tests * [x] test some sub-commands (manually ....) --------- Co-authored-by: Jason Song <i@wolfogre.com> Co-authored-by: delvh <dev.lh@web.de> Co-authored-by: Giteabot <teabot@gitea.io>
2023-05-21 17:35:11 -05:00
defer log.GetManager().Close()
managerCtx, cancel := context.WithCancel(context.Background())
graceful.InitManager(managerCtx)
defer cancel()
tests.InitTest(true)
testWebRoutes = routers.NormalRoutes()
// integration test settings...
if setting.CfgProvider != nil {
testingCfg := setting.CfgProvider.Section("integration-tests")
Rewrite queue (#24505) # ⚠️ Breaking Many deprecated queue config options are removed (actually, they should have been removed in 1.18/1.19). If you see the fatal message when starting Gitea: "Please update your app.ini to remove deprecated config options", please follow the error messages to remove these options from your app.ini. Example: ``` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options ``` Many options in `[queue]` are are dropped, including: `WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`, `BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed from app.ini. # The problem The old queue package has some legacy problems: * complexity: I doubt few people could tell how it works. * maintainability: Too many channels and mutex/cond are mixed together, too many different structs/interfaces depends each other. * stability: due to the complexity & maintainability, sometimes there are strange bugs and difficult to debug, and some code doesn't have test (indeed some code is difficult to test because a lot of things are mixed together). * general applicability: although it is called "queue", its behavior is not a well-known queue. * scalability: it doesn't seem easy to make it work with a cluster without breaking its behaviors. It came from some very old code to "avoid breaking", however, its technical debt is too heavy now. It's a good time to introduce a better "queue" package. # The new queue package It keeps using old config and concept as much as possible. * It only contains two major kinds of concepts: * The "base queue": channel, levelqueue, redis * They have the same abstraction, the same interface, and they are tested by the same testing code. * The "WokerPoolQueue", it uses the "base queue" to provide "worker pool" function, calls the "handler" to process the data in the base queue. * The new code doesn't do "PushBack" * Think about a queue with many workers, the "PushBack" can't guarantee the order for re-queued unhandled items, so in new code it just does "normal push" * The new code doesn't do "pause/resume" * The "pause/resume" was designed to handle some handler's failure: eg: document indexer (elasticsearch) is down * If a queue is paused for long time, either the producers blocks or the new items are dropped. * The new code doesn't do such "pause/resume" trick, it's not a common queue's behavior and it doesn't help much. * If there are unhandled items, the "push" function just blocks for a few seconds and then re-queue them and retry. * The new code doesn't do "worker booster" * Gitea's queue's handlers are light functions, the cost is only the go-routine, so it doesn't make sense to "boost" them. * The new code only use "max worker number" to limit the concurrent workers. * The new "Push" never blocks forever * Instead of creating more and more blocking goroutines, return an error is more friendly to the server and to the end user. There are more details in code comments: eg: the "Flush" problem, the strange "code.index" hanging problem, the "immediate" queue problem. Almost ready for review. TODO: * [x] add some necessary comments during review * [x] add some more tests if necessary * [x] update documents and config options * [x] test max worker / active worker * [x] re-run the CI tasks to see whether any test is flaky * [x] improve the `handleOldLengthConfiguration` to provide more friendly messages * [x] fine tune default config values (eg: length?) ## Code coverage: ![image](https://user-images.githubusercontent.com/2114189/236620635-55576955-f95d-4810-b12f-879026a3afdf.png)
2023-05-08 06:49:59 -05:00
testlogger.SlowTest = testingCfg.Key("SLOW_TEST").MustDuration(testlogger.SlowTest)
testlogger.SlowFlush = testingCfg.Key("SLOW_FLUSH").MustDuration(testlogger.SlowFlush)
}
if os.Getenv("GITEA_SLOW_TEST_TIME") != "" {
duration, err := time.ParseDuration(os.Getenv("GITEA_SLOW_TEST_TIME"))
if err == nil {
Rewrite queue (#24505) # ⚠️ Breaking Many deprecated queue config options are removed (actually, they should have been removed in 1.18/1.19). If you see the fatal message when starting Gitea: "Please update your app.ini to remove deprecated config options", please follow the error messages to remove these options from your app.ini. Example: ``` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options ``` Many options in `[queue]` are are dropped, including: `WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`, `BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed from app.ini. # The problem The old queue package has some legacy problems: * complexity: I doubt few people could tell how it works. * maintainability: Too many channels and mutex/cond are mixed together, too many different structs/interfaces depends each other. * stability: due to the complexity & maintainability, sometimes there are strange bugs and difficult to debug, and some code doesn't have test (indeed some code is difficult to test because a lot of things are mixed together). * general applicability: although it is called "queue", its behavior is not a well-known queue. * scalability: it doesn't seem easy to make it work with a cluster without breaking its behaviors. It came from some very old code to "avoid breaking", however, its technical debt is too heavy now. It's a good time to introduce a better "queue" package. # The new queue package It keeps using old config and concept as much as possible. * It only contains two major kinds of concepts: * The "base queue": channel, levelqueue, redis * They have the same abstraction, the same interface, and they are tested by the same testing code. * The "WokerPoolQueue", it uses the "base queue" to provide "worker pool" function, calls the "handler" to process the data in the base queue. * The new code doesn't do "PushBack" * Think about a queue with many workers, the "PushBack" can't guarantee the order for re-queued unhandled items, so in new code it just does "normal push" * The new code doesn't do "pause/resume" * The "pause/resume" was designed to handle some handler's failure: eg: document indexer (elasticsearch) is down * If a queue is paused for long time, either the producers blocks or the new items are dropped. * The new code doesn't do such "pause/resume" trick, it's not a common queue's behavior and it doesn't help much. * If there are unhandled items, the "push" function just blocks for a few seconds and then re-queue them and retry. * The new code doesn't do "worker booster" * Gitea's queue's handlers are light functions, the cost is only the go-routine, so it doesn't make sense to "boost" them. * The new code only use "max worker number" to limit the concurrent workers. * The new "Push" never blocks forever * Instead of creating more and more blocking goroutines, return an error is more friendly to the server and to the end user. There are more details in code comments: eg: the "Flush" problem, the strange "code.index" hanging problem, the "immediate" queue problem. Almost ready for review. TODO: * [x] add some necessary comments during review * [x] add some more tests if necessary * [x] update documents and config options * [x] test max worker / active worker * [x] re-run the CI tasks to see whether any test is flaky * [x] improve the `handleOldLengthConfiguration` to provide more friendly messages * [x] fine tune default config values (eg: length?) ## Code coverage: ![image](https://user-images.githubusercontent.com/2114189/236620635-55576955-f95d-4810-b12f-879026a3afdf.png)
2023-05-08 06:49:59 -05:00
testlogger.SlowTest = duration
}
}
if os.Getenv("GITEA_SLOW_FLUSH_TIME") != "" {
duration, err := time.ParseDuration(os.Getenv("GITEA_SLOW_FLUSH_TIME"))
if err == nil {
Rewrite queue (#24505) # ⚠️ Breaking Many deprecated queue config options are removed (actually, they should have been removed in 1.18/1.19). If you see the fatal message when starting Gitea: "Please update your app.ini to remove deprecated config options", please follow the error messages to remove these options from your app.ini. Example: ``` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options ``` Many options in `[queue]` are are dropped, including: `WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`, `BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed from app.ini. # The problem The old queue package has some legacy problems: * complexity: I doubt few people could tell how it works. * maintainability: Too many channels and mutex/cond are mixed together, too many different structs/interfaces depends each other. * stability: due to the complexity & maintainability, sometimes there are strange bugs and difficult to debug, and some code doesn't have test (indeed some code is difficult to test because a lot of things are mixed together). * general applicability: although it is called "queue", its behavior is not a well-known queue. * scalability: it doesn't seem easy to make it work with a cluster without breaking its behaviors. It came from some very old code to "avoid breaking", however, its technical debt is too heavy now. It's a good time to introduce a better "queue" package. # The new queue package It keeps using old config and concept as much as possible. * It only contains two major kinds of concepts: * The "base queue": channel, levelqueue, redis * They have the same abstraction, the same interface, and they are tested by the same testing code. * The "WokerPoolQueue", it uses the "base queue" to provide "worker pool" function, calls the "handler" to process the data in the base queue. * The new code doesn't do "PushBack" * Think about a queue with many workers, the "PushBack" can't guarantee the order for re-queued unhandled items, so in new code it just does "normal push" * The new code doesn't do "pause/resume" * The "pause/resume" was designed to handle some handler's failure: eg: document indexer (elasticsearch) is down * If a queue is paused for long time, either the producers blocks or the new items are dropped. * The new code doesn't do such "pause/resume" trick, it's not a common queue's behavior and it doesn't help much. * If there are unhandled items, the "push" function just blocks for a few seconds and then re-queue them and retry. * The new code doesn't do "worker booster" * Gitea's queue's handlers are light functions, the cost is only the go-routine, so it doesn't make sense to "boost" them. * The new code only use "max worker number" to limit the concurrent workers. * The new "Push" never blocks forever * Instead of creating more and more blocking goroutines, return an error is more friendly to the server and to the end user. There are more details in code comments: eg: the "Flush" problem, the strange "code.index" hanging problem, the "immediate" queue problem. Almost ready for review. TODO: * [x] add some necessary comments during review * [x] add some more tests if necessary * [x] update documents and config options * [x] test max worker / active worker * [x] re-run the CI tasks to see whether any test is flaky * [x] improve the `handleOldLengthConfiguration` to provide more friendly messages * [x] fine tune default config values (eg: length?) ## Code coverage: ![image](https://user-images.githubusercontent.com/2114189/236620635-55576955-f95d-4810-b12f-879026a3afdf.png)
2023-05-08 06:49:59 -05:00
testlogger.SlowFlush = duration
}
}
os.Unsetenv("GIT_AUTHOR_NAME")
os.Unsetenv("GIT_AUTHOR_EMAIL")
os.Unsetenv("GIT_AUTHOR_DATE")
os.Unsetenv("GIT_COMMITTER_NAME")
os.Unsetenv("GIT_COMMITTER_EMAIL")
os.Unsetenv("GIT_COMMITTER_DATE")
err := unittest.InitFixtures(
unittest.FixturesOptions{
Dir: filepath.Join(filepath.Dir(setting.AppPath), "models/fixtures/"),
},
)
if err != nil {
fmt.Printf("Error initializing test database: %v\n", err)
os.Exit(1)
}
// FIXME: the console logger is deleted by mistake, so if there is any `log.Fatal`, developers won't see any error message.
// Instead, "No tests were found", last nonsense log is "According to the configuration, subsequent logs will not be printed to the console"
exitCode := m.Run()
Rewrite queue (#24505) # ⚠️ Breaking Many deprecated queue config options are removed (actually, they should have been removed in 1.18/1.19). If you see the fatal message when starting Gitea: "Please update your app.ini to remove deprecated config options", please follow the error messages to remove these options from your app.ini. Example: ``` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options ``` Many options in `[queue]` are are dropped, including: `WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`, `BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed from app.ini. # The problem The old queue package has some legacy problems: * complexity: I doubt few people could tell how it works. * maintainability: Too many channels and mutex/cond are mixed together, too many different structs/interfaces depends each other. * stability: due to the complexity & maintainability, sometimes there are strange bugs and difficult to debug, and some code doesn't have test (indeed some code is difficult to test because a lot of things are mixed together). * general applicability: although it is called "queue", its behavior is not a well-known queue. * scalability: it doesn't seem easy to make it work with a cluster without breaking its behaviors. It came from some very old code to "avoid breaking", however, its technical debt is too heavy now. It's a good time to introduce a better "queue" package. # The new queue package It keeps using old config and concept as much as possible. * It only contains two major kinds of concepts: * The "base queue": channel, levelqueue, redis * They have the same abstraction, the same interface, and they are tested by the same testing code. * The "WokerPoolQueue", it uses the "base queue" to provide "worker pool" function, calls the "handler" to process the data in the base queue. * The new code doesn't do "PushBack" * Think about a queue with many workers, the "PushBack" can't guarantee the order for re-queued unhandled items, so in new code it just does "normal push" * The new code doesn't do "pause/resume" * The "pause/resume" was designed to handle some handler's failure: eg: document indexer (elasticsearch) is down * If a queue is paused for long time, either the producers blocks or the new items are dropped. * The new code doesn't do such "pause/resume" trick, it's not a common queue's behavior and it doesn't help much. * If there are unhandled items, the "push" function just blocks for a few seconds and then re-queue them and retry. * The new code doesn't do "worker booster" * Gitea's queue's handlers are light functions, the cost is only the go-routine, so it doesn't make sense to "boost" them. * The new code only use "max worker number" to limit the concurrent workers. * The new "Push" never blocks forever * Instead of creating more and more blocking goroutines, return an error is more friendly to the server and to the end user. There are more details in code comments: eg: the "Flush" problem, the strange "code.index" hanging problem, the "immediate" queue problem. Almost ready for review. TODO: * [x] add some necessary comments during review * [x] add some more tests if necessary * [x] update documents and config options * [x] test max worker / active worker * [x] re-run the CI tasks to see whether any test is flaky * [x] improve the `handleOldLengthConfiguration` to provide more friendly messages * [x] fine tune default config values (eg: length?) ## Code coverage: ![image](https://user-images.githubusercontent.com/2114189/236620635-55576955-f95d-4810-b12f-879026a3afdf.png)
2023-05-08 06:49:59 -05:00
testlogger.WriterCloser.Reset()
if err = util.RemoveAll(setting.Indexer.IssuePath); err != nil {
fmt.Printf("util.RemoveAll: %v\n", err)
os.Exit(1)
}
if err = util.RemoveAll(setting.Indexer.RepoPath); err != nil {
fmt.Printf("Unable to remove repo indexer: %v\n", err)
os.Exit(1)
}
os.Exit(exitCode)
}
type TestSession struct {
jar http.CookieJar
}
func (s *TestSession) GetCookie(name string) *http.Cookie {
baseURL, err := url.Parse(setting.AppURL)
if err != nil {
return nil
}
for _, c := range s.jar.Cookies(baseURL) {
if c.Name == name {
return c
}
}
return nil
}
func (s *TestSession) MakeRequest(t testing.TB, rw *RequestWrapper, expectedStatus int) *httptest.ResponseRecorder {
t.Helper()
req := rw.Request
baseURL, err := url.Parse(setting.AppURL)
assert.NoError(t, err)
for _, c := range s.jar.Cookies(baseURL) {
req.AddCookie(c)
}
resp := MakeRequest(t, rw, expectedStatus)
ch := http.Header{}
2019-06-12 14:41:28 -05:00
ch.Add("Cookie", strings.Join(resp.Header()["Set-Cookie"], ";"))
cr := http.Request{Header: ch}
s.jar.SetCookies(baseURL, cr.Cookies())
return resp
}
func (s *TestSession) MakeRequestNilResponseRecorder(t testing.TB, rw *RequestWrapper, expectedStatus int) *NilResponseRecorder {
t.Helper()
req := rw.Request
baseURL, err := url.Parse(setting.AppURL)
assert.NoError(t, err)
for _, c := range s.jar.Cookies(baseURL) {
req.AddCookie(c)
}
resp := MakeRequestNilResponseRecorder(t, rw, expectedStatus)
ch := http.Header{}
2019-06-12 14:41:28 -05:00
ch.Add("Cookie", strings.Join(resp.Header()["Set-Cookie"], ";"))
cr := http.Request{Header: ch}
s.jar.SetCookies(baseURL, cr.Cookies())
return resp
}
func (s *TestSession) MakeRequestNilResponseHashSumRecorder(t testing.TB, rw *RequestWrapper, expectedStatus int) *NilResponseHashSumRecorder {
t.Helper()
req := rw.Request
baseURL, err := url.Parse(setting.AppURL)
assert.NoError(t, err)
for _, c := range s.jar.Cookies(baseURL) {
req.AddCookie(c)
}
resp := MakeRequestNilResponseHashSumRecorder(t, rw, expectedStatus)
ch := http.Header{}
ch.Add("Cookie", strings.Join(resp.Header()["Set-Cookie"], ";"))
cr := http.Request{Header: ch}
s.jar.SetCookies(baseURL, cr.Cookies())
return resp
}
const userPassword = "password"
func emptyTestSession(t testing.TB) *TestSession {
t.Helper()
jar, err := cookiejar.New(nil)
assert.NoError(t, err)
return &TestSession{jar: jar}
}
func getUserToken(t testing.TB, userName string, scope ...auth.AccessTokenScope) string {
return getTokenForLoggedInUser(t, loginUser(t, userName), scope...)
}
[TESTS] coverage for SignInOAuthCallback (cherry picked from commit f8e1619b993943eafb8ee12bf06f4cdb5862de70) (cherry picked from commit 46d8bc9bdf68b53767211dc103e6130f55bcdb64) (cherry picked from commit e0c7b7055f5f4eeca84f1d0b1260b7b9622d3aab) (cherry picked from commit faab747f8e7eb09262f755445462a77f8a6fb953) (cherry picked from commit 46acb6a9a79e7ce588b2863aa37bf26805afb2b1) (cherry picked from commit 22d964e74407c52ffcd3d3a84b0a66e2c186b0fa) (cherry picked from commit 4c8a6031acf760c2383d9e103c703ee5ececb8e8) (cherry picked from commit 032e8c7a9a357a13f41410063c2f7fb925dba5ac) (cherry picked from commit 7a17a3b0fb979e2923019de4b9a7318f578b73b8) (cherry picked from commit 8ea71c2a31ea7492f5f2e3de529c7fd0b232d3e3) (cherry picked from commit 4b027e2d37cb91c5951f1d10a018778b19590eb0) (cherry picked from commit d787089a5de09fa11f8e82a66ec43e4abdde1b2e) (cherry picked from commit 7b9999357a5d34861b5fd7390cc400f497896246) (cherry picked from commit 80eb531c380914c66d30a29159b81154e7adefeb) (cherry picked from commit 373b198bfbc29855c409294ee487639f83516a55) (cherry picked from commit 15781eedf755713ad4bbc83cf0b82e899e05d075) (cherry picked from commit 46bdb17a2fb25c23336ef493449ff3ff0eb05409) (cherry picked from commit 22ec6c11ee779cc06c2e6e6dca3213129033389e) (cherry picked from commit 3f94b9a11103458d6b4f44dfda8158b748a2e3ad) (cherry picked from commit a4194c29ffcca46f20d2ccc660f8c95cf527c7a4) (cherry picked from commit aa80ba2ed1e529a85eda01beeb25c6732d2bc9bf) (cherry picked from commit d349f3e80ec764f6f402ea6183e41511f73cd33f) (cherry picked from commit ccb073f71ac855b1d7c7dd1e71a29939a14a20c5) (cherry picked from commit d8a996a9c1052a7c4b7693cb75f10ee0cbce1534) (cherry picked from commit af12965737bf60bb74fed2ca5363b034eca15fe4) (cherry picked from commit 3867b17a485e441198b248be08cbe14bb8bd3946) (cherry picked from commit 0c48072b2e19f70530d76de459bddd9e7c539c0d) (cherry picked from commit 9c5d675ded22eb2777df5b4bbd24e4b1341b8b26) (cherry picked from commit 665119370f4e9103978853c53c6ae9258a415cbc) (cherry picked from commit 658417e7ad06e8a03ee562f3c8ef8b3c2abd158e)
2023-06-27 08:00:15 -05:00
func mockCompleteUserAuth(mock func(res http.ResponseWriter, req *http.Request) (goth.User, error)) func() {
old := gothic.CompleteUserAuth
gothic.CompleteUserAuth = mock
return func() {
gothic.CompleteUserAuth = old
}
}
[TESTS] oauth2: add integration test helpers (cherry picked from commit e11dcc60f291f1b882a993f60f8381fe4561d6d0) use backticks to avoid backslash (cherry picked from commit 34212791eef2031ef09ea118a2ee5b98082174dc) (cherry picked from commit bde9473c69eaf6306457b4218d9704af64cb6cc8) (cherry picked from commit d4deb43084eec4ce0de786a01acef52921a39b13) (cherry picked from commit 08e91649b0057258ea5d775447d84093c31ad523) (cherry picked from commit 2b988e5415b35e608726facb5d23a920334fda1c) [TESTS] auth LinkAccount test coverage (squash) (cherry picked from commit a2b2e3066bee46ca15ce66d0deb7ef3e89915248) (cherry picked from commit 841d1b50731a94b9330b6a623a40f8aa0a6befa8) (cherry picked from commit 35da630ad884a9ffff5bd873123687af169a6cac) (cherry picked from commit caf2dc4fa7c6fb45a19edc5a025579d42d8db455) (cherry picked from commit 6eb81e67ba69aeb9f1290f6717ec6c6a367752c3) (cherry picked from commit d59757239f4fd6353dafd88f2460145b88ef38a1) (cherry picked from commit 38a121b6880538f381799fb69666e13abf667502) (cherry picked from commit 20613874ee04286a5ecb28045ec80af0fd850582) (cherry picked from commit 6d2705e10858baf5e33df0ced047c544ed826fd3) (cherry picked from commit f177b728142911fed6709339dd0e686017b610b0) (cherry picked from commit 75e1fc4c8318b378f94065a268b079ac152657ef) (cherry picked from commit ba64fa9867b06fb0b390a799ef4c3f39f554bb0b) (cherry picked from commit 0b8ab0893ec6b6d689534b5e4ac50cdfe36c34e9) (cherry picked from commit 1419d11435b0cdf7c41cb7175dffaf521ecfacd7) (cherry picked from commit 38766847e0441f4b3841b05b34e3442f4e23af06) (cherry picked from commit 6f23426a6ab09df7bb5817d364301975715dc10b) (cherry picked from commit 9e0ff9ca54505723ad39a3fb221b94cbcef2da66) (cherry picked from commit 353f3601c318f77a07fba0976fc9e3d28b2fc818) (cherry picked from commit 6e4ae401d815bf32ca21e2fdada5aa1ac528c756) (cherry picked from commit 1a7afe41530378cf194ce7c302cfe6bf757a2838) (cherry picked from commit f9f3e0cc02fda87ef769ee8410e9d926963d2d97) (cherry picked from commit 22fd0337f3cc57e4365c783b80db553627022f6d) (cherry picked from commit ee57e138d1a89508f7613d1e6782a9909977b153) (cherry picked from commit 21f9b7e73ddf12948feb220ec5432e14b75e0baa) (cherry picked from commit 17c548c09298472af65526f1334fecffd1e72d1e) (cherry picked from commit 02d31865174d94273e993248aa152f482fa14802) (cherry picked from commit f02a040fa27afdbcf12d197894e9adc0a8a17734) (cherry picked from commit 3cf9f82b282fe62d2124e1d3c1d75ea5f92ddce0) (cherry picked from commit aa9d06dbac2a14cde066f0c1f896c3993a49aae0) (cherry picked from commit 689421315464c16462938b3dbd710978e1fd14f3)
2023-06-05 04:29:07 -05:00
func addAuthSource(t *testing.T, payload map[string]string) *auth.Source {
session := loginUser(t, "user1")
payload["_csrf"] = GetCSRF(t, session, "/admin/auths/new")
req := NewRequestWithValues(t, "POST", "/admin/auths/new", payload)
session.MakeRequest(t, req, http.StatusSeeOther)
source, err := auth.GetSourceByName(context.Background(), payload["name"])
assert.NoError(t, err)
return source
}
func authSourcePayloadOAuth2(name string) map[string]string {
return map[string]string{
"type": fmt.Sprintf("%d", auth.OAuth2),
"name": name,
"is_active": "on",
}
}
func authSourcePayloadGitLab(name string) map[string]string {
payload := authSourcePayloadOAuth2(name)
payload["oauth2_provider"] = "gitlab"
return payload
}
func authSourcePayloadGitLabCustom(name string) map[string]string {
payload := authSourcePayloadGitLab(name)
payload["oauth2_use_custom_url"] = "on"
payload["oauth2_auth_url"] = goth_gitlab.AuthURL
payload["oauth2_token_url"] = goth_gitlab.TokenURL
payload["oauth2_profile_url"] = goth_gitlab.ProfileURL
return payload
}
[TESTS] createUser via the user model helper for integration tests (cherry picked from commit c1d14c5fffeb823385b2984cfcdb3e195bfb151d) (cherry picked from commit e0e8aabc985af153cf1fcb2064c17f68ec37f3a2) (cherry picked from commit 392a415070af7f4e872c7bb013f259e2dcc8b7b4) (cherry picked from commit c7cf1307ca227ff9f4935391582c058823419d7b) (cherry picked from commit 93b13d092b46c261fc3c0f70ecd9d4f45ee84029) [TESTS] createUser via the user model helper for integration tests (squash) (cherry picked from commit 6ff23839528db66ae91e88d6b687b63c769f8f43) (cherry picked from commit de2a6fe8c374965d50bcd6421d44c3a5d312a689) (cherry picked from commit 398a6ab072e8364db3c5236bf258ec6bb068dc29) (cherry picked from commit 16abc89780a228fdc288ed2ce44a95daa6368e36) (cherry picked from commit 312a3ec5d9d50864af9dbfb8672778789b2e11ca) (cherry picked from commit 85c6d8e29065641ccbaf57fcebe72b93141a14a4) (cherry picked from commit 79150d30a4d9a91e9e5272f3d46bd0ee836bcab1) (cherry picked from commit 436137962d3f150d4c28093461c30c9910f2d32d) (cherry picked from commit e4eb8d471e8e1c8e14a6a3208418bde0b121472c) (cherry picked from commit a7257052f49a480dd2ccfbef5c0573c9b5b73adb) (cherry picked from commit d5eba9a6dd93eb21350dc611edff7c9a13cb2552) (cherry picked from commit d89ef2ffa9a83249b498867a02996a3dcb17aa31) (cherry picked from commit f1d25aa30762ae97d7994048dd03c656028dedbe) (cherry picked from commit 60c7c0735376d7987717c0d375fa409ca9b90e8f) (cherry picked from commit cfeff3afdb80785019f53312369953e2d996de89) (cherry picked from commit 608ac6bd683d60fe295163ba5562aee5037b6e01) (cherry picked from commit c64e530a13fe4afc89ad3de481fff16fe190c3d1) (cherry picked from commit 513db02971fa98ff88f9b06bf25b03c31d0a857c) (cherry picked from commit 43eaaa5a61134928ff9a83e571552407450d0a09) (cherry picked from commit 678eb494408e3784336a3aeb5e5f54bd738d009e) (cherry picked from commit f7458dabda74dedecb431a75e9f049e545fc973e) (cherry picked from commit 32568e43b412bf53bc207f5b429ee4ef77693857) (cherry picked from commit 3fcbb2043d04fb701a7baa26cc599c98dfaa5601) (cherry picked from commit 95f0b8eefad590c647951f42bd0f467685901bd2) (cherry picked from commit f285b9469d0b644abc70ebbc79c4b14fe5a03e56) (cherry picked from commit 4b8a02ad088bc925dee1b2f0b759ea9ef77d96e8) (cherry picked from commit 70eb98025b6012f42c0161999f7675f069bde877) (cherry picked from commit dc2d448705404872c7f67887f7cf80658e42c3fa) (cherry picked from commit e1402ca319f549f8ed36a9f8b58f7aba09793415) (cherry picked from commit 807d587285c06b82d636fba371f946c68f0a22e3)
2023-06-05 04:43:31 -05:00
func createUser(ctx context.Context, t testing.TB, user *user_model.User) func() {
user.MustChangePassword = false
user.LowerName = strings.ToLower(user.Name)
assert.NoError(t, db.Insert(ctx, user))
if len(user.Email) > 0 {
assert.NoError(t, user_service.ReplacePrimaryEmailAddress(ctx, user, user.Email))
[TESTS] createUser via the user model helper for integration tests (cherry picked from commit c1d14c5fffeb823385b2984cfcdb3e195bfb151d) (cherry picked from commit e0e8aabc985af153cf1fcb2064c17f68ec37f3a2) (cherry picked from commit 392a415070af7f4e872c7bb013f259e2dcc8b7b4) (cherry picked from commit c7cf1307ca227ff9f4935391582c058823419d7b) (cherry picked from commit 93b13d092b46c261fc3c0f70ecd9d4f45ee84029) [TESTS] createUser via the user model helper for integration tests (squash) (cherry picked from commit 6ff23839528db66ae91e88d6b687b63c769f8f43) (cherry picked from commit de2a6fe8c374965d50bcd6421d44c3a5d312a689) (cherry picked from commit 398a6ab072e8364db3c5236bf258ec6bb068dc29) (cherry picked from commit 16abc89780a228fdc288ed2ce44a95daa6368e36) (cherry picked from commit 312a3ec5d9d50864af9dbfb8672778789b2e11ca) (cherry picked from commit 85c6d8e29065641ccbaf57fcebe72b93141a14a4) (cherry picked from commit 79150d30a4d9a91e9e5272f3d46bd0ee836bcab1) (cherry picked from commit 436137962d3f150d4c28093461c30c9910f2d32d) (cherry picked from commit e4eb8d471e8e1c8e14a6a3208418bde0b121472c) (cherry picked from commit a7257052f49a480dd2ccfbef5c0573c9b5b73adb) (cherry picked from commit d5eba9a6dd93eb21350dc611edff7c9a13cb2552) (cherry picked from commit d89ef2ffa9a83249b498867a02996a3dcb17aa31) (cherry picked from commit f1d25aa30762ae97d7994048dd03c656028dedbe) (cherry picked from commit 60c7c0735376d7987717c0d375fa409ca9b90e8f) (cherry picked from commit cfeff3afdb80785019f53312369953e2d996de89) (cherry picked from commit 608ac6bd683d60fe295163ba5562aee5037b6e01) (cherry picked from commit c64e530a13fe4afc89ad3de481fff16fe190c3d1) (cherry picked from commit 513db02971fa98ff88f9b06bf25b03c31d0a857c) (cherry picked from commit 43eaaa5a61134928ff9a83e571552407450d0a09) (cherry picked from commit 678eb494408e3784336a3aeb5e5f54bd738d009e) (cherry picked from commit f7458dabda74dedecb431a75e9f049e545fc973e) (cherry picked from commit 32568e43b412bf53bc207f5b429ee4ef77693857) (cherry picked from commit 3fcbb2043d04fb701a7baa26cc599c98dfaa5601) (cherry picked from commit 95f0b8eefad590c647951f42bd0f467685901bd2) (cherry picked from commit f285b9469d0b644abc70ebbc79c4b14fe5a03e56) (cherry picked from commit 4b8a02ad088bc925dee1b2f0b759ea9ef77d96e8) (cherry picked from commit 70eb98025b6012f42c0161999f7675f069bde877) (cherry picked from commit dc2d448705404872c7f67887f7cf80658e42c3fa) (cherry picked from commit e1402ca319f549f8ed36a9f8b58f7aba09793415) (cherry picked from commit 807d587285c06b82d636fba371f946c68f0a22e3)
2023-06-05 04:43:31 -05:00
}
return func() {
assert.NoError(t, user_service.DeleteUser(ctx, user, true))
}
}
func loginUser(t testing.TB, userName string) *TestSession {
t.Helper()
return loginUserWithPassword(t, userName, userPassword)
}
func loginUserWithPassword(t testing.TB, userName, password string) *TestSession {
t.Helper()
[SECURITY] Rework long-term authentication - This is a 'front-port' of the already existing patch on v1.21 and v1.20, but applied on top of what Gitea has done to rework the LTA mechanism. Forgejo will stick with the reworked mechanism by the Forgejo Security team for the time being. The removal of legacy code (AES-GCM) has been left out. - The current architecture is inherently insecure, because you can construct the 'secret' cookie value with values that are available in the database. Thus provides zero protection when a database is dumped/leaked. - This patch implements a new architecture that's inspired from: [Paragonie Initiative](https://paragonie.com/blog/2015/04/secure-authentication-php-with-long-term-persistence#secure-remember-me-cookies). - Integration testing is added to ensure the new mechanism works. - Removes a setting, because it's not used anymore. (cherry picked from commit e3d6622a63da9c33eed1e3d102cf28a92ff653d6) (cherry picked from commit fef1a6dac5e25579e42d40209c4cfc06879948b9) (cherry picked from commit b0c5165145fa52f2f7bbec1f50b308bdf1d20ef3) (cherry picked from commit 7ad51b9f8d0647eecacd258f6ee26155da3872e1) (cherry picked from commit 64f053f3834e764112cde26bb0d16c5e88d6b2af) (cherry picked from commit f5e78e4c204ce50b800645d614218b6b6096eecb) Conflicts: services/auth/auth_token_test.go https://codeberg.org/forgejo/forgejo/pulls/2069 (cherry picked from commit f69fc23d4bbadf388c7857040ee0774b824e418e) (cherry picked from commit d955ab3ab02cbb7f1245a8cddec426d64d3ac500) (cherry picked from commit 9220088f902a25c4690bcabf5a40a8d02e784182) (cherry picked from commit c73ac636962c41c71814c273510146f0533264ab) (cherry picked from commit 747a176048ea93085b406429db0e25bb21912eda) Conflicts: models/user/user.go routers/web/user/setting/account.go https://codeberg.org/forgejo/forgejo/pulls/2295
2023-11-22 11:26:21 -05:00
return loginUserWithPasswordRemember(t, userName, password, false)
}
func loginUserWithPasswordRemember(t testing.TB, userName, password string, rememberMe bool) *TestSession {
t.Helper()
req := NewRequest(t, "GET", "/user/login")
resp := MakeRequest(t, req, http.StatusOK)
doc := NewHTMLParser(t, resp.Body)
req = NewRequestWithValues(t, "POST", "/user/login", map[string]string{
"_csrf": doc.GetCSRF(),
"user_name": userName,
"password": password,
[SECURITY] Rework long-term authentication - This is a 'front-port' of the already existing patch on v1.21 and v1.20, but applied on top of what Gitea has done to rework the LTA mechanism. Forgejo will stick with the reworked mechanism by the Forgejo Security team for the time being. The removal of legacy code (AES-GCM) has been left out. - The current architecture is inherently insecure, because you can construct the 'secret' cookie value with values that are available in the database. Thus provides zero protection when a database is dumped/leaked. - This patch implements a new architecture that's inspired from: [Paragonie Initiative](https://paragonie.com/blog/2015/04/secure-authentication-php-with-long-term-persistence#secure-remember-me-cookies). - Integration testing is added to ensure the new mechanism works. - Removes a setting, because it's not used anymore. (cherry picked from commit e3d6622a63da9c33eed1e3d102cf28a92ff653d6) (cherry picked from commit fef1a6dac5e25579e42d40209c4cfc06879948b9) (cherry picked from commit b0c5165145fa52f2f7bbec1f50b308bdf1d20ef3) (cherry picked from commit 7ad51b9f8d0647eecacd258f6ee26155da3872e1) (cherry picked from commit 64f053f3834e764112cde26bb0d16c5e88d6b2af) (cherry picked from commit f5e78e4c204ce50b800645d614218b6b6096eecb) Conflicts: services/auth/auth_token_test.go https://codeberg.org/forgejo/forgejo/pulls/2069 (cherry picked from commit f69fc23d4bbadf388c7857040ee0774b824e418e) (cherry picked from commit d955ab3ab02cbb7f1245a8cddec426d64d3ac500) (cherry picked from commit 9220088f902a25c4690bcabf5a40a8d02e784182) (cherry picked from commit c73ac636962c41c71814c273510146f0533264ab) (cherry picked from commit 747a176048ea93085b406429db0e25bb21912eda) Conflicts: models/user/user.go routers/web/user/setting/account.go https://codeberg.org/forgejo/forgejo/pulls/2295
2023-11-22 11:26:21 -05:00
"remember": strconv.FormatBool(rememberMe),
})
resp = MakeRequest(t, req, http.StatusSeeOther)
ch := http.Header{}
2019-06-12 14:41:28 -05:00
ch.Add("Cookie", strings.Join(resp.Header()["Set-Cookie"], ";"))
cr := http.Request{Header: ch}
session := emptyTestSession(t)
baseURL, err := url.Parse(setting.AppURL)
assert.NoError(t, err)
session.jar.SetCookies(baseURL, cr.Cookies())
return session
}
// token has to be unique this counter take care of
var tokenCounter int64
// getTokenForLoggedInUser returns a token for a logged in user.
// The scope is an optional list of snake_case strings like the frontend form fields,
// but without the "scope_" prefix.
func getTokenForLoggedInUser(t testing.TB, session *TestSession, scopes ...auth.AccessTokenScope) string {
t.Helper()
var token string
req := NewRequest(t, "GET", "/user/settings/applications")
resp := session.MakeRequest(t, req, http.StatusOK)
var csrf string
for _, cookie := range resp.Result().Cookies() {
if cookie.Name != "_csrf" {
continue
}
csrf = cookie.Value
break
}
if csrf == "" {
doc := NewHTMLParser(t, resp.Body)
csrf = doc.GetCSRF()
}
assert.NotEmpty(t, csrf)
urlValues := url.Values{}
urlValues.Add("_csrf", csrf)
urlValues.Add("name", fmt.Sprintf("api-testing-token-%d", atomic.AddInt64(&tokenCounter, 1)))
for _, scope := range scopes {
urlValues.Add("scope", string(scope))
}
req = NewRequestWithURLValues(t, "POST", "/user/settings/applications", urlValues)
resp = session.MakeRequest(t, req, http.StatusSeeOther)
// Log the flash values on failure
if !assert.Equal(t, resp.Result().Header["Location"], []string{"/user/settings/applications"}) {
for _, cookie := range resp.Result().Cookies() {
if cookie.Name != gitea_context.CookieNameFlash {
continue
}
flash, _ := url.ParseQuery(cookie.Value)
for key, value := range flash {
t.Logf("Flash %q: %q", key, value)
}
}
}
req = NewRequest(t, "GET", "/user/settings/applications")
resp = session.MakeRequest(t, req, http.StatusOK)
htmlDoc := NewHTMLParser(t, resp.Body)
token = htmlDoc.doc.Find(".ui.info p").Text()
assert.NotEmpty(t, token)
return token
}
type RequestWrapper struct {
*http.Request
}
func (req *RequestWrapper) AddBasicAuth(username string) *RequestWrapper {
req.Request.SetBasicAuth(username, userPassword)
return req
}
func (req *RequestWrapper) AddTokenAuth(token string) *RequestWrapper {
if token == "" {
return req
}
if !strings.HasPrefix(token, "Bearer ") {
token = "Bearer " + token
}
req.Request.Header.Set("Authorization", token)
return req
}
func (req *RequestWrapper) SetHeader(name, value string) *RequestWrapper {
req.Request.Header.Set(name, value)
return req
}
func NewRequest(t testing.TB, method, urlStr string) *RequestWrapper {
t.Helper()
return NewRequestWithBody(t, method, urlStr, nil)
}
func NewRequestf(t testing.TB, method, urlFormat string, args ...any) *RequestWrapper {
t.Helper()
return NewRequest(t, method, fmt.Sprintf(urlFormat, args...))
}
func NewRequestWithValues(t testing.TB, method, urlStr string, values map[string]string) *RequestWrapper {
t.Helper()
urlValues := url.Values{}
for key, value := range values {
urlValues[key] = []string{value}
}
return NewRequestWithURLValues(t, method, urlStr, urlValues)
}
func NewRequestWithURLValues(t testing.TB, method, urlStr string, urlValues url.Values) *RequestWrapper {
t.Helper()
return NewRequestWithBody(t, method, urlStr, bytes.NewBufferString(urlValues.Encode())).
SetHeader("Content-Type", "application/x-www-form-urlencoded")
}
func NewRequestWithJSON(t testing.TB, method, urlStr string, v any) *RequestWrapper {
t.Helper()
jsonBytes, err := json.Marshal(v)
assert.NoError(t, err)
return NewRequestWithBody(t, method, urlStr, bytes.NewBuffer(jsonBytes)).
SetHeader("Content-Type", "application/json")
}
func NewRequestWithBody(t testing.TB, method, urlStr string, body io.Reader) *RequestWrapper {
t.Helper()
Move macaron to chi (#14293) Use [chi](https://github.com/go-chi/chi) instead of the forked [macaron](https://gitea.com/macaron/macaron). Since macaron and chi have conflicts with session share, this big PR becomes a have-to thing. According my previous idea, we can replace macaron step by step but I'm wrong. :( Below is a list of big changes on this PR. - [x] Define `context.ResponseWriter` interface with an implementation `context.Response`. - [x] Use chi instead of macaron, and also a customize `Route` to wrap chi so that the router usage is similar as before. - [x] Create different routers for `web`, `api`, `internal` and `install` so that the codes will be more clear and no magic . - [x] Use https://github.com/unrolled/render instead of macaron's internal render - [x] Use https://github.com/NYTimes/gziphandler instead of https://gitea.com/macaron/gzip - [x] Use https://gitea.com/go-chi/session which is a modified version of https://gitea.com/macaron/session and removed `nodb` support since it will not be maintained. **BREAK** - [x] Use https://gitea.com/go-chi/captcha which is a modified version of https://gitea.com/macaron/captcha - [x] Use https://gitea.com/go-chi/cache which is a modified version of https://gitea.com/macaron/cache - [x] Use https://gitea.com/go-chi/binding which is a modified version of https://gitea.com/macaron/binding - [x] Use https://github.com/go-chi/cors instead of https://gitea.com/macaron/cors - [x] Dropped https://gitea.com/macaron/i18n and make a new one in `code.gitea.io/gitea/modules/translation` - [x] Move validation form structs from `code.gitea.io/gitea/modules/auth` to `code.gitea.io/gitea/modules/forms` to avoid dependency cycle. - [x] Removed macaron log service because it's not need any more. **BREAK** - [x] All form structs have to be get by `web.GetForm(ctx)` in the route function but not as a function parameter on routes definition. - [x] Move Git HTTP protocol implementation to use routers directly. - [x] Fix the problem that chi routes don't support trailing slash but macaron did. - [x] `/api/v1/swagger` now will be redirect to `/api/swagger` but not render directly so that `APIContext` will not create a html render. Notices: - Chi router don't support request with trailing slash - Integration test `TestUserHeatmap` maybe mysql version related. It's failed on my macOS(mysql 5.7.29 installed via brew) but succeed on CI. Co-authored-by: 6543 <6543@obermui.de>
2021-01-26 10:36:53 -05:00
if !strings.HasPrefix(urlStr, "http") && !strings.HasPrefix(urlStr, "/") {
urlStr = "/" + urlStr
}
req, err := http.NewRequest(method, urlStr, body)
assert.NoError(t, err)
req.RequestURI = urlStr
return &RequestWrapper{req}
}
const NoExpectedStatus = -1
func MakeRequest(t testing.TB, rw *RequestWrapper, expectedStatus int) *httptest.ResponseRecorder {
t.Helper()
req := rw.Request
recorder := httptest.NewRecorder()
if req.RemoteAddr == "" {
req.RemoteAddr = "test-mock:12345"
}
testWebRoutes.ServeHTTP(recorder, req)
if expectedStatus != NoExpectedStatus {
if !assert.EqualValues(t, expectedStatus, recorder.Code, "Request: %s %s", req.Method, req.URL.String()) {
logUnexpectedResponse(t, recorder)
}
}
return recorder
}
func MakeRequestNilResponseRecorder(t testing.TB, rw *RequestWrapper, expectedStatus int) *NilResponseRecorder {
t.Helper()
req := rw.Request
recorder := NewNilResponseRecorder()
testWebRoutes.ServeHTTP(recorder, req)
if expectedStatus != NoExpectedStatus {
if !assert.EqualValues(t, expectedStatus, recorder.Code,
"Request: %s %s", req.Method, req.URL.String()) {
logUnexpectedResponse(t, &recorder.ResponseRecorder)
}
}
return recorder
}
func MakeRequestNilResponseHashSumRecorder(t testing.TB, rw *RequestWrapper, expectedStatus int) *NilResponseHashSumRecorder {
t.Helper()
req := rw.Request
recorder := NewNilResponseHashSumRecorder()
testWebRoutes.ServeHTTP(recorder, req)
if expectedStatus != NoExpectedStatus {
if !assert.EqualValues(t, expectedStatus, recorder.Code,
"Request: %s %s", req.Method, req.URL.String()) {
logUnexpectedResponse(t, &recorder.ResponseRecorder)
}
}
return recorder
}
// logUnexpectedResponse logs the contents of an unexpected response.
func logUnexpectedResponse(t testing.TB, recorder *httptest.ResponseRecorder) {
t.Helper()
respBytes := recorder.Body.Bytes()
if len(respBytes) == 0 {
return
} else if len(respBytes) < 500 {
// if body is short, just log the whole thing
t.Log("Response: ", string(respBytes))
return
}
t.Log("Response length: ", len(respBytes))
// log the "flash" error message, if one exists
// we must create a new buffer, so that we don't "use up" resp.Body
htmlDoc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(respBytes))
if err != nil {
return // probably a non-HTML response
}
errMsg := htmlDoc.Find(".ui.negative.message").Text()
if len(errMsg) > 0 {
t.Log("A flash error message was found:", errMsg)
}
}
func DecodeJSON(t testing.TB, resp *httptest.ResponseRecorder, v any) {
t.Helper()
decoder := json.NewDecoder(resp.Body)
assert.NoError(t, decoder.Decode(v))
}
func VerifyJSONSchema(t testing.TB, resp *httptest.ResponseRecorder, schemaFile string) {
t.Helper()
schemaFilePath := filepath.Join(filepath.Dir(setting.AppPath), "tests", "integration", "schemas", schemaFile)
_, schemaFileErr := os.Stat(schemaFilePath)
assert.Nil(t, schemaFileErr)
schema, schemaFileReadErr := os.ReadFile(schemaFilePath)
assert.Nil(t, schemaFileReadErr)
assert.True(t, len(schema) > 0)
nodeinfoSchema := gojsonschema.NewStringLoader(string(schema))
nodeinfoString := gojsonschema.NewStringLoader(resp.Body.String())
result, schemaValidationErr := gojsonschema.Validate(nodeinfoSchema, nodeinfoString)
assert.Nil(t, schemaValidationErr)
assert.Empty(t, result.Errors())
assert.True(t, result.Valid())
}
func GetCSRF(t testing.TB, session *TestSession, urlStr string) string {
t.Helper()
req := NewRequest(t, "GET", urlStr)
resp := session.MakeRequest(t, req, http.StatusOK)
doc := NewHTMLParser(t, resp.Body)
return doc.GetCSRF()
}