diff --git a/packages/core/src/queries/application.test.ts b/packages/core/src/queries/application.test.ts
index 40c03dda3..7b371e3ee 100644
--- a/packages/core/src/queries/application.test.ts
+++ b/packages/core/src/queries/application.test.ts
@@ -23,6 +23,7 @@ const {
   findTotalNumberOfApplications,
   findApplicationById,
   findApplicationByProtectedAppHost,
+  findApplicationByProtectedAppCustomDomain,
   insertApplication,
   updateApplicationById,
   deleteApplicationById,
@@ -88,6 +89,28 @@ describe('application query', () => {
     await findApplicationByProtectedAppHost(host);
   });
 
+  it('findApplicationByProtectedAppCustomDomain', async () => {
+    const domain = 'my.blog.com';
+    const rowData = { domain };
+
+    const expectSql = sql`
+      select ${sql.join(Object.values(fields), sql`, `)}
+      from ${table}
+      where ${fields.protectedAppMetadata} ? 'customDomains' 
+      and ${fields.protectedAppMetadata}->'customDomains' @> $1::jsonb
+      and ${fields.type} = $2
+    `;
+
+    mockQuery.mockImplementationOnce(async (sql, values) => {
+      expectSqlAssert(sql, expectSql.sql);
+      expect(values).toEqual([JSON.stringify([domain]), ApplicationType.Protected]);
+
+      return createMockQueryResult([rowData]);
+    });
+
+    await findApplicationByProtectedAppCustomDomain(domain);
+  });
+
   it('insertApplication', async () => {
     const keys = excludeAutoSetFields(Applications.fieldKeys);
 
diff --git a/packages/core/src/queries/application.ts b/packages/core/src/queries/application.ts
index 7a20d559a..3c01fb217 100644
--- a/packages/core/src/queries/application.ts
+++ b/packages/core/src/queries/application.ts
@@ -138,6 +138,19 @@ export const createApplicationQueries = (pool: CommonQueryMethods) => {
       and ${fields.type} = ${ApplicationType.Protected}
     `);
 
+  /**
+   * Find an protected application by its custom domain.
+   * the domain is stored in the `customDomains` field of the `protectedAppMetadata` field.
+   */
+  const findApplicationByProtectedAppCustomDomain = async (domain: string) =>
+    pool.maybeOne<Application>(sql`
+      select ${sql.join(Object.values(fields), sql`, `)}
+      from ${table}
+      where ${fields.protectedAppMetadata} ? 'customDomains'
+      and ${fields.protectedAppMetadata}->'customDomains' @> ${sql.jsonb([domain])}
+      and ${fields.type} = ${ApplicationType.Protected}
+    `);
+
   const insertApplication = buildInsertIntoWithPool(pool)(Applications, {
     returning: true,
   });
@@ -240,6 +253,7 @@ export const createApplicationQueries = (pool: CommonQueryMethods) => {
     findTotalNumberOfApplications,
     findApplicationById,
     findApplicationByProtectedAppHost,
+    findApplicationByProtectedAppCustomDomain,
     insertApplication,
     updateApplication,
     updateApplicationById,
diff --git a/packages/core/src/routes/applications/application-protected-app-metadata.test.ts b/packages/core/src/routes/applications/application-protected-app-metadata.test.ts
index 264acbe49..fc7b3b759 100644
--- a/packages/core/src/routes/applications/application-protected-app-metadata.test.ts
+++ b/packages/core/src/routes/applications/application-protected-app-metadata.test.ts
@@ -1,5 +1,6 @@
-import { DomainStatus } from '@logto/schemas';
+import { type Application, DomainStatus } from '@logto/schemas';
 import { pickDefault } from '@logto/shared/esm';
+import { type Nullable } from '@silverhand/essentials';
 
 import { mockProtectedApplication } from '#src/__mocks__/index.js';
 import { mockIdGenerators } from '#src/test-utils/nanoid.js';
@@ -11,6 +12,9 @@ const mockDomain = 'app.example.com';
 
 const updateApplicationById = jest.fn();
 const findApplicationById = jest.fn(async () => mockProtectedApplication);
+const findApplicationByProtectedAppCustomDomain = jest.fn(
+  async (): Promise<Nullable<Application>> => null
+);
 
 const mockDomainResponse = {
   domain: mockDomain,
@@ -35,6 +39,7 @@ const tenantContext = new MockTenant(
     applications: {
       findApplicationById,
       updateApplicationById,
+      findApplicationByProtectedAppCustomDomain,
     },
   },
   undefined,
@@ -86,5 +91,15 @@ describe('application protected app metadata routes', () => {
         });
       expect(response.status).toEqual(400);
     });
+
+    it('throw when the domain is already in use', async () => {
+      findApplicationByProtectedAppCustomDomain.mockResolvedValueOnce(mockProtectedApplication);
+      const response = await requester
+        .post(`/applications/asdf/protected-app-metadata/custom-domains`)
+        .send({
+          domain: mockDomain,
+        });
+      expect(response.status).toEqual(422);
+    });
   });
 });
diff --git a/packages/core/src/routes/applications/application-protected-app-metadata.ts b/packages/core/src/routes/applications/application-protected-app-metadata.ts
index 4dc0b5a76..2130f2220 100644
--- a/packages/core/src/routes/applications/application-protected-app-metadata.ts
+++ b/packages/core/src/routes/applications/application-protected-app-metadata.ts
@@ -1,5 +1,6 @@
 import { z } from 'zod';
 
+import RequestError from '#src/errors/RequestError/index.js';
 import koaGuard from '#src/middleware/koa-guard.js';
 import assertThat from '#src/utils/assert-that.js';
 
@@ -10,7 +11,11 @@ export default function applicationProtectedAppMetadataRoutes<T extends AuthedRo
     router,
     {
       queries: {
-        applications: { findApplicationById, updateApplicationById },
+        applications: {
+          findApplicationById,
+          updateApplicationById,
+          findApplicationByProtectedAppCustomDomain,
+        },
       },
       libraries: {
         applications: { validateProtectedApplicationById },
@@ -51,7 +56,13 @@ export default function applicationProtectedAppMetadataRoutes<T extends AuthedRo
         'domain.limit_to_one_domain'
       );
 
-      // TODO: LOG-8066 check if domain is already in use
+      assertThat(
+        !(await findApplicationByProtectedAppCustomDomain(domain)),
+        new RequestError({
+          code: 'domain.hostname_already_exists',
+          status: 422,
+        })
+      );
 
       const customDomain = await addDomainToRemote(domain);
       await updateApplicationById(id, {