Infrastructure/ghost
0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2025-04-15 03:01:37 -05:00
Code Issues Activity

Added remapped twitter api data structures

Browse source
This commit is contained in:
Ronald Langeveld 2025-01-13 14:58:06 +09:00
parent 95d5c62607
commit 8e11c4e861
11 changed files with 611 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
ghost/core/core/server/services/oembed/RettiwtOEmbedProvider.js
View file

@ -7,10 +7,6 @@ const logging = require('@tryghost/logging');
const TWITTER_PATH_REGEX = /\/status\/(\d+)/;
/**
* @implements ICustomProvider
*/
function mapTweetEntity(tweet) {
return {
id: tweet?.id,
@ -66,6 +62,11 @@ function mapTweetEntity(tweet) {
}
};
}
/**
* @implements ICustomProvider
*/
class RettiwtOEmbedProvider {
/**
* @param {object} dependencies

14
ghost/core/core/server/services/oembed/service.js
View file

@ -2,7 +2,7 @@ const config = require('../../../shared/config');
const storage = require('../../adapters/storage');
const externalRequest = require('../../lib/request-external');
const {Rettiwt} = require('rettiwt-api');
const XEmbedProvider = require('@tryghost/x-embed-provider');
const OEmbed = require('@tryghost/oembed-service');
const oembed = new OEmbed({config, externalRequest, storage});
@ -13,11 +13,15 @@ const nft = new NFT({
}
});
const Twitter = require('./RettiwtOEmbedProvider');
// const Twitter = require('./TwitterOEmbedProvider');
// const twitter = new Twitter({
// config: {
// bearerToken: config.get('twitter').privateReadOnlyToken
// }
// });
const fetcher = new Rettiwt();
const twitter = new Twitter({
externalRequest: fetcher
});
const twitter = new XEmbedProvider(fetcher);
oembed.registerProvider(nft);
oembed.registerProvider(twitter);

7
ghost/x-embed-provider/.eslintrc.js Normal file
View file

@ -0,0 +1,7 @@
module.exports = {
parser: '@typescript-eslint/parser',
plugins: ['ghost'],
extends: [
'plugin:ghost/node'
]
};

23
ghost/x-embed-provider/README.md Normal file
View file

@ -0,0 +1,23 @@
# X Embed Provider
Embed Provider for Twitter / X
## Usage
## Develop
This is a monorepo package.
Follow the instructions for the top-level repo.
1. `git clone` this repo & `cd` into it as usual
2. Run `yarn` to install top-level dependencies.
## Test
- `yarn lint` run just eslint
- `yarn test` run lint and tests

33
ghost/x-embed-provider/package.json Normal file
View file

@ -0,0 +1,33 @@
{
"name": "@tryghost/x-embed-provider",
"version": "0.0.0",
"repository": "https://github.com/TryGhost/Ghost/tree/main/packages/x-embed-provider",
"author": "Ghost Foundation",
"private": true,
"main": "build/index.js",
"types": "build/index.d.ts",
"scripts": {
"dev": "tsc --watch --preserveWatchOutput --sourceMap",
"build": "tsc --sourceMap",
"prepare": "tsc",
"test:unit": "NODE_ENV=testing c8 --src src --all --check-coverage --100 --reporter text --reporter cobertura mocha -r ts-node/register './test/**/*.test.ts'",
"test": "yarn test:types && yarn test:unit",
"test:types": "tsc --noEmit",
"lint:code": "eslint src/ --ext .ts --cache",
"lint": "yarn lint:code && yarn lint:test",
"lint:test": "eslint -c test/.eslintrc.js test/ --ext .ts --cache"
},
"files": [
"build"
],
"devDependencies": {
"c8": "10.1.3",
"mocha": "11.0.1",
"sinon": "19.0.2",
"ts-node": "10.9.2",
"typescript": "5.7.2"
},
"dependencies": {
"rettiwt-api": "4.1.4"
}
}

350
ghost/x-embed-provider/src/XEmbedProvider.ts Normal file
View file

@ -0,0 +1,350 @@
import {ValidationError} from '@tryghost/errors';
// import type {FetcherService} from 'rettiwt-api';
type DependenciesType = {
config?: {
bearerToken?: string;
},
_fetcher: (tweetId: string) => Promise<any>; // eslint-disable-line
}
type IXEmbedProvider = {
_dependencies:DependenciesType;
canSupportRequest(url: URL): Promise<boolean>; // eslint-disable-line
getOEmbedData(url: URL): Promise<OEmbedData>; // eslint-disable-line
}
// reverse engineered tweet data from https://developer.x.com/en/docs/x-api/migrate/data-formats/standard-v1-1-to-v2
type TweetData = {
id: string;
text: string;
created_at: string;
author_id: string;
public_metrics: {
retweet_count: number;
like_count: number;
reply_count?: number;
quote_count?: number;
};
lang?: string;
conversation_id?: string;
in_reply_to_user_id?: string;
possibly_sensitive?: boolean;
reply_settings?: 'everyone' | 'mentioned_users' | 'followers';
source?: string;
withheld?: {
country_codes?: string[];
scope?: string;
};
context_annotations?: {
domain: {
id: string;
name: string;
description?: string;
};
entity: {
id: string;
name: string;
description?: string;
};
}[];
referenced_tweets?: {
type: 'retweeted' | 'replied_to' | 'quoted';
id: string;
author_id?: string;
}[];
geo?: {
place_id?: string;
};
entities?: {
mentions?: {
start: number;
end: number;
username: string;
}[];
urls?: {
start: number;
end: number;
url: string;
display_url?: string;
}[];
hashtags?: {
start: number;
end: number;
tag: string;
}[];
annotations?: {
start: number;
end: number;
probability: number;
type: string;
normalized_text: string;
}[];
};
attachments?: {
media_keys?: string[];
poll_ids?: string[];
};
includes?: {
media?: {
duration_ms?: number;
height?: number;
media_key: string;
preview_image_url?: string;
url?: string;
type?: string;
width?: number;
public_metrics?: {
view_count?: number;
like_count?: number;
retweet_count?: number;
reply_count?: number;
};
alt_text?: string;
}[];
places?: {
id: string;
full_name: string;
contained_within?: string[];
country: string;
country_code: string;
name: string;
place_type: string;
geo?: {
type: string;
bbox: number[];
properties: object;
};
}[];
polls?: {
id: string;
duration_minutes?: number;
end_datetime?: string;
options: {
position: number;
label: string;
votes: number;
}[];
voting_status: string;
}[];
users?: {
id: string;
name: string;
username: string;
created_at?: string;
description?: string;
location?: string;
pinned_tweet_id?: string;
profile_image_url?: string;
protected?: boolean;
verified?: boolean;
url?: string;
withheld?: {
country_codes?: string[];
};
entities?: {
url?: {
urls: {
start: number;
end: number;
url: string;
expanded_url: string;
display_url: string;
}[];
};
description?: {
urls: {
start: number;
end: number;
url: string;
expanded_url: string;
display_url: string;
}[];
};
};
public_metrics?: {
followers_count?: number;
following_count?: number;
tweet_count?: number;
listed_count?: number;
};
}[];
};
};
type OEmbedData = {
tweet_data: TweetData;
type: string;
}
const TWITTER_PATH_REGEX = /\/status\/(\d+)/;
export class XEmbedProvider implements IXEmbedProvider {
_dependencies:DependenciesType;
constructor(dependencies:DependenciesType) {
this._dependencies = dependencies;
}
async canSupportRequest(url:URL) {
return (url.host === 'twitter.com' || url.host === 'x.com') && TWITTER_PATH_REGEX.test(url.pathname);
}
async getTweetId(url: URL) {
const match = TWITTER_PATH_REGEX.exec(url.pathname);
if (!match) {
throw new ValidationError({
message: 'Invalid URL'
});
}
return match[1];
}
// since we can get v1 data without logging into twitter, we can use remap the data to v2 format
// to ensure compatibility with our templates
async mapTweetToTweetData(rawTweetData: any) : Promise<TweetData> {
const tweetData: TweetData = {
id: rawTweetData.id_str,
text: rawTweetData.full_text || rawTweetData.text,
created_at: rawTweetData.created_at,
author_id: rawTweetData.user.id_str,
public_metrics: {
retweet_count: rawTweetData.retweet_count,
like_count: rawTweetData.favorite_count
},
lang: rawTweetData.lang || 'en',
conversation_id: rawTweetData.conversation_id,
in_reply_to_user_id: rawTweetData.in_reply_to_user_id_str || undefined,
possibly_sensitive: rawTweetData.possibly_sensitive || false,
reply_settings: rawTweetData.reply_settings || 'everyone',
source: rawTweetData.source,
withheld: rawTweetData.withheld ? {
country_codes: rawTweetData.withheld.country_codes
} : undefined,
context_annotations: rawTweetData.context_annotations?.map((annotation: any) => ({
domain: {
id: annotation.domain.id,
name: annotation.domain.name,
description: annotation.domain.description
},
entity: {
id: annotation.entity.id,
name: annotation.entity.name,
description: annotation.entity.description
}
})) || [],
referenced_tweets: [],
geo: rawTweetData.geo ? {place_id: rawTweetData.geo.place_id} : undefined,
entities: {
mentions: rawTweetData.entities?.user_mentions?.map((mention: any) => ({
start: mention.indices[0],
end: mention.indices[1] - 1,
username: mention.screen_name
})) || [],
hashtags: rawTweetData.entities?.hashtags?.map((hashtag: any) => ({
start: hashtag.indices[0],
end: hashtag.indices[1] - 1,
tag: hashtag.text
})) || [],
urls: rawTweetData.entities?.urls?.map((url: any) => ({
start: url.indices[0],
end: url.indices[1] - 1,
url: url.expanded_url,
display_url: url.display_url
})) || []
},
includes: {
media: [],
users: [],
places: [],
polls: []
},
attachments: {
media_keys: [],
poll_ids: []
}
};
tweetData.attachments = tweetData.attachments || {media_keys: [], poll_ids: []};
tweetData.includes = tweetData.includes || {media: [], users: [], places: [], polls: []};
// Handling media attachments
if (rawTweetData.extended_entities?.media) {
// initialise attachments and includes.media
tweetData.attachments.media_keys = rawTweetData.extended_entities.media.map(
(media: any) => media.id_str
);
tweetData.includes = tweetData.includes || {media: [], users: [], places: [], polls: []};
tweetData.includes.media = rawTweetData.extended_entities.media.map((media: any) => ({
media_key: media.id_str,
url: media.media_url_https,
preview_image_url: media.media_url_https,
type: media.type,
height: media.sizes?.large?.h,
width: media.sizes?.large?.w
}));
}
// Handling referenced tweets (replies, retweets, quotes)
tweetData.referenced_tweets = [];
if (rawTweetData.in_reply_to_status_id_str) {
tweetData.referenced_tweets.push({
type: 'replied_to',
id: rawTweetData.in_reply_to_status_id_str
});
}
if (rawTweetData.retweeted_status) {
tweetData.referenced_tweets.push({
type: 'retweeted',
id: rawTweetData.retweeted_status.id_str,
author_id: rawTweetData.retweeted_status.user.id_str
});
}
if (rawTweetData.quoted_status_id_str) {
tweetData.referenced_tweets.push({
type: 'quoted',
id: rawTweetData.quoted_status_id_str
});
}
// Handling user information
tweetData.includes.users = tweetData.includes.users || [];
tweetData.includes.users.push({
id: rawTweetData.user.id_str,
name: rawTweetData.user.name,
username: rawTweetData.user.screen_name,
profile_image_url: rawTweetData.user.profile_image_url_https,
created_at: rawTweetData.user.created_at,
description: rawTweetData.user.description,
verified: rawTweetData.user.verified,
protected: rawTweetData.user.protected,
public_metrics: {
followers_count: rawTweetData.user.followers_count,
following_count: rawTweetData.user.friends_count,
tweet_count: rawTweetData.user.statuses_count,
listed_count: rawTweetData.user.listed_count
}
});
// Handling polls (v1.1 doesn't support polls explicitly)
if (rawTweetData.attachments?.poll_ids) {
tweetData.attachments.poll_ids = rawTweetData.attachments.poll_ids;
}
return tweetData;
}
async getOEmbedData(url: URL) : Promise<OEmbedData> {
const tweetId = await this.getTweetId(url);
const tweetData = await this._dependencies._fetcher(tweetId);
const oembed = {
tweet_data: await this.mapTweetToTweetData(tweetData),
type: 'tweet'
};
return oembed;
}
}

1
ghost/x-embed-provider/src/index.ts Normal file
View file

@ -0,0 +1 @@
export * from './XEmbedProvider';

7
ghost/x-embed-provider/test/.eslintrc.js Normal file
View file

@ -0,0 +1,7 @@
module.exports = {
parser: '@typescript-eslint/parser',
plugins: ['ghost'],
extends: [
'plugin:ghost/test'
]
};

112
ghost/x-embed-provider/test/x-embed-provider.test.ts Normal file
View file

@ -0,0 +1,112 @@
import assert from 'assert/strict';
import {XEmbedProvider} from '../src/XEmbedProvider';
describe('X Embed Providers', function () {
let dependencies = {
config: {
bearerToken: '123'
},
_fetcher: async (tweetId: string) => {
return {
id_str: tweetId,
text: 'Hello World',
created_at: '2021-01-01',
user: {
id_str: '123',
name: 'Test User',
screen_name: 'testuser',
profile_image_url: 'https://test.com/test.jpg'
},
retweet_count: 1,
like_count: 1,
entities: {
user_mentions: [
{
indices: [0, 5],
screen_name: 'testuser'
}
],
hashtags: [
{
indices: [0, 5],
text: 'test'
}
],
urls: [
{
indices: [0, 5],
expanded_url: 'https://test.com',
display_url: 'test.com'
}
]
},
attachments: {
media_keys: ['123']
}
};
}
};
it('Can Initialise XEmbedProvider', function () {
const xEmbedProvider = new XEmbedProvider(dependencies);
assert.ok(xEmbedProvider);
});
it('Can Initialise XEmbedProvider without Config', function () {
const xEmbedProvider = new XEmbedProvider(dependencies);
assert.ok(xEmbedProvider);
});
it('Can Support Twitter URL', async function () {
const xEmbedProvider = new XEmbedProvider(dependencies);
const tweetURL = new URL('https://twitter.com/status/123');
const result = await xEmbedProvider.canSupportRequest(tweetURL);
assert.equal(result, true);
});
it ('Can Support X URL', async function () {
const xEmbedProvider = new XEmbedProvider(dependencies);
const tweetURL = new URL('https://x.com/status/123');
const result = await xEmbedProvider.canSupportRequest(tweetURL);
assert.equal(result, true);
});
it ('Cannot Support Invalid URL', async function () {
const xEmbedProvider = new XEmbedProvider(dependencies);
const tweetURL = new URL('https://invalid.com/invalid');
const result = await xEmbedProvider.canSupportRequest(tweetURL);
assert.equal(result, false);
});
it ('Cannot Support Invalid Host', async function () {
const xEmbedProvider = new XEmbedProvider(dependencies);
const tweetURL = new URL('https://invalid.com/status/123');
const result = await xEmbedProvider.canSupportRequest(tweetURL);
assert.equal(result, false);
});
it('can get TweetId', async function () {
const xEmbedProvider = new XEmbedProvider(dependencies);
const tweetURL = new URL('https://twitter.com/status/123');
const result = await xEmbedProvider.getTweetId(tweetURL);
assert.equal(result, '123');
});
it('cannot get TweetId from invalid URL', async function () {
const xEmbedProvider = new XEmbedProvider(dependencies);
const tweetURL = new URL('https://twitter.com/invalid');
assert.rejects(async () => {
await xEmbedProvider.getTweetId(tweetURL);
});
});
it('can get OEmbedData', async function () {
const xEmbedProvider = new XEmbedProvider(dependencies);
const tweetURL = new URL('https://x.com/teslaownersSV/status/1876891406603530610');
const result = await xEmbedProvider.getOEmbedData(tweetURL);
assert.equal(result.type, 'rich');
assert.equal(result.tweet_data.id, '1876891406603530610');
assert.equal(result.tweet_data.text, 'Hello World');
assert.equal(result.tweet_data.created_at, '2021-01-01');
});
});

9
ghost/x-embed-provider/tsconfig.json Normal file
View file

@ -0,0 +1,9 @@
{
"extends": "../tsconfig.json",
"include": [
"src/**/*"
],
"compilerOptions": {
"outDir": "build"
}
}

57
yarn.lock
View file

@ -1919,6 +1919,11 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@bcoe/v8-coverage@^1.0.1":
version "1.0.1"
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-1.0.1.tgz#d72197747b8c7f7d63faa4f91de26fa649955a6d"
integrity sha512-W+a0/JpU28AqH4IKtwUPcEUnUyXMDLALcn5/JLczGGT9fHE2sIby/xP/oQnx3nxkForzgzPy201RAKcB4xPAFQ==
"@breejs/later@4.2.0", "@breejs/later@^4.1.0":
version "4.2.0"
resolved "https://registry.yarnpkg.com/@breejs/later/-/later-4.2.0.tgz#669661f3a02535ef900f360c74e48c3f5483c786"
@ -12012,6 +12017,23 @@ c8@10.1.2:
yargs "^17.7.2"
yargs-parser "^21.1.1"
c8@10.1.3:
version "10.1.3"
resolved "https://registry.yarnpkg.com/c8/-/c8-10.1.3.tgz#54afb25ebdcc7f3b00112482c6d90d7541ad2fcd"
integrity sha512-LvcyrOAaOnrrlMpW22n690PUvxiq4Uf9WMhQwNJ9vgagkL/ph1+D4uvjvDA5XCbykrc0sx+ay6pVi9YZ1GnhyA==
dependencies:
"@bcoe/v8-coverage" "^1.0.1"
"@istanbuljs/schema" "^0.1.3"
find-up "^5.0.0"
foreground-child "^3.1.1"
istanbul-lib-coverage "^3.2.0"
istanbul-lib-report "^3.0.1"
istanbul-reports "^3.1.6"
test-exclude "^7.0.1"
v8-to-istanbul "^9.0.0"
yargs "^17.7.2"
yargs-parser "^21.1.1"
c8@7.14.0:
version "7.14.0"
resolved "https://registry.yarnpkg.com/c8/-/c8-7.14.0.tgz#f368184c73b125a80565e9ab2396ff0be4d732f3"
@ -18611,7 +18633,7 @@ glob@8.1.0, glob@^8.0.3, glob@^8.1.0:
minimatch "^5.0.1"
once "^1.3.0"
glob@^10.0.0, glob@^10.2.2, glob@^10.3.10, glob@^10.3.7, glob@^10.4.1:
glob@^10.0.0, glob@^10.2.2, glob@^10.3.10, glob@^10.3.7, glob@^10.4.1, glob@^10.4.5:
version "10.4.5"
resolved "https://registry.yarnpkg.com/glob/-/glob-10.4.5.tgz#f4d9f0b90ffdbab09c9d77f5f29b4262517b0956"
integrity sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==
@ -23689,6 +23711,32 @@ mocha@10.8.2:
yargs-parser "^20.2.9"
yargs-unparser "^2.0.0"
mocha@11.0.1:
version "11.0.1"
resolved "https://registry.yarnpkg.com/mocha/-/mocha-11.0.1.tgz#85c1c0e806275fe2479245be4ac4a0d81f533aa8"
integrity sha512-+3GkODfsDG71KSCQhc4IekSW+ItCK/kiez1Z28ksWvYhKXV/syxMlerR/sC7whDp7IyreZ4YxceMLdTs5hQE8A==
dependencies:
ansi-colors "^4.1.3"
browser-stdout "^1.3.1"
chokidar "^3.5.3"
debug "^4.3.5"
diff "^5.2.0"
escape-string-regexp "^4.0.0"
find-up "^5.0.0"
glob "^10.4.5"
he "^1.2.0"
js-yaml "^4.1.0"
log-symbols "^4.1.0"
minimatch "^5.1.6"
ms "^2.1.3"
serialize-javascript "^6.0.2"
strip-json-comments "^3.1.1"
supports-color "^8.1.1"
workerpool "^6.5.1"
yargs "^16.2.0"
yargs-parser "^20.2.9"
yargs-unparser "^2.0.0"
mocha@^2.5.3:
version "2.5.3"
resolved "https://registry.yarnpkg.com/mocha/-/mocha-2.5.3.tgz#161be5bdeb496771eb9b35745050b622b5aefc58"
@ -27954,7 +28002,7 @@ retry@^0.12.0:
resolved "https://registry.yarnpkg.com/retry/-/retry-0.12.0.tgz#1b42a6266a21f07421d1b0b54b7dc167b01c013b"
integrity sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==
rettiwt-api@^4.1.4:
rettiwt-api@4.1.4, rettiwt-api@^4.1.4:
version "4.1.4"
resolved "https://registry.yarnpkg.com/rettiwt-api/-/rettiwt-api-4.1.4.tgz#76d0eba3c86eb3fc2c4c27f7a1ce78d52dc857de"
integrity sha512-0NXu6KOy8iPiGHreib2cu8Uf7OWGMgiyR87KT9XkLRr5eiKgbVZxdq+Rv13CYLUJhiTz3qVNYPy5ai6wZR6shg==
@ -30801,6 +30849,11 @@ typescript@5.6.3, typescript@^5.0.4:
resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.6.3.tgz#5f3449e31c9d94febb17de03cc081dd56d81db5b"
integrity sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==
typescript@5.7.2:
version "5.7.2"
resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.7.2.tgz#3169cf8c4c8a828cde53ba9ecb3d2b1d5dd67be6"
integrity sha512-i5t66RHxDvVN40HfDd1PsEThGNnlMCMT3jMUuoh9/0TaqWevNontacunWyN02LA9/fIbEWlcHZcgTKb9QoaLfg==
ua-parser-js@1.0.39:
version "1.0.39"
resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-1.0.39.tgz#bfc07f361549bf249bd8f4589a4cccec18fd2018"