Merge branch 'transfem-org:develop' into develop

README.md

@@ -18,6 +18,8 @@
 
 <a href="https://ko-fi.com/transfem">
 		<img src="https://custom-icon-badges.herokuapp.com/badge/donate-F96854?logoColor=F96854&style=for-the-badge&logo=kofi&labelColor=363B40" alt="donate"/></a>
+<a href="https://hosted.weblate.org/projects/sharkey/">
+		<img src="https://custom-icon-badges.herokuapp.com/badge/translate-sharkey-124437?logoColor=acea31&style=for-the-badge&logo=translate-sharkey&labelColor=363B40" alt="Translate Sharkey"/></a>
 
 ---
 

locales/en-US.yml

@@ -1594,6 +1594,7 @@ _role:
     gtlAvailable: "Can view the global timeline"
     ltlAvailable: "Can view the local timeline"
     canPublicNote: "Can send public notes"
+    canImportNotes: "Can import notes"
     canInvite: "Can create instance invite codes"
     inviteLimit: "Invite limit"
     inviteLimitCycle: "Invite limit cooldown"

locales/index.d.ts

@@ -1696,6 +1696,7 @@ export interface Locale {
             "gtlAvailable": string;
             "ltlAvailable": string;
             "canPublicNote": string;
+            "canImportNotes": string;
             "canInvite": string;
             "inviteLimit": string;
             "inviteLimitCycle": string;

locales/ja-JP.yml

@@ -1605,6 +1605,7 @@ _role:
     gtlAvailable: "グローバルタイムラインの閲覧"
     ltlAvailable: "ローカルタイムラインの閲覧"
     canPublicNote: "パブリック投稿の許可"
+    canImportNotes: "ノートのインポートが可能"
     canInvite: "サーバー招待コードの発行"
     inviteLimit: "招待コードの作成可能数"
     inviteLimitCycle: "招待コードの発行間隔"

packages/backend/migration/1699819257000-defaultLike.js

@@ -0,0 +1,11 @@
+export class instanceDefaultLike1699819257000 {
+    name = 'instanceDefaultLike1699819257000'
+
+    async up(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" ADD "defaultLike" character varying(500) DEFAULT '❤️'`);
+    }
+
+    async down(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "defaultLike"`);
+    }
+}

packages/backend/src/core/NoteCreateService.ts

@@ -378,6 +378,167 @@ export class NoteCreateService implements OnApplicationShutdown {
 		return note;
 	}
 
+	@bindThis
+	public async import(user: {
+		id: MiUser['id'];
+		username: MiUser['username'];
+		host: MiUser['host'];
+		isBot: MiUser['isBot'];
+		isIndexable: MiUser['isIndexable'];
+	}, data: Option, silent = false): Promise<MiNote> {
+		// チャンネル外にリプライしたら対象のスコープに合わせる
+		// (クライアントサイドでやっても良い処理だと思うけどとりあえずサーバーサイドで)
+		if (data.reply && data.channel && data.reply.channelId !== data.channel.id) {
+			if (data.reply.channelId) {
+				data.channel = await this.channelsRepository.findOneBy({ id: data.reply.channelId });
+			} else {
+				data.channel = null;
+			}
+		}
+
+		// チャンネル内にリプライしたら対象のスコープに合わせる
+		// (クライアントサイドでやっても良い処理だと思うけどとりあえずサーバーサイドで)
+		if (data.reply && (data.channel == null) && data.reply.channelId) {
+			data.channel = await this.channelsRepository.findOneBy({ id: data.reply.channelId });
+		}
+
+		if (data.createdAt == null) data.createdAt = new Date();
+		if (data.visibility == null) data.visibility = 'public';
+		if (data.localOnly == null) data.localOnly = false;
+		if (data.channel != null) data.visibility = 'public';
+		if (data.channel != null) data.visibleUsers = [];
+		if (data.channel != null) data.localOnly = true;
+
+		const meta = await this.metaService.fetch();
+
+		if (data.visibility === 'public' && data.channel == null) {
+			const sensitiveWords = meta.sensitiveWords;
+			if (this.isSensitive(data, sensitiveWords)) {
+				data.visibility = 'home';
+			} else if ((await this.roleService.getUserPolicies(user.id)).canPublicNote === false) {
+				data.visibility = 'home';
+			}
+		}
+
+		const inSilencedInstance = this.utilityService.isSilencedHost(meta.silencedHosts, user.host);
+
+		if (data.visibility === 'public' && inSilencedInstance && user.host !== null) {
+			data.visibility = 'home';
+		}
+
+		if (data.renote) {
+			switch (data.renote.visibility) {
+				case 'public':
+					// public noteは無条件にrenote可能
+					break;
+				case 'home':
+					// home noteはhome以下にrenote可能
+					if (data.visibility === 'public') {
+						data.visibility = 'home';
+					}
+					break;
+				case 'followers':
+					// 他人のfollowers noteはreject
+					if (data.renote.userId !== user.id) {
+						throw new Error('Renote target is not public or home');
+					}
+
+					// Renote対象がfollowersならfollowersにする
+					data.visibility = 'followers';
+					break;
+				case 'specified':
+					// specified / direct noteはreject
+					throw new Error('Renote target is not public or home');
+			}
+		}
+
+		// Check blocking
+		if (data.renote && data.text == null && data.poll == null && (data.files == null || data.files.length === 0)) {
+			if (data.renote.userHost === null) {
+				if (data.renote.userId !== user.id) {
+					const blocked = await this.userBlockingService.checkBlocked(data.renote.userId, user.id);
+					if (blocked) {
+						throw new Error('blocked');
+					}
+				}
+			}
+		}
+
+		// 返信対象がpublicではないならhomeにする
+		if (data.reply && data.reply.visibility !== 'public' && data.visibility === 'public') {
+			data.visibility = 'home';
+		}
+
+		// ローカルのみをRenoteしたらローカルのみにする
+		if (data.renote && data.renote.localOnly && data.channel == null) {
+			data.localOnly = true;
+		}
+
+		// ローカルのみにリプライしたらローカルのみにする
+		if (data.reply && data.reply.localOnly && data.channel == null) {
+			data.localOnly = true;
+		}
+
+		if (data.text) {
+			if (data.text.length > DB_MAX_NOTE_TEXT_LENGTH) {
+				data.text = data.text.slice(0, DB_MAX_NOTE_TEXT_LENGTH);
+			}
+			data.text = data.text.trim();
+		} else {
+			data.text = null;
+		}
+
+		let tags = data.apHashtags;
+		let emojis = data.apEmojis;
+		let mentionedUsers = data.apMentions;
+
+		// Parse MFM if needed
+		if (!tags || !emojis || !mentionedUsers) {
+			const tokens = (data.text ? mfm.parse(data.text)! : []);
+			const cwTokens = data.cw ? mfm.parse(data.cw)! : [];
+			const choiceTokens = data.poll && data.poll.choices
+				? concat(data.poll.choices.map(choice => mfm.parse(choice)!))
+				: [];
+
+			const combinedTokens = tokens.concat(cwTokens).concat(choiceTokens);
+
+			tags = data.apHashtags ?? extractHashtags(combinedTokens);
+
+			emojis = data.apEmojis ?? extractCustomEmojisFromMfm(combinedTokens);
+
+			mentionedUsers = data.apMentions ?? await this.extractMentionedUsers(user, combinedTokens);
+		}
+
+		tags = tags.filter(tag => Array.from(tag).length <= 128).splice(0, 32);
+
+		if (data.reply && (user.id !== data.reply.userId) && !mentionedUsers.some(u => u.id === data.reply!.userId)) {
+			mentionedUsers.push(await this.usersRepository.findOneByOrFail({ id: data.reply!.userId }));
+		}
+
+		if (data.visibility === 'specified') {
+			if (data.visibleUsers == null) throw new Error('invalid param');
+
+			for (const u of data.visibleUsers) {
+				if (!mentionedUsers.some(x => x.id === u.id)) {
+					mentionedUsers.push(u);
+				}
+			}
+
+			if (data.reply && !data.visibleUsers.some(x => x.id === data.reply!.userId)) {
+				data.visibleUsers.push(await this.usersRepository.findOneByOrFail({ id: data.reply!.userId }));
+			}
+		}
+
+		const note = await this.insertNote(user, data, tags, emojis, mentionedUsers);
+
+		setImmediate('post created', { signal: this.#shutdownController.signal }).then(
+			() => this.postNoteImported(note, user, data, silent, tags!, mentionedUsers!),
+			() => { /* aborted, ignore this */ },
+		);
+
+		return note;
+	}
+
 	@bindThis
 	private async insertNote(user: { id: MiUser['id']; host: MiUser['host']; }, data: Option, tags: string[], emojis: string[], mentionedUsers: MinimumUser[]) {
 		const insert = new MiNote({
@@ -715,6 +876,109 @@ export class NoteCreateService implements OnApplicationShutdown {
 		if (user.isIndexable) this.index(note);
 	}
 
+	@bindThis
+	private async postNoteImported(note: MiNote, user: {
+		id: MiUser['id'];
+		username: MiUser['username'];
+		host: MiUser['host'];
+		isBot: MiUser['isBot'];
+		isIndexable: MiUser['isIndexable'];
+	}, data: Option, silent: boolean, tags: string[], mentionedUsers: MinimumUser[]) {
+		const meta = await this.metaService.fetch();
+
+		this.notesChart.update(note, true);
+		if (meta.enableChartsForRemoteUser || (user.host == null)) {
+			this.perUserNotesChart.update(user, note, true);
+		}
+
+		// Register host
+		if (this.userEntityService.isRemoteUser(user)) {
+			this.federatedInstanceService.fetch(user.host).then(async i => {
+				if (note.renote && note.text) {
+					this.instancesRepository.increment({ id: i.id }, 'notesCount', 1);
+				} else if (!note.renote) {
+					this.instancesRepository.increment({ id: i.id }, 'notesCount', 1);
+				}
+				if ((await this.metaService.fetch()).enableChartsForFederatedInstances) {
+					this.instanceChart.updateNote(i.host, note, true);
+				}
+			});
+		}
+
+		if (data.renote && data.text) {
+			// Increment notes count (user)
+			this.incNotesCountOfUser(user);
+		} else if (!data.renote) {
+			// Increment notes count (user)
+			this.incNotesCountOfUser(user);
+		}
+
+		this.pushToTl(note, user);
+
+		this.antennaService.addNoteToAntennas(note, user);
+
+		if (data.reply) {
+			this.saveReply(data.reply, note);
+		}
+
+		if (data.reply == null) {
+			// TODO: キャッシュ
+			this.followingsRepository.findBy({
+				followeeId: user.id,
+				notify: 'normal',
+			}).then(followings => {
+				for (const following of followings) {
+					// TODO: ワードミュート考慮
+					this.notificationService.createNotification(following.followerId, 'note', {
+						noteId: note.id,
+					}, user.id);
+				}
+			});
+		}
+
+		if (data.renote && data.text == null && data.renote.userId !== user.id && !user.isBot) {
+			this.incRenoteCount(data.renote);
+		}
+
+		if (data.poll && data.poll.expiresAt) {
+			const delay = data.poll.expiresAt.getTime() - Date.now();
+			this.queueService.endedPollNotificationQueue.add(note.id, {
+				noteId: note.id,
+			}, {
+				delay,
+				removeOnComplete: true,
+			});
+		}
+
+		// Pack the note
+		const noteObj = await this.noteEntityService.pack(note, null, { skipHide: true, withReactionAndUserPairCache: true });
+
+		this.globalEventService.publishNotesStream(noteObj);
+
+		this.roleService.addNoteToRoleTimeline(noteObj);
+
+		if (data.channel) {
+			this.channelsRepository.increment({ id: data.channel.id }, 'notesCount', 1);
+			this.channelsRepository.update(data.channel.id, {
+				lastNotedAt: new Date(),
+			});
+
+			this.notesRepository.countBy({
+				userId: user.id,
+				channelId: data.channel.id,
+			}).then(count => {
+				// この処理が行われるのはノート作成後なので、ノートが一つしかなかったら最初の投稿だと判断できる
+				// TODO: とはいえノートを削除して何回も投稿すればその分だけインクリメントされる雑さもあるのでどうにかしたい
+				if (count === 1) {
+					this.channelsRepository.increment({ id: data.channel!.id }, 'usersCount', 1);
+				}
+			});
+		}
+
+		// Register to search database
+		if (user.isIndexable) this.index(note);
+	}
+
 	@bindThis
 	private isSensitive(note: Option, sensitiveWord: string[]): boolean {
 		if (sensitiveWord.length > 0) {

packages/backend/src/core/QueueService.ts

@@ -16,6 +16,7 @@ import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, Obj
 import type { DbJobData, DeliverJobData, RelationshipJobData, ThinUser } from '../queue/types.js';
 import type httpSignature from '@peertube/http-signature';
 import type * as Bull from 'bullmq';
+import { MiNote } from '@/models/Note.js';
 
 @Injectable()
 export class QueueService {
@@ -258,6 +259,48 @@ export class QueueService {
 		});
 	}
 
+	@bindThis
+	public createImportNotesJob(user: ThinUser, fileId: MiDriveFile['id'], type: string | null | undefined) {
+		return this.dbQueue.add('importNotes', {
+			user: { id: user.id },
+			fileId: fileId,
+			type: type,
+		}, {
+			removeOnComplete: true,
+			removeOnFail: true,
+		});
+	}
+
+	@bindThis
+	public createImportTweetsToDbJob(user: ThinUser, targets: string[]) {
+		const jobs = targets.map(rel => this.generateToDbJobData('importTweetsToDb', { user, target: rel }));
+		return this.dbQueue.addBulk(jobs);
+	}
+
+	@bindThis
+	public createImportMastoToDbJob(user: ThinUser, targets: string[]) {
+		const jobs = targets.map(rel => this.generateToDbJobData('importMastoToDb', { user, target: rel }));
+		return this.dbQueue.addBulk(jobs);
+	}
+
+	@bindThis
+	public createImportPleroToDbJob(user: ThinUser, targets: string[]) {
+		const jobs = targets.map(rel => this.generateToDbJobData('importPleroToDb', { user, target: rel }));
+		return this.dbQueue.addBulk(jobs);
+	}
+
+	@bindThis
+	public createImportKeyNotesToDbJob(user: ThinUser, targets: string[], note: MiNote['id'] | null) {
+		const jobs = targets.map(rel => this.generateToDbJobData('importKeyNotesToDb', { user, target: rel, note }));
+		return this.dbQueue.addBulk(jobs);
+	}
+
+	@bindThis
+	public createImportIGToDbJob(user: ThinUser, targets: string[]) {
+		const jobs = targets.map(rel => this.generateToDbJobData('importIGToDb', { user, target: rel }));
+		return this.dbQueue.addBulk(jobs);
+	}
+
 	@bindThis
 	public createImportFollowingToDbJob(user: ThinUser, targets: string[], withReplies?: boolean) {
 		const jobs = targets.map(rel => this.generateToDbJobData('importFollowingToDb', { user, target: rel, withReplies }));
@@ -293,7 +336,7 @@ export class QueueService {
 	}
 
 	@bindThis
-	private generateToDbJobData<T extends 'importFollowingToDb' | 'importBlockingToDb', D extends DbJobData<T>>(name: T, data: D): {
+	private generateToDbJobData<T extends 'importFollowingToDb' | 'importBlockingToDb' | 'importTweetsToDb' | 'importIGToDb' | 'importMastoToDb' | 'importPleroToDb' | 'importKeyNotesToDb', D extends DbJobData<T>>(name: T, data: D): {
 		name: string,
 		data: D,
 		opts: Bull.JobsOptions,