Omer/youtuber-blog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ara

Browse Source
This commit is contained in:
Omer Sabic 2024-04-28 20:58:31 +02:00
parent 6b47ffaf8c
commit 00fb06aaad
9 changed files with 165 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
package.json
View File

@ -30,12 +30,13 @@
"dependencies": { "dependencies": {
"@fastify/cookie": "^9.3.1", "@fastify/cookie": "^9.3.1",
"@fastify/cors": "^8.4.2", "@fastify/cors": "^8.4.2",
"@fastify/multipart": "^8.2.0",
"@fastify/oauth2": "^7.8.0", "@fastify/oauth2": "^7.8.0",
"drizzle-orm": "^0.29.1", "drizzle-orm": "^0.29.1",
"fastify": "^4.25.0", "fastify": "^4.25.0",
"fastify-plugin": "^4.5.1", "fastify-plugin": "^4.5.1",
"googleapis": "^134.0.0", "googleapis": "^134.0.0",
"openai": "^4.38.2", "openai": "^4.38.5",
"pg": "^8.11.3", "pg": "^8.11.3",
"redis": "^4.6.11", "redis": "^4.6.11",
"simple-get": "^4.0.1", "simple-get": "^4.0.1",

6
src/index.js
View File

@ -5,6 +5,7 @@ import fastify from "fastify";
import { middleware } from "./modules/middleware.js"; import { middleware } from "./modules/middleware.js";
import oauth from '@fastify/oauth2'; import oauth from '@fastify/oauth2';
import fastifyCookie from "@fastify/cookie"; import fastifyCookie from "@fastify/cookie";
// import fastifyMultipart from "@fastify/multipart";
const API_VERSION = "v1"; const API_VERSION = "v1";
@ -12,11 +13,16 @@ export const main = async () => {
const server = fastify({ const server = fastify({
bodyLimit: 1_000_000, bodyLimit: 1_000_000,
trustProxy: true, trustProxy: true,
// logger: true
}); });
await initDb(); await initDb();
// await Redis.initialize(); // await Redis.initialize();
// server.register(fastifyMultipart, {
// // attachFieldsToBody: true,
// });
server.register(fastifyCookie, { server.register(fastifyCookie, {
secret: "my-secret", // for cookies signature secret: "my-secret", // for cookies signature
hook: 'preParsing', // set to false to disable cookie autoparsing or set autoparsing on any of the following hooks: 'onRequest', 'preParsing', 'preHandler', 'preValidation'. default: 'onRequest' hook: 'preParsing', // set to false to disable cookie autoparsing or set autoparsing on any of the following hooks: 'onRequest', 'preParsing', 'preHandler', 'preValidation'. default: 'onRequest'

47
src/routes/auth.js
View File

@ -4,6 +4,7 @@ import { db } from '../db/index.js';
import { users as usersTable } from '../db/schemas.js'; import { users as usersTable } from '../db/schemas.js';
import { eq } from 'drizzle-orm'; import { eq } from 'drizzle-orm';
import { env } from '../utils/env.js'; import { env } from '../utils/env.js';
import { authMiddleware, authMiddlewareFn } from '../modules/middleware.js';
/** @typedef {import("fastify").FastifyInstance} FastifyInstance */ /** @typedef {import("fastify").FastifyInstance} FastifyInstance */
/** /**
@ -44,20 +45,30 @@ export const authRoutes = (fastify, _, done) => {
return; return;
} }
} }
let session_id;
const {session_id} = await createSession(user.id, { if(token.refresh_token) {
let session_info = await createSession(user.id, {
access_token: token.access_token, access_token: token.access_token,
refresh_token: token.refresh_token, refresh_token: token.refresh_token,
expires_at: new Date(token.expires_at) expires_at: new Date(token.expires_at)
}); });
response.setCookie("token", session_id, { session_id = session_info.session_id;
httpOnly: false, }
path: "/", else {
sameSite: false, console.error("ERROR: NOT IMPLEMENTED")
maxAge: 1000 * 60 * 60 * 24 * 7, }
domain: ".omersabic.com"
}).redirect(env.FRONTEND_URL); // response.setCookie("token", session_id, {
// httpOnly: false,
// path: "/",
// sameSite: false,
// maxAge: 1000 * 60 * 60 * 24 * 7,
// domain: ".omersabic.com"
// }).redirect(env.FRONTEND_URL);
response.redirect(env.FRONTEND_URL+"/auth?token="+session_id);
// response.send({ // response.send({
// token: session_id // token: session_id
// }); // });
@ -69,5 +80,23 @@ export const authRoutes = (fastify, _, done) => {
} }
}); });
fastify.post("/logout", {
preValidation: authMiddlewareFn
}, async (req, reply) => {
try {
await fastify.googleOAuth2.revokeToken(req.session, "refresh_token", {
"content-type": "application/json"
});
console.log("revoked")
// console.log(await res.json());
reply.send({
success: true
});
} catch (e) {
console.log(e);
}
})
done(); done();
}; };

49
src/routes/blog.js
View File

@ -1,8 +1,9 @@
import { eq } from "drizzle-orm"; import { eq } from "drizzle-orm";
import { db } from "../db/index.js"; import { db } from "../db/index.js";
import { authMiddleware, authMiddlewareFn } from "../modules/middleware.js"; import { authMiddleware, authMiddlewareFn } from "../modules/middleware.js";
import { getAccessToken, getChannelInfo } from "../utils/youtube.js"; import { getAccessToken, getCaptionText, getChannelInfo, getVideoCaptions, parseTextFromCaptions } from "../utils/youtube.js";
import { articles as articlesTable, sites } from "../db/schemas.js"; import { articles as articlesTable, sites } from "../db/schemas.js";
import { createBlogFromCaptions } from "../utils/ai.js";
/** /**
* *
@ -50,20 +51,50 @@ export const blogRoutes = (fastify, _, done) => {
fastify.post("/create", { fastify.post("/create", {
schema: { schema: {
body: { body: {
youtube_url: { type: "object",
type: "string" required: ["video_id"],
properties: {
video_id: {
type: "string",
}, },
length: { length: {
type: ["string", null], type: "number",
enum: ["short", "medium", "long"]
}, },
format: { format: {
type: ["string", null] type: "string"
} },
}
} }
},
},
preValidation: authMiddlewareFn
}, async (req, reply) => { }, async (req, reply) => {
reply.send() try {
console.log("creating blog post...")
const access_token = await getAccessToken(fastify, req);
const captions = await getVideoCaptions(access_token, req.body.video_id);
const preferred_caption_id = captions.find(x => x.snippet.language == 'en').id;
const caption_body = await getCaptionText(access_token, preferred_caption_id);
const caption_text = parseTextFromCaptions(caption_body).substring(28);
const blog_content = await createBlogFromCaptions(caption_text, req.body);
// TODO: once I add multiple sites per user, this should come from the client
const site = await db.select().from(sites).where(eq(sites.user_id, req.session.user_id));
console.log(site)
const article = await db.insert(articlesTable).values({
site_id: site[0].id,
content: blog_content,
source_video_id: req.body.video_id
}).returning({ id: articlesTable.id });
reply.send({
success: true,
article_id: article[0].id
});
} catch (e) {
console.log(e);
}
}) })
fastify.register(authMiddleware); fastify.register(authMiddleware);

15
src/routes/me.js
View File

@ -18,12 +18,25 @@ export const meRoutes = (fastify, _, done) => {
try { try {
const user = await db.select().from(users).where(eq(users.id, request.session.user_id)); const user = await db.select().from(users).where(eq(users.id, request.session.user_id));
if (typeof user == typeof Error) {
response.status(400).send({
success: false,
message: "User not found"
});
return;
}
response.send({ response.send({
success: true, success: true,
user: user[0] user: user[0]
}); });
} catch (e) { } catch (e) {
console.log(e); response.status(400).send({
success: false,
message: "User not found",
log: e.message
});
return;
} }
}); });

4
src/routes/videos.js
View File

@ -20,12 +20,12 @@ export const videoRoutes = (fastify, _, done) => {
try { try {
const token = await getAccessToken(fastify, request); const token = await getAccessToken(fastify, request);
const [user] = await db.select().from(users).where(eq(users.id, request.session.user_id)); const [user] = await db.select().from(users).where(eq(users.id, request.session.user_id));
const videos = await getVideosFromPlaylist(token.access_token, user.uploads_playlist_id); const videos = await getVideosFromPlaylist(token, user.uploads_playlist_id);
response.send({ response.send({
success: true, success: true,
videos videos
}) });
} catch (e) { } catch (e) {
console.log(e); console.log(e);
} }

9
src/utils/ai.js
View File

@ -1,4 +1,10 @@
const defaultModel = '@hf/mistralai/mistral-7b-instruct-v0.2'; const defaultModel = '@hf/mistralai/mistral-7b-instruct-v0.2';
import OpenAI from "openai";
const openai = new OpenAI({
apiKey: "",
baseURL: "https://api.pawan.krd/pai-001/v1"
})
async function cf_prompt(prompt, model = defaultModel) { async function cf_prompt(prompt, model = defaultModel) {
const options = { const options = {
@ -36,7 +42,6 @@ async function promptGPT(prompt, model = "gpt-3-turbo") {
console.log("prompting pai-001...") console.log("prompting pai-001...")
const res = await fetch('https://api.pawan.krd/pai-001/v1/chat/completions', options) const res = await fetch('https://api.pawan.krd/pai-001/v1/chat/completions', options)
.then(response => { .then(response => {
console.log("responded!")
return response.json(); return response.json();
}) })
.catch(err => console.log(err)); .catch(err => console.log(err));
@ -57,7 +62,7 @@ export async function createBlogFromCaptions(captions, {
format, format,
tone tone
} = {length: 500, language: "English", format: "summary", tone: "informal"}) { } = {length: 500, language: "English", format: "summary", tone: "informal"}) {
const prompt = `"Please convert the following video transcript into a blog post. The approximate length should be around ${length || 500} characters, written in ${language || "English"}. The desired format of the blog post is ${format || "summary"}. Please ensure the blog post has a ${tone || "informal"} tone throughout. Use markdown to format the article. Here is the transcript: "` const prompt = `Please convert the following video transcript into a blog post. The approximate length should be around ${length || 500} characters, written in ${language || "English"}. The desired format of the blog post is a ${format || "summary"}. Please ensure the blog post has a ${tone || "informal"} tone throughout. Use markdown to format the article. Here is the transcript: `
const result = await promptGPT(prompt + captions); const result = await promptGPT(prompt + captions);

19
src/utils/youtube.js
View File

@ -33,7 +33,7 @@ export async function getCaptionText(access_token, caption_id) {
const caption_text = await service.captions.download({ const caption_text = await service.captions.download({
id: caption_id, id: caption_id,
part: "snippet", part: "snippet",
tfmt: "srt", tfmt: "vtt",
tlang: "en", tlang: "en",
headers: { headers: {
"Authorization": "Bearer " + access_token "Authorization": "Bearer " + access_token
@ -43,6 +43,19 @@ export async function getCaptionText(access_token, caption_id) {
return caption_text; return caption_text;
} }
export function parseTextFromCaptions(caption_text) {
let text_content = "";
const captionEntries = caption_text.split(/\n\n/);
for (const entry of captionEntries) {
const lines = entry.trim().split('\n');
if (lines.length >= 2 && !lines[1].includes('-->')) {
text_content += lines.slice(1).join(' ').trim() + ' ';
}
}
return text_content
}
/** /**
* *
* @param {string} access_token * @param {string} access_token
@ -113,14 +126,14 @@ export async function getAccessToken(fastify, request) {
if((new Date().getTime() + 10) > request.session.expires_at) { if((new Date().getTime() + 10) > request.session.expires_at) {
/** @type {import('@fastify/oauth2').Token} */ /** @type {import('@fastify/oauth2').Token} */
const {token} = await fastify.googleOAuth2.getNewAccessTokenUsingRefreshToken(fastify); const {token} = await fastify.googleOAuth2.getNewAccessTokenUsingRefreshToken(request.session);
access_token = token.access_token; access_token = token.access_token;
await db.update(sessions).set({ await db.update(sessions).set({
expires_at: token.expires_at, expires_at: token.expires_at,
access_token: token.access_token access_token: token.access_token
}); }).where(eq(sessions.user_id, request.session.user_id));
} }

42
yarn.lock
View File

@ -371,15 +371,12 @@
resolved "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz" resolved "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz"
integrity sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA== integrity sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==
"@fastify/cookie@^9.0.4": "@fastify/busboy@^2.1.0":
version "9.3.1" version "2.1.1"
resolved "https://registry.npmjs.org/@fastify/cookie/-/cookie-9.3.1.tgz" resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.1.tgz#b9da6a878a371829a0502c9b6c1c143ef6663f4d"
integrity sha512-h1NAEhB266+ZbZ0e9qUE6NnNR07i7DnNXWG9VbbZ8uC6O/hxHpl+Zoe5sw1yfdZ2U6XhToUGDnzQtWJdCaPwfg== integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==
dependencies:
cookie-signature "^1.1.0"
fastify-plugin "^4.0.0"
"@fastify/cookie@^9.3.1": "@fastify/cookie@^9.0.4", "@fastify/cookie@^9.3.1":
version "9.3.1" version "9.3.1"
resolved "https://registry.yarnpkg.com/@fastify/cookie/-/cookie-9.3.1.tgz#48b89a356a23860c666e2fe522a084cc5c943d33" resolved "https://registry.yarnpkg.com/@fastify/cookie/-/cookie-9.3.1.tgz#48b89a356a23860c666e2fe522a084cc5c943d33"
integrity sha512-h1NAEhB266+ZbZ0e9qUE6NnNR07i7DnNXWG9VbbZ8uC6O/hxHpl+Zoe5sw1yfdZ2U6XhToUGDnzQtWJdCaPwfg== integrity sha512-h1NAEhB266+ZbZ0e9qUE6NnNR07i7DnNXWG9VbbZ8uC6O/hxHpl+Zoe5sw1yfdZ2U6XhToUGDnzQtWJdCaPwfg==
@ -400,7 +397,7 @@
resolved "https://registry.npmjs.org/@fastify/deepmerge/-/deepmerge-1.3.0.tgz" resolved "https://registry.npmjs.org/@fastify/deepmerge/-/deepmerge-1.3.0.tgz"
integrity sha512-J8TOSBq3SoZbDhM9+R/u77hP93gz/rajSA+K2kGyijPpORPWUXHUpTaleoj+92As0S9uPRP7Oi8IqMf0u+ro6A== integrity sha512-J8TOSBq3SoZbDhM9+R/u77hP93gz/rajSA+K2kGyijPpORPWUXHUpTaleoj+92As0S9uPRP7Oi8IqMf0u+ro6A==
"@fastify/error@^3.4.0": "@fastify/error@^3.0.0", "@fastify/error@^3.4.0":
version "3.4.1" version "3.4.1"
resolved "https://registry.npmjs.org/@fastify/error/-/error-3.4.1.tgz" resolved "https://registry.npmjs.org/@fastify/error/-/error-3.4.1.tgz"
integrity sha512-wWSvph+29GR783IhmvdwWnN4bUxTD01Vm5Xad4i7i1VuAOItLvbPAb69sb0IQ2N57yprvhNIwAP5B6xfKTmjmQ== integrity sha512-wWSvph+29GR783IhmvdwWnN4bUxTD01Vm5Xad4i7i1VuAOItLvbPAb69sb0IQ2N57yprvhNIwAP5B6xfKTmjmQ==
@ -412,6 +409,18 @@
dependencies: dependencies:
fast-json-stringify "^5.7.0" fast-json-stringify "^5.7.0"
"@fastify/multipart@^8.2.0":
version "8.2.0"
resolved "https://registry.yarnpkg.com/@fastify/multipart/-/multipart-8.2.0.tgz#90359b78eccd0f944cf145a2e907d167a3a6c731"
integrity sha512-OZ8nsyyoS2TV7Yeu3ZdrdDGsKUTAbfjrKC9jSxGgT2qdgek+BxpWX31ZubTrWMNZyU5xwk4ox6AvTjAbYWjrWg==
dependencies:
"@fastify/busboy" "^2.1.0"
"@fastify/deepmerge" "^1.0.0"
"@fastify/error" "^3.0.0"
fastify-plugin "^4.0.0"
secure-json-parse "^2.4.0"
stream-wormhole "^1.1.0"
"@fastify/oauth2@^7.8.0": "@fastify/oauth2@^7.8.0":
version "7.8.0" version "7.8.0"
resolved "https://registry.npmjs.org/@fastify/oauth2/-/oauth2-7.8.0.tgz" resolved "https://registry.npmjs.org/@fastify/oauth2/-/oauth2-7.8.0.tgz"
@ -3485,10 +3494,10 @@ open@^9.1.0:
is-inside-container "^1.0.0" is-inside-container "^1.0.0"
is-wsl "^2.2.0" is-wsl "^2.2.0"
openai@^4.38.2: openai@^4.38.5:
version "4.38.2" version "4.38.5"
resolved "https://registry.npmjs.org/openai/-/openai-4.38.2.tgz" resolved "https://registry.yarnpkg.com/openai/-/openai-4.38.5.tgz#87de78eed9f7e63331fb6b1307d8c9dd986b39d0"
integrity sha512-M16ehj0D84Gjq5cjvBzXRb5X+UvtWlxPDRAWAWMC0EN+6nHqnULIn5fWWeiexDPup25FeSZYv/ldp8KefcZVJQ== integrity sha512-Ym5GJL98ZhLJJ7enBx53jjG3vwN/fsB+Ozh46nnRZZS9W1NiYqbwkJ+sXd3dkCIiWIgcyyOPL2Zr8SQAzbpj3g==
dependencies: dependencies:
"@types/node" "^18.11.18" "@types/node" "^18.11.18"
"@types/node-fetch" "^2.6.4" "@types/node-fetch" "^2.6.4"
@ -4107,7 +4116,7 @@ safe-stable-stringify@^2.3.1:
resolved "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz" resolved "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz"
integrity sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g== integrity sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==
secure-json-parse@^2.7.0: secure-json-parse@^2.4.0, secure-json-parse@^2.7.0:
version "2.7.0" version "2.7.0"
resolved "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz" resolved "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz"
integrity sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw== integrity sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==
@ -4322,6 +4331,11 @@ stoppable@^1.1.0:
resolved "https://registry.npmjs.org/stoppable/-/stoppable-1.1.0.tgz" resolved "https://registry.npmjs.org/stoppable/-/stoppable-1.1.0.tgz"
integrity sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw== integrity sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==
stream-wormhole@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/stream-wormhole/-/stream-wormhole-1.1.0.tgz#300aff46ced553cfec642a05251885417693c33d"
integrity sha512-gHFfL3px0Kctd6Po0M8TzEvt3De/xu6cnRrjlfYNhwbhLPLwigI2t1nc6jrzNuaYg5C4YF78PPFuQPzRiqn9ew==
string-width@^4.2.0: string-width@^4.2.0:
version "4.2.3" version "4.2.3"
resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz"