diff --git a/package.json b/package.json index c83b8a7..3e246e8 100644 --- a/package.json +++ b/package.json @@ -30,12 +30,13 @@ "dependencies": { "@fastify/cookie": "^9.3.1", "@fastify/cors": "^8.4.2", + "@fastify/multipart": "^8.2.0", "@fastify/oauth2": "^7.8.0", "drizzle-orm": "^0.29.1", "fastify": "^4.25.0", "fastify-plugin": "^4.5.1", "googleapis": "^134.0.0", - "openai": "^4.38.2", + "openai": "^4.38.5", "pg": "^8.11.3", "redis": "^4.6.11", "simple-get": "^4.0.1", diff --git a/src/index.js b/src/index.js index 5bef0ea..6378e80 100644 --- a/src/index.js +++ b/src/index.js @@ -5,6 +5,7 @@ import fastify from "fastify"; import { middleware } from "./modules/middleware.js"; import oauth from '@fastify/oauth2'; import fastifyCookie from "@fastify/cookie"; +// import fastifyMultipart from "@fastify/multipart"; const API_VERSION = "v1"; @@ -12,11 +13,16 @@ export const main = async () => { const server = fastify({ bodyLimit: 1_000_000, trustProxy: true, + // logger: true }); await initDb(); // await Redis.initialize(); + // server.register(fastifyMultipart, { + // // attachFieldsToBody: true, + // }); + server.register(fastifyCookie, { secret: "my-secret", // for cookies signature hook: 'preParsing', // set to false to disable cookie autoparsing or set autoparsing on any of the following hooks: 'onRequest', 'preParsing', 'preHandler', 'preValidation'. default: 'onRequest' diff --git a/src/routes/auth.js b/src/routes/auth.js index 0e1a684..38deb99 100644 --- a/src/routes/auth.js +++ b/src/routes/auth.js @@ -4,6 +4,7 @@ import { db } from '../db/index.js'; import { users as usersTable } from '../db/schemas.js'; import { eq } from 'drizzle-orm'; import { env } from '../utils/env.js'; +import { authMiddleware, authMiddlewareFn } from '../modules/middleware.js'; /** @typedef {import("fastify").FastifyInstance} FastifyInstance */ /** @@ -44,20 +45,30 @@ export const authRoutes = (fastify, _, done) => { return; } } + let session_id; + if(token.refresh_token) { + let session_info = await createSession(user.id, { + access_token: token.access_token, + refresh_token: token.refresh_token, + expires_at: new Date(token.expires_at) + }); - const {session_id} = await createSession(user.id, { - access_token: token.access_token, - refresh_token: token.refresh_token, - expires_at: new Date(token.expires_at) - }); + session_id = session_info.session_id; + } + else { + console.error("ERROR: NOT IMPLEMENTED") + } + + // response.setCookie("token", session_id, { + // httpOnly: false, + // path: "/", + // sameSite: false, + // maxAge: 1000 * 60 * 60 * 24 * 7, + // domain: ".omersabic.com" + // }).redirect(env.FRONTEND_URL); + + response.redirect(env.FRONTEND_URL+"/auth?token="+session_id); - response.setCookie("token", session_id, { - httpOnly: false, - path: "/", - sameSite: false, - maxAge: 1000 * 60 * 60 * 24 * 7, - domain: ".omersabic.com" - }).redirect(env.FRONTEND_URL); // response.send({ // token: session_id // }); @@ -69,5 +80,23 @@ export const authRoutes = (fastify, _, done) => { } }); + fastify.post("/logout", { + preValidation: authMiddlewareFn + }, async (req, reply) => { + try { + await fastify.googleOAuth2.revokeToken(req.session, "refresh_token", { + "content-type": "application/json" + }); + console.log("revoked") + // console.log(await res.json()); + + reply.send({ + success: true + }); + } catch (e) { + console.log(e); + } + }) + done(); }; diff --git a/src/routes/blog.js b/src/routes/blog.js index 8e8c10a..9d76421 100644 --- a/src/routes/blog.js +++ b/src/routes/blog.js @@ -1,8 +1,9 @@ import { eq } from "drizzle-orm"; import { db } from "../db/index.js"; import { authMiddleware, authMiddlewareFn } from "../modules/middleware.js"; -import { getAccessToken, getChannelInfo } from "../utils/youtube.js"; +import { getAccessToken, getCaptionText, getChannelInfo, getVideoCaptions, parseTextFromCaptions } from "../utils/youtube.js"; import { articles as articlesTable, sites } from "../db/schemas.js"; +import { createBlogFromCaptions } from "../utils/ai.js"; /** * @@ -16,7 +17,7 @@ export const blogRoutes = (fastify, _, done) => { try { const mine = request.query.mine != 'false' || true; const blog_id = request.query.blog_id; - if(!mine && !blog_id) { + if (!mine && !blog_id) { response.send({ success: false, message: "Request can either have \"mine\" set to true or provide a blog_id" @@ -24,12 +25,12 @@ export const blogRoutes = (fastify, _, done) => { } let clause; - if(mine) { - if(!(await authMiddlewareFn(request, response))) return; + if (mine) { + if (!(await authMiddlewareFn(request, response))) return; clause = eq(sites.user_id, request.session.user_id); } - if(mine == false) { + if (mine == false) { clause = eq(sites.user_id, request.query.blog_id); } // const access_token = await getAccessToken(fastify, request); @@ -50,20 +51,50 @@ export const blogRoutes = (fastify, _, done) => { fastify.post("/create", { schema: { body: { - youtube_url: { - type: "string" - }, - length: { - type: ["string", null], - enum: ["short", "medium", "long"] - }, - format: { - type: ["string", null] + type: "object", + required: ["video_id"], + properties: { + video_id: { + type: "string", + }, + length: { + type: "number", + }, + format: { + type: "string" + }, } - } - } + }, + }, + preValidation: authMiddlewareFn }, async (req, reply) => { - reply.send() + try { + console.log("creating blog post...") + const access_token = await getAccessToken(fastify, req); + const captions = await getVideoCaptions(access_token, req.body.video_id); + const preferred_caption_id = captions.find(x => x.snippet.language == 'en').id; + + const caption_body = await getCaptionText(access_token, preferred_caption_id); + const caption_text = parseTextFromCaptions(caption_body).substring(28); + + const blog_content = await createBlogFromCaptions(caption_text, req.body); + // TODO: once I add multiple sites per user, this should come from the client + const site = await db.select().from(sites).where(eq(sites.user_id, req.session.user_id)); + console.log(site) + + const article = await db.insert(articlesTable).values({ + site_id: site[0].id, + content: blog_content, + source_video_id: req.body.video_id + }).returning({ id: articlesTable.id }); + + reply.send({ + success: true, + article_id: article[0].id + }); + } catch (e) { + console.log(e); + } }) fastify.register(authMiddleware); diff --git a/src/routes/me.js b/src/routes/me.js index 5c4df6e..d16702c 100644 --- a/src/routes/me.js +++ b/src/routes/me.js @@ -17,13 +17,26 @@ export const meRoutes = (fastify, _, done) => { fastify.get("/", async (request, response) => { try { const user = await db.select().from(users).where(eq(users.id, request.session.user_id)); - + + if (typeof user == typeof Error) { + response.status(400).send({ + success: false, + message: "User not found" + }); + return; + } + response.send({ success: true, user: user[0] }); } catch (e) { - console.log(e); + response.status(400).send({ + success: false, + message: "User not found", + log: e.message + }); + return; } }); diff --git a/src/routes/videos.js b/src/routes/videos.js index e3ee1ef..6894145 100644 --- a/src/routes/videos.js +++ b/src/routes/videos.js @@ -20,12 +20,12 @@ export const videoRoutes = (fastify, _, done) => { try { const token = await getAccessToken(fastify, request); const [user] = await db.select().from(users).where(eq(users.id, request.session.user_id)); - const videos = await getVideosFromPlaylist(token.access_token, user.uploads_playlist_id); + const videos = await getVideosFromPlaylist(token, user.uploads_playlist_id); response.send({ success: true, videos - }) + }); } catch (e) { console.log(e); } diff --git a/src/utils/ai.js b/src/utils/ai.js index e6ac341..e3f6b58 100644 --- a/src/utils/ai.js +++ b/src/utils/ai.js @@ -1,4 +1,10 @@ const defaultModel = '@hf/mistralai/mistral-7b-instruct-v0.2'; +import OpenAI from "openai"; + +const openai = new OpenAI({ + apiKey: "", + baseURL: "https://api.pawan.krd/pai-001/v1" +}) async function cf_prompt(prompt, model = defaultModel) { const options = { @@ -36,7 +42,6 @@ async function promptGPT(prompt, model = "gpt-3-turbo") { console.log("prompting pai-001...") const res = await fetch('https://api.pawan.krd/pai-001/v1/chat/completions', options) .then(response => { - console.log("responded!") return response.json(); }) .catch(err => console.log(err)); @@ -57,7 +62,7 @@ export async function createBlogFromCaptions(captions, { format, tone } = {length: 500, language: "English", format: "summary", tone: "informal"}) { - const prompt = `"Please convert the following video transcript into a blog post. The approximate length should be around ${length || 500} characters, written in ${language || "English"}. The desired format of the blog post is ${format || "summary"}. Please ensure the blog post has a ${tone || "informal"} tone throughout. Use markdown to format the article. Here is the transcript: "` + const prompt = `Please convert the following video transcript into a blog post. The approximate length should be around ${length || 500} characters, written in ${language || "English"}. The desired format of the blog post is a ${format || "summary"}. Please ensure the blog post has a ${tone || "informal"} tone throughout. Use markdown to format the article. Here is the transcript: ` const result = await promptGPT(prompt + captions); diff --git a/src/utils/youtube.js b/src/utils/youtube.js index 57f9dfa..b146e0c 100644 --- a/src/utils/youtube.js +++ b/src/utils/youtube.js @@ -33,7 +33,7 @@ export async function getCaptionText(access_token, caption_id) { const caption_text = await service.captions.download({ id: caption_id, part: "snippet", - tfmt: "srt", + tfmt: "vtt", tlang: "en", headers: { "Authorization": "Bearer " + access_token @@ -43,6 +43,19 @@ export async function getCaptionText(access_token, caption_id) { return caption_text; } +export function parseTextFromCaptions(caption_text) { + let text_content = ""; + const captionEntries = caption_text.split(/\n\n/); + for (const entry of captionEntries) { + const lines = entry.trim().split('\n'); + if (lines.length >= 2 && !lines[1].includes('-->')) { + text_content += lines.slice(1).join(' ').trim() + ' '; + } + } + + return text_content +} + /** * * @param {string} access_token @@ -113,14 +126,14 @@ export async function getAccessToken(fastify, request) { if((new Date().getTime() + 10) > request.session.expires_at) { /** @type {import('@fastify/oauth2').Token} */ - const {token} = await fastify.googleOAuth2.getNewAccessTokenUsingRefreshToken(fastify); + const {token} = await fastify.googleOAuth2.getNewAccessTokenUsingRefreshToken(request.session); access_token = token.access_token; await db.update(sessions).set({ expires_at: token.expires_at, access_token: token.access_token - }); + }).where(eq(sessions.user_id, request.session.user_id)); } diff --git a/yarn.lock b/yarn.lock index 24f51ec..9fc58f4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -371,15 +371,12 @@ resolved "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz" integrity sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA== -"@fastify/cookie@^9.0.4": - version "9.3.1" - resolved "https://registry.npmjs.org/@fastify/cookie/-/cookie-9.3.1.tgz" - integrity sha512-h1NAEhB266+ZbZ0e9qUE6NnNR07i7DnNXWG9VbbZ8uC6O/hxHpl+Zoe5sw1yfdZ2U6XhToUGDnzQtWJdCaPwfg== - dependencies: - cookie-signature "^1.1.0" - fastify-plugin "^4.0.0" +"@fastify/busboy@^2.1.0": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.1.tgz#b9da6a878a371829a0502c9b6c1c143ef6663f4d" + integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA== -"@fastify/cookie@^9.3.1": +"@fastify/cookie@^9.0.4", "@fastify/cookie@^9.3.1": version "9.3.1" resolved "https://registry.yarnpkg.com/@fastify/cookie/-/cookie-9.3.1.tgz#48b89a356a23860c666e2fe522a084cc5c943d33" integrity sha512-h1NAEhB266+ZbZ0e9qUE6NnNR07i7DnNXWG9VbbZ8uC6O/hxHpl+Zoe5sw1yfdZ2U6XhToUGDnzQtWJdCaPwfg== @@ -400,7 +397,7 @@ resolved "https://registry.npmjs.org/@fastify/deepmerge/-/deepmerge-1.3.0.tgz" integrity sha512-J8TOSBq3SoZbDhM9+R/u77hP93gz/rajSA+K2kGyijPpORPWUXHUpTaleoj+92As0S9uPRP7Oi8IqMf0u+ro6A== -"@fastify/error@^3.4.0": +"@fastify/error@^3.0.0", "@fastify/error@^3.4.0": version "3.4.1" resolved "https://registry.npmjs.org/@fastify/error/-/error-3.4.1.tgz" integrity sha512-wWSvph+29GR783IhmvdwWnN4bUxTD01Vm5Xad4i7i1VuAOItLvbPAb69sb0IQ2N57yprvhNIwAP5B6xfKTmjmQ== @@ -412,6 +409,18 @@ dependencies: fast-json-stringify "^5.7.0" +"@fastify/multipart@^8.2.0": + version "8.2.0" + resolved "https://registry.yarnpkg.com/@fastify/multipart/-/multipart-8.2.0.tgz#90359b78eccd0f944cf145a2e907d167a3a6c731" + integrity sha512-OZ8nsyyoS2TV7Yeu3ZdrdDGsKUTAbfjrKC9jSxGgT2qdgek+BxpWX31ZubTrWMNZyU5xwk4ox6AvTjAbYWjrWg== + dependencies: + "@fastify/busboy" "^2.1.0" + "@fastify/deepmerge" "^1.0.0" + "@fastify/error" "^3.0.0" + fastify-plugin "^4.0.0" + secure-json-parse "^2.4.0" + stream-wormhole "^1.1.0" + "@fastify/oauth2@^7.8.0": version "7.8.0" resolved "https://registry.npmjs.org/@fastify/oauth2/-/oauth2-7.8.0.tgz" @@ -3485,10 +3494,10 @@ open@^9.1.0: is-inside-container "^1.0.0" is-wsl "^2.2.0" -openai@^4.38.2: - version "4.38.2" - resolved "https://registry.npmjs.org/openai/-/openai-4.38.2.tgz" - integrity sha512-M16ehj0D84Gjq5cjvBzXRb5X+UvtWlxPDRAWAWMC0EN+6nHqnULIn5fWWeiexDPup25FeSZYv/ldp8KefcZVJQ== +openai@^4.38.5: + version "4.38.5" + resolved "https://registry.yarnpkg.com/openai/-/openai-4.38.5.tgz#87de78eed9f7e63331fb6b1307d8c9dd986b39d0" + integrity sha512-Ym5GJL98ZhLJJ7enBx53jjG3vwN/fsB+Ozh46nnRZZS9W1NiYqbwkJ+sXd3dkCIiWIgcyyOPL2Zr8SQAzbpj3g== dependencies: "@types/node" "^18.11.18" "@types/node-fetch" "^2.6.4" @@ -4107,7 +4116,7 @@ safe-stable-stringify@^2.3.1: resolved "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz" integrity sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g== -secure-json-parse@^2.7.0: +secure-json-parse@^2.4.0, secure-json-parse@^2.7.0: version "2.7.0" resolved "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz" integrity sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw== @@ -4322,6 +4331,11 @@ stoppable@^1.1.0: resolved "https://registry.npmjs.org/stoppable/-/stoppable-1.1.0.tgz" integrity sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw== +stream-wormhole@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/stream-wormhole/-/stream-wormhole-1.1.0.tgz#300aff46ced553cfec642a05251885417693c33d" + integrity sha512-gHFfL3px0Kctd6Po0M8TzEvt3De/xu6cnRrjlfYNhwbhLPLwigI2t1nc6jrzNuaYg5C4YF78PPFuQPzRiqn9ew== + string-width@^4.2.0: version "4.2.3" resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz"