diff --git a/src/db/migrations/0011_normal_thunderball.sql b/src/db/migrations/0011_normal_thunderball.sql new file mode 100644 index 0000000..ca830d1 --- /dev/null +++ b/src/db/migrations/0011_normal_thunderball.sql @@ -0,0 +1 @@ +ALTER TYPE "article_status" ADD VALUE 'error'; \ No newline at end of file diff --git a/src/db/migrations/meta/0011_snapshot.json b/src/db/migrations/meta/0011_snapshot.json new file mode 100644 index 0000000..f284ab8 --- /dev/null +++ b/src/db/migrations/meta/0011_snapshot.json @@ -0,0 +1,486 @@ +{ + "id": "1b69d63a-4517-42bc-aa1f-0ae6ed4ac685", + "prevId": "9e95f80e-79e7-4af0-9a8e-327fb358ec26", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.articles": { + "name": "articles", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "site_id": { + "name": "site_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_video_id": { + "name": "source_video_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "seo_slug": { + "name": "seo_slug", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_public": { + "name": "is_public", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "views": { + "name": "views", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "seo_title": { + "name": "seo_title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "seo_description": { + "name": "seo_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "excerp": { + "name": "excerp", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "article_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": false, + "default": "'queued'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "articles_site_id_sites_id_fk": { + "name": "articles_site_id_sites_id_fk", + "tableFrom": "articles", + "tableTo": "sites", + "columnsFrom": [ + "site_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "google_access_token": { + "name": "google_access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "google_refresh_token": { + "name": "google_refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.signups": { + "name": "signups", + "schema": "", + "columns": { + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "site_id": { + "name": "site_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "signups_site_id_sites_id_fk": { + "name": "signups_site_id_sites_id_fk", + "tableFrom": "signups", + "tableTo": "sites", + "columnsFrom": [ + "site_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.sites": { + "name": "sites", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "primary_color_hex": { + "name": "primary_color_hex", + "type": "varchar(6)", + "primaryKey": false, + "notNull": true, + "default": "'c4ced4'" + }, + "secondary_color_hex": { + "name": "secondary_color_hex", + "type": "varchar(6)", + "primaryKey": false, + "notNull": true, + "default": "'27251f'" + }, + "text_color_hex": { + "name": "text_color_hex", + "type": "varchar(6)", + "primaryKey": false, + "notNull": true, + "default": "'ffffff'" + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'The best blog in the world!'" + }, + "subtitle": { + "name": "subtitle", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'Some extra info about the best blog in the world!'" + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "NULL" + }, + "send_freebie": { + "name": "send_freebie", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "freebie_name": { + "name": "freebie_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "''" + }, + "freebie_url": { + "name": "freebie_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "''" + }, + "freebie_text": { + "name": "freebie_text", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "''" + }, + "freebie_image_url": { + "name": "freebie_image_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "''" + }, + "subdomain_slug": { + "name": "subdomain_slug", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "social_medias": { + "name": "social_medias", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "auto_publish": { + "name": "auto_publish", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "pubsub_expiry": { + "name": "pubsub_expiry", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "sites_user_id_users_id_fk": { + "name": "sites_user_id_users_id_fk", + "tableFrom": "sites", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sites_domain_unique": { + "name": "sites_domain_unique", + "nullsNotDistinct": false, + "columns": [ + "domain" + ] + } + } + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "google_id": { + "name": "google_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "stripe_id": { + "name": "stripe_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "uploads_playlist_id": { + "name": "uploads_playlist_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "subscription_tier": { + "name": "subscription_tier", + "type": "subscription_tier", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'free'" + }, + "tokens": { + "name": "tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 200 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.waitlist": { + "name": "waitlist", + "schema": "", + "columns": { + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + } + }, + "enums": { + "public.article_status": { + "name": "article_status", + "schema": "public", + "values": [ + "queued", + "transcribing", + "generating", + "done", + "error" + ] + }, + "public.subscription_tier": { + "name": "subscription_tier", + "schema": "public", + "values": [ + "free", + "basic", + "pro", + "enterprise" + ] + } + }, + "schemas": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/src/db/migrations/meta/_journal.json b/src/db/migrations/meta/_journal.json index 080df13..aa9fae5 100644 --- a/src/db/migrations/meta/_journal.json +++ b/src/db/migrations/meta/_journal.json @@ -78,6 +78,13 @@ "when": 1718818704691, "tag": "0010_complex_malice", "breakpoints": true + }, + { + "idx": 11, + "version": "7", + "when": 1719007763731, + "tag": "0011_normal_thunderball", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/db/schemas.js b/src/db/schemas.js index d1850c5..eff9887 100644 --- a/src/db/schemas.js +++ b/src/db/schemas.js @@ -2,7 +2,7 @@ import { sql } from "drizzle-orm"; import { boolean, date, integer, jsonb, pgEnum, pgTable, text, timestamp, unique, uniqueIndex, uuid, varchar } from "drizzle-orm/pg-core"; export const subscription_enum = pgEnum("subscription_tier", ["free", "basic", "pro", "enterprise"]) -export const article_status_enum = pgEnum("article_status", ["queued", "transcribing", "generating", "done"]) +export const article_status_enum = pgEnum("article_status", ["queued", "transcribing", "generating", "done", "error"]) export const users = pgTable("users", { id: uuid("id").defaultRandom().primaryKey(), diff --git a/src/routes/blog.js b/src/routes/blog.js index a3f5836..34274f0 100644 --- a/src/routes/blog.js +++ b/src/routes/blog.js @@ -82,24 +82,12 @@ export const blogRoutes = (fastify, _, done) => { const [{ total }] = await db.select({ total: sql`COUNT(*)` }).from(articlesTable).where(clause); - - let queue = []; - if (mine) { - queue = await db.select({ - id: articlesTable.id, - title: articlesTable.title, - created_at: articlesTable.created_at, - status: articlesTable.status, - source_id: articlesTable.source_video_id - }).from(articlesTable).where(and(eq(articlesTable.site_id, site.id), not(eq(articlesTable.status, "done")))); - } - + response.send({ success: true, articles: results, total_articles: total, - site, - queue + site }); } catch (e) { console.log(e); diff --git a/src/routes/dashboard.js b/src/routes/dashboard.js index 59b28c7..3e26d29 100644 --- a/src/routes/dashboard.js +++ b/src/routes/dashboard.js @@ -1,10 +1,10 @@ /** @typedef {import("fastify").FastifyInstance} FastifyInstance */ -import { and, desc, eq, getTableColumns, sql } from "drizzle-orm"; +import { and, desc, eq, getTableColumns, inArray, notInArray, sql } from "drizzle-orm"; import { db } from "../db/index.js"; import { articles, articles as articlesTable, signups as signupsTable, sites, users } from "../db/schemas.js"; import { authMiddleware, authMiddlewareFn } from "../modules/middleware.js"; -import { jsonToCsv, createBlogFromCaptions, createArticleSlug, getVideoById, env, getWhisperCaptions, getVideoWithCaptions } from "../utils/index.js"; +import { jsonToCsv, createBlogFromCaptions, createArticleSlug, getVideoById, env, getWhisperCaptions, getVideoDetails } from "../utils/index.js"; const websubVerifyToken = "FQNI4Suzih"; @@ -202,6 +202,7 @@ export const dashboardRoutes = (fastify, _, done) => { }, } }, async (req, reply) => { + let article; try { const [{ tokens }] = await db.select({ tokens: users.tokens @@ -227,33 +228,43 @@ export const dashboardRoutes = (fastify, _, done) => { }); return; } - - // youtube-dl --write-sub --sub-lang en --skip-download URL - - const video_data = await getVideoWithCaptions(req.body.video_id); + const site = await db.select().from(sites).where(eq(sites.user_id, req.session.user_id)); + + const video_data = await getVideoDetails(req.body.video_id); reply.send({ success: true }); + article = (await db.insert(articlesTable).values({ + title: video_data.title, + source_video_id: match[2], + status: "transcribing", + site_id: site[0].id, + is_public: false + }).returning({ id: articlesTable.id }))[0]; + + video_data.captions = await getWhisperCaptions(req.body.video_id); + // const video_data = await getVideoById(access_token, req.body.video_id); + await db.update(articlesTable).set({ + status: "generating" + }).where(eq(articlesTable.id, article.id)); const blog_content_json = await createBlogFromCaptions(video_data.captions, { title: video_data.title, description: video_data.description }, req.body); // TODO: once I add multiple sites per user, this should come from the client - const site = await db.select().from(sites).where(eq(sites.user_id, req.session.user_id)); - await db.insert(articlesTable).values({ - site_id: site[0].id, + await db.update(articlesTable).set({ title: blog_content_json.title, content: blog_content_json.content, meta_title: blog_content_json.meta_title, meta_desc: blog_content_json.meta_desc, excerp: blog_content_json.excerp, - source_video_id: req.body.video_id, seo_slug: createArticleSlug(blog_content_json.title), - is_public: false - }).returning({ id: articlesTable.id }); + is_public: false, + status: "done" + }).where(eq(articlesTable.id, article.id)); await db.update(users).set({ tokens: tokens - 1 @@ -261,6 +272,41 @@ export const dashboardRoutes = (fastify, _, done) => { } catch (e) { console.log(e); + + article ? await db.update(articlesTable).set({ + status: "error" + }).where(eq(articlesTable.id), article.id) : ""; + + reply.status(500).send({ + success: false, + message: "problem_creating_article" + }) + } + }); + + fastify.get("/queue", async (req, reply) => { + try { + const site_id = (await db.select({ site_id: sites.id }).from(sites).where(eq(sites.user_id, req.session.user_id)))[0].site_id; + if(!site_id) throw new Error("Could not find site_id"); + const queue = await db.select({ + id: articlesTable.id, + title: articlesTable.title, + status: articlesTable.status, + created_at: articlesTable.created_at, + source_id: articlesTable.source_video_id + }).from(articlesTable) + .where(and(eq(articlesTable.site_id, site_id), notInArray(articles.status, ["done", "error"]))) + .orderBy(desc(articlesTable.created_at)); + + reply.send({ + success: true, + queue + }); + } catch (err) { + reply.status(500).send({ + success: false, + message: err.message + }) } }); @@ -287,7 +333,7 @@ export const dashboardRoutes = (fastify, _, done) => { } }, async (req, reply) => { - }) + }); fastify.put("/website", { schema: { @@ -369,9 +415,9 @@ export const dashboardRoutes = (fastify, _, done) => { }); } - if(site.auto_publish !== req.body.auto_publish) { + if (site.auto_publish !== req.body.auto_publish) { const [user] = await db.select().from(users).where(eq(users.id, req.session.user_id)); - if(!user) throw new Error("Problem getting user"); + if (!user) throw new Error("Problem getting user"); let form = new FormData(); form.set("hub.callback", env.PUBLIC_API_URL + "/webhooks/youtube"); form.set("hub.topic", "https://www.youtube.com/xml/feeds/videos.xml?channel_id=" + user.channel_id); @@ -398,25 +444,4 @@ export const dashboardRoutes = (fastify, _, done) => { }); done(); -}; - -function convertSRV1ToPlainText(transcript) { - // Define regular expression to extract text content - const textRegex = /]+>(.*?)<\/text>/g; - - // Initialize an empty array to hold the plain text lines - const plainTextLines = []; - - // Match text segments using regular expression - let match; - while ((match = textRegex.exec(transcript)) !== null) { - // Extract text content and remove any HTML tags - const textContent = match[1].replace(/<[^>]+>/g, ''); - - // Add text content to the plain text lines array - plainTextLines.push(textContent); - } - - // Join the plain text lines into a single string and return it - return plainTextLines.join('\n'); -} \ No newline at end of file +}; \ No newline at end of file diff --git a/src/utils/ai.js b/src/utils/ai.js index 93747ef..ffaecd1 100644 --- a/src/utils/ai.js +++ b/src/utils/ai.js @@ -184,6 +184,7 @@ export async function getWhisperCaptions(video_url) { */ function getYouTubeAudioBuffer(videoUrl) { return new Promise((resolve, reject) => { + if(!videoUrl) reject(); // Download audio from YouTube const audioStream = ytdl(videoUrl, { filter: "audioonly", quality: "lowestaudio" }); diff --git a/src/utils/youtube.js b/src/utils/youtube.js index ddc596f..8232e69 100644 --- a/src/utils/youtube.js +++ b/src/utils/youtube.js @@ -4,6 +4,8 @@ import { sessions } from '../db/schemas.js'; import { google } from 'googleapis'; import ytdl from 'ytdl-core'; import { getWhisperCaptions } from './ai.js'; +import { articles as articlesTable } from "../db/schemas.js"; + const service = google.youtube("v3"); export async function getVideoById(access_token, video_id) { @@ -24,7 +26,7 @@ export async function getVideoById(access_token, video_id) { * @returns {Promise} */ export async function getVideosFromPlaylist(access_token, playlist_id) { - if(!access_token) return []; + if (!access_token) return []; let videos; try { videos = await service.playlistItems.list({ @@ -100,9 +102,9 @@ export async function getAccessToken(fastify, request) { if ((new Date().getTime() + 10) > request.session.expires_at) { /** @type {import('@fastify/oauth2').Token} */ - const { token } = await fastify.googleOAuth2.getNewAccessTokenUsingRefreshToken(request.session).catch(e => ({token: null})); + const { token } = await fastify.googleOAuth2.getNewAccessTokenUsingRefreshToken(request.session).catch(e => ({ token: null })); - if(!token) return "" + if (!token) return "" access_token = token.access_token; @@ -116,9 +118,18 @@ export async function getAccessToken(fastify, request) { return access_token; } -export async function getVideoWithCaptions(video_url) { - if(!(ytdl.validateURL(video_url))) throw new Error("Invalid Youtube URL"); +export async function getVideoDetails(video_url) { + if (!(ytdl.validateURL(video_url))) throw new Error("Invalid Youtube URL"); const info = await ytdl.getBasicInfo(video_url); + return { + title: info.videoDetails.title, + description: info.videoDetails.description + } +} + +export async function getVideoWithCaptions(video_url) { + const info = getVideoDetails(video_url); + const captions = await getWhisperCaptions(video_url); return {