Omer/youtuber-blog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added queue

Browse Source
This commit is contained in:
Omer Sabic 2024-06-22 01:12:21 +02:00
parent e2fa79c3d1
commit b4bae7460f
8 changed files with 576 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/db/migrations/0011_normal_thunderball.sql Normal file
View File

@ -0,0 +1 @@
ALTER TYPE "article_status" ADD VALUE 'error';

486
src/db/migrations/meta/0011_snapshot.json Normal file
View File

@ -0,0 +1,486 @@
{
"id": "1b69d63a-4517-42bc-aa1f-0ae6ed4ac685",
"prevId": "9e95f80e-79e7-4af0-9a8e-327fb358ec26",
"version": "7",
"dialect": "postgresql",
"tables": {
"public.articles": {
"name": "articles",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "uuid",
"primaryKey": true,
"notNull": true,
"default": "gen_random_uuid()"
},
"site_id": {
"name": "site_id",
"type": "uuid",
"primaryKey": false,
"notNull": false
},
"content": {
"name": "content",
"type": "text",
"primaryKey": false,
"notNull": false
},
"source_video_id": {
"name": "source_video_id",
"type": "text",
"primaryKey": false,
"notNull": false
},
"title": {
"name": "title",
"type": "text",
"primaryKey": false,
"notNull": false
},
"seo_slug": {
"name": "seo_slug",
"type": "text",
"primaryKey": false,
"notNull": false
},
"is_public": {
"name": "is_public",
"type": "boolean",
"primaryKey": false,
"notNull": false,
"default": false
},
"views": {
"name": "views",
"type": "integer",
"primaryKey": false,
"notNull": false,
"default": 0
},
"seo_title": {
"name": "seo_title",
"type": "text",
"primaryKey": false,
"notNull": false
},
"seo_description": {
"name": "seo_description",
"type": "text",
"primaryKey": false,
"notNull": false
},
"excerp": {
"name": "excerp",
"type": "text",
"primaryKey": false,
"notNull": false
},
"status": {
"name": "status",
"type": "article_status",
"typeSchema": "public",
"primaryKey": false,
"notNull": false,
"default": "'queued'"
},
"created_at": {
"name": "created_at",
"type": "timestamp",
"primaryKey": false,
"notNull": false,
"default": "now()"
}
},
"indexes": {},
"foreignKeys": {
"articles_site_id_sites_id_fk": {
"name": "articles_site_id_sites_id_fk",
"tableFrom": "articles",
"tableTo": "sites",
"columnsFrom": [
"site_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
},
"public.sessions": {
"name": "sessions",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "uuid",
"primaryKey": true,
"notNull": true,
"default": "gen_random_uuid()"
},
"user_id": {
"name": "user_id",
"type": "uuid",
"primaryKey": false,
"notNull": false
},
"google_access_token": {
"name": "google_access_token",
"type": "text",
"primaryKey": false,
"notNull": false
},
"google_refresh_token": {
"name": "google_refresh_token",
"type": "text",
"primaryKey": false,
"notNull": false
},
"expires_at": {
"name": "expires_at",
"type": "timestamp",
"primaryKey": false,
"notNull": false
}
},
"indexes": {},
"foreignKeys": {
"sessions_user_id_users_id_fk": {
"name": "sessions_user_id_users_id_fk",
"tableFrom": "sessions",
"tableTo": "users",
"columnsFrom": [
"user_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
},
"public.signups": {
"name": "signups",
"schema": "",
"columns": {
"email": {
"name": "email",
"type": "text",
"primaryKey": false,
"notNull": true
},
"site_id": {
"name": "site_id",
"type": "uuid",
"primaryKey": false,
"notNull": true
},
"source": {
"name": "source",
"type": "text",
"primaryKey": false,
"notNull": false
},
"created_at": {
"name": "created_at",
"type": "timestamp",
"primaryKey": false,
"notNull": true,
"default": "now()"
}
},
"indexes": {},
"foreignKeys": {
"signups_site_id_sites_id_fk": {
"name": "signups_site_id_sites_id_fk",
"tableFrom": "signups",
"tableTo": "sites",
"columnsFrom": [
"site_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
},
"public.sites": {
"name": "sites",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "uuid",
"primaryKey": true,
"notNull": true,
"default": "gen_random_uuid()"
},
"user_id": {
"name": "user_id",
"type": "uuid",
"primaryKey": false,
"notNull": false
},
"name": {
"name": "name",
"type": "text",
"primaryKey": false,
"notNull": false
},
"primary_color_hex": {
"name": "primary_color_hex",
"type": "varchar(6)",
"primaryKey": false,
"notNull": true,
"default": "'c4ced4'"
},
"secondary_color_hex": {
"name": "secondary_color_hex",
"type": "varchar(6)",
"primaryKey": false,
"notNull": true,
"default": "'27251f'"
},
"text_color_hex": {
"name": "text_color_hex",
"type": "varchar(6)",
"primaryKey": false,
"notNull": true,
"default": "'ffffff'"
},
"title": {
"name": "title",
"type": "text",
"primaryKey": false,
"notNull": false,
"default": "'The best blog in the world!'"
},
"subtitle": {
"name": "subtitle",
"type": "text",
"primaryKey": false,
"notNull": false,
"default": "'Some extra info about the best blog in the world!'"
},
"domain": {
"name": "domain",
"type": "text",
"primaryKey": false,
"notNull": false,
"default": "NULL"
},
"send_freebie": {
"name": "send_freebie",
"type": "boolean",
"primaryKey": false,
"notNull": false,
"default": false
},
"freebie_name": {
"name": "freebie_name",
"type": "text",
"primaryKey": false,
"notNull": false,
"default": "''"
},
"freebie_url": {
"name": "freebie_url",
"type": "text",
"primaryKey": false,
"notNull": false,
"default": "''"
},
"freebie_text": {
"name": "freebie_text",
"type": "text",
"primaryKey": false,
"notNull": false,
"default": "''"
},
"freebie_image_url": {
"name": "freebie_image_url",
"type": "text",
"primaryKey": false,
"notNull": false,
"default": "''"
},
"subdomain_slug": {
"name": "subdomain_slug",
"type": "text",
"primaryKey": false,
"notNull": false
},
"social_medias": {
"name": "social_medias",
"type": "jsonb",
"primaryKey": false,
"notNull": false
},
"auto_publish": {
"name": "auto_publish",
"type": "boolean",
"primaryKey": false,
"notNull": false,
"default": false
},
"pubsub_expiry": {
"name": "pubsub_expiry",
"type": "timestamp",
"primaryKey": false,
"notNull": false
}
},
"indexes": {},
"foreignKeys": {
"sites_user_id_users_id_fk": {
"name": "sites_user_id_users_id_fk",
"tableFrom": "sites",
"tableTo": "users",
"columnsFrom": [
"user_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {
"sites_domain_unique": {
"name": "sites_domain_unique",
"nullsNotDistinct": false,
"columns": [
"domain"
]
}
}
},
"public.users": {
"name": "users",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "uuid",
"primaryKey": true,
"notNull": true,
"default": "gen_random_uuid()"
},
"google_id": {
"name": "google_id",
"type": "text",
"primaryKey": false,
"notNull": false
},
"stripe_id": {
"name": "stripe_id",
"type": "text",
"primaryKey": false,
"notNull": false
},
"name": {
"name": "name",
"type": "text",
"primaryKey": false,
"notNull": false
},
"email": {
"name": "email",
"type": "text",
"primaryKey": false,
"notNull": false
},
"channel_id": {
"name": "channel_id",
"type": "text",
"primaryKey": false,
"notNull": false
},
"uploads_playlist_id": {
"name": "uploads_playlist_id",
"type": "text",
"primaryKey": false,
"notNull": false
},
"subscription_tier": {
"name": "subscription_tier",
"type": "subscription_tier",
"typeSchema": "public",
"primaryKey": false,
"notNull": true,
"default": "'free'"
},
"tokens": {
"name": "tokens",
"type": "integer",
"primaryKey": false,
"notNull": true,
"default": 200
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
},
"public.waitlist": {
"name": "waitlist",
"schema": "",
"columns": {
"email": {
"name": "email",
"type": "text",
"primaryKey": false,
"notNull": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
}
},
"enums": {
"public.article_status": {
"name": "article_status",
"schema": "public",
"values": [
"queued",
"transcribing",
"generating",
"done",
"error"
]
},
"public.subscription_tier": {
"name": "subscription_tier",
"schema": "public",
"values": [
"free",
"basic",
"pro",
"enterprise"
]
}
},
"schemas": {},
"_meta": {
"columns": {},
"schemas": {},
"tables": {}
}
}

7
src/db/migrations/meta/_journal.json
View File

@ -78,6 +78,13 @@
"when": 1718818704691, "when": 1718818704691,
"tag": "0010_complex_malice", "tag": "0010_complex_malice",
"breakpoints": true "breakpoints": true
},
{
"idx": 11,
"version": "7",
"when": 1719007763731,
"tag": "0011_normal_thunderball",
"breakpoints": true
} }
] ]
} }

2
src/db/schemas.js
View File

@ -2,7 +2,7 @@ import { sql } from "drizzle-orm";
import { boolean, date, integer, jsonb, pgEnum, pgTable, text, timestamp, unique, uniqueIndex, uuid, varchar } from "drizzle-orm/pg-core"; import { boolean, date, integer, jsonb, pgEnum, pgTable, text, timestamp, unique, uniqueIndex, uuid, varchar } from "drizzle-orm/pg-core";
export const subscription_enum = pgEnum("subscription_tier", ["free", "basic", "pro", "enterprise"]) export const subscription_enum = pgEnum("subscription_tier", ["free", "basic", "pro", "enterprise"])
export const article_status_enum = pgEnum("article_status", ["queued", "transcribing", "generating", "done"]) export const article_status_enum = pgEnum("article_status", ["queued", "transcribing", "generating", "done", "error"])
export const users = pgTable("users", { export const users = pgTable("users", {
id: uuid("id").defaultRandom().primaryKey(), id: uuid("id").defaultRandom().primaryKey(),

14
src/routes/blog.js
View File

@ -83,23 +83,11 @@ export const blogRoutes = (fastify, _, done) => {
total: sql`COUNT(*)` total: sql`COUNT(*)`
}).from(articlesTable).where(clause); }).from(articlesTable).where(clause);
let queue = [];
if (mine) {
queue = await db.select({
id: articlesTable.id,
title: articlesTable.title,
created_at: articlesTable.created_at,
status: articlesTable.status,
source_id: articlesTable.source_video_id
}).from(articlesTable).where(and(eq(articlesTable.site_id, site.id), not(eq(articlesTable.status, "done"))));
}
response.send({ response.send({
success: true, success: true,
articles: results, articles: results,
total_articles: total, total_articles: total,
site, site
queue
}); });
} catch (e) { } catch (e) {
console.log(e); console.log(e);

91
src/routes/dashboard.js
View File

@ -1,10 +1,10 @@
/** @typedef {import("fastify").FastifyInstance} FastifyInstance */ /** @typedef {import("fastify").FastifyInstance} FastifyInstance */
import { and, desc, eq, getTableColumns, sql } from "drizzle-orm"; import { and, desc, eq, getTableColumns, inArray, notInArray, sql } from "drizzle-orm";
import { db } from "../db/index.js"; import { db } from "../db/index.js";
import { articles, articles as articlesTable, signups as signupsTable, sites, users } from "../db/schemas.js"; import { articles, articles as articlesTable, signups as signupsTable, sites, users } from "../db/schemas.js";
import { authMiddleware, authMiddlewareFn } from "../modules/middleware.js"; import { authMiddleware, authMiddlewareFn } from "../modules/middleware.js";
import { jsonToCsv, createBlogFromCaptions, createArticleSlug, getVideoById, env, getWhisperCaptions, getVideoWithCaptions } from "../utils/index.js"; import { jsonToCsv, createBlogFromCaptions, createArticleSlug, getVideoById, env, getWhisperCaptions, getVideoDetails } from "../utils/index.js";
const websubVerifyToken = "FQNI4Suzih"; const websubVerifyToken = "FQNI4Suzih";
@ -202,6 +202,7 @@ export const dashboardRoutes = (fastify, _, done) => {
}, },
} }
}, async (req, reply) => { }, async (req, reply) => {
let article;
try { try {
const [{ tokens }] = await db.select({ const [{ tokens }] = await db.select({
tokens: users.tokens tokens: users.tokens
@ -228,32 +229,42 @@ export const dashboardRoutes = (fastify, _, done) => {
return; return;
} }
// youtube-dl --write-sub --sub-lang en --skip-download URL const site = await db.select().from(sites).where(eq(sites.user_id, req.session.user_id));
const video_data = await getVideoDetails(req.body.video_id);
const video_data = await getVideoWithCaptions(req.body.video_id);
reply.send({ reply.send({
success: true success: true
}); });
article = (await db.insert(articlesTable).values({
title: video_data.title,
source_video_id: match[2],
status: "transcribing",
site_id: site[0].id,
is_public: false
}).returning({ id: articlesTable.id }))[0];
video_data.captions = await getWhisperCaptions(req.body.video_id);
// const video_data = await getVideoById(access_token, req.body.video_id); // const video_data = await getVideoById(access_token, req.body.video_id);
await db.update(articlesTable).set({
status: "generating"
}).where(eq(articlesTable.id, article.id));
const blog_content_json = await createBlogFromCaptions(video_data.captions, { title: video_data.title, description: video_data.description }, req.body); const blog_content_json = await createBlogFromCaptions(video_data.captions, { title: video_data.title, description: video_data.description }, req.body);
// TODO: once I add multiple sites per user, this should come from the client // TODO: once I add multiple sites per user, this should come from the client
const site = await db.select().from(sites).where(eq(sites.user_id, req.session.user_id));
await db.insert(articlesTable).values({ await db.update(articlesTable).set({
site_id: site[0].id,
title: blog_content_json.title, title: blog_content_json.title,
content: blog_content_json.content, content: blog_content_json.content,
meta_title: blog_content_json.meta_title, meta_title: blog_content_json.meta_title,
meta_desc: blog_content_json.meta_desc, meta_desc: blog_content_json.meta_desc,
excerp: blog_content_json.excerp, excerp: blog_content_json.excerp,
source_video_id: req.body.video_id,
seo_slug: createArticleSlug(blog_content_json.title), seo_slug: createArticleSlug(blog_content_json.title),
is_public: false is_public: false,
}).returning({ id: articlesTable.id }); status: "done"
}).where(eq(articlesTable.id, article.id));
await db.update(users).set({ await db.update(users).set({
tokens: tokens - 1 tokens: tokens - 1
@ -261,6 +272,41 @@ export const dashboardRoutes = (fastify, _, done) => {
} catch (e) { } catch (e) {
console.log(e); console.log(e);
article ? await db.update(articlesTable).set({
status: "error"
}).where(eq(articlesTable.id), article.id) : "";
reply.status(500).send({
success: false,
message: "problem_creating_article"
})
}
});
fastify.get("/queue", async (req, reply) => {
try {
const site_id = (await db.select({ site_id: sites.id }).from(sites).where(eq(sites.user_id, req.session.user_id)))[0].site_id;
if(!site_id) throw new Error("Could not find site_id");
const queue = await db.select({
id: articlesTable.id,
title: articlesTable.title,
status: articlesTable.status,
created_at: articlesTable.created_at,
source_id: articlesTable.source_video_id
}).from(articlesTable)
.where(and(eq(articlesTable.site_id, site_id), notInArray(articles.status, ["done", "error"])))
.orderBy(desc(articlesTable.created_at));
reply.send({
success: true,
queue
});
} catch (err) {
reply.status(500).send({
success: false,
message: err.message
})
} }
}); });
@ -287,7 +333,7 @@ export const dashboardRoutes = (fastify, _, done) => {
} }
}, async (req, reply) => { }, async (req, reply) => {
}) });
fastify.put("/website", { fastify.put("/website", {
schema: { schema: {
@ -399,24 +445,3 @@ export const dashboardRoutes = (fastify, _, done) => {
done(); done();
}; };
function convertSRV1ToPlainText(transcript) {
// Define regular expression to extract text content
const textRegex = /<text[^>]+>(.*?)<\/text>/g;
// Initialize an empty array to hold the plain text lines
const plainTextLines = [];
// Match text segments using regular expression
let match;
while ((match = textRegex.exec(transcript)) !== null) {
// Extract text content and remove any HTML tags
const textContent = match[1].replace(/<[^>]+>/g, '');
// Add text content to the plain text lines array
plainTextLines.push(textContent);
}
// Join the plain text lines into a single string and return it
return plainTextLines.join('\n');
}

1
src/utils/ai.js
View File

@ -184,6 +184,7 @@ export async function getWhisperCaptions(video_url) {
*/ */
function getYouTubeAudioBuffer(videoUrl) { function getYouTubeAudioBuffer(videoUrl) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
if(!videoUrl) reject();
// Download audio from YouTube // Download audio from YouTube
const audioStream = ytdl(videoUrl, { filter: "audioonly", quality: "lowestaudio" }); const audioStream = ytdl(videoUrl, { filter: "audioonly", quality: "lowestaudio" });

13
src/utils/youtube.js
View File

@ -4,6 +4,8 @@ import { sessions } from '../db/schemas.js';
import { google } from 'googleapis'; import { google } from 'googleapis';
import ytdl from 'ytdl-core'; import ytdl from 'ytdl-core';
import { getWhisperCaptions } from './ai.js'; import { getWhisperCaptions } from './ai.js';
import { articles as articlesTable } from "../db/schemas.js";
const service = google.youtube("v3"); const service = google.youtube("v3");
export async function getVideoById(access_token, video_id) { export async function getVideoById(access_token, video_id) {
@ -116,9 +118,18 @@ export async function getAccessToken(fastify, request) {
return access_token; return access_token;
} }
export async function getVideoWithCaptions(video_url) { export async function getVideoDetails(video_url) {
if (!(ytdl.validateURL(video_url))) throw new Error("Invalid Youtube URL"); if (!(ytdl.validateURL(video_url))) throw new Error("Invalid Youtube URL");
const info = await ytdl.getBasicInfo(video_url); const info = await ytdl.getBasicInfo(video_url);
return {
title: info.videoDetails.title,
description: info.videoDetails.description
}
}
export async function getVideoWithCaptions(video_url) {
const info = getVideoDetails(video_url);
const captions = await getWhisperCaptions(video_url); const captions = await getWhisperCaptions(video_url);
return { return {