orginized
This commit is contained in:
test
parent
786f3934f9
commit
565ee99aac
@ -1,25 +1,14 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
#from langchain.embeddings import FastEmbedEmbeddings
|
||||
#from langchain.schema.output_parser import StrOutputParser
|
||||
from langchain.document_loaders import UnstructuredFileLoader, WebBaseLoader, YoutubeLoader, TextLoader
|
||||
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
||||
#from langchain.schema.runnable import RunnablePassthrough
|
||||
#from langchain.prompts import PromptTemplate
|
||||
#from langchain.schema.document import Document
|
||||
from langchain.vectorstores.utils import filter_complex_metadata
|
||||
#from langchain_community.embeddings import OllamaEmbeddings
|
||||
#import mimetypes
|
||||
import os
|
||||
import json
|
||||
import requests
|
||||
#from pathlib import Path
|
||||
from rich.markdown import Markdown
|
||||
from rich.console import Console
|
||||
import re
|
||||
import sys
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
#from youtube_transcript_api import YouTubeTranscriptApi
|
||||
import nltk
|
||||
from tqdm import tqdm
|
||||
from markdown_pdf import Section, MarkdownPdf
|
||||
@ -37,21 +26,23 @@ title = os.path.basename(filename).replace(file_extension, '')
|
||||
pdf.add_section(Section(f"# {title}\n", toc=True))
|
||||
|
||||
|
||||
model = "dolphin-mistral:latest"
|
||||
#model = "mistral:latest"
|
||||
model = "llama3:latest"
|
||||
# model = "mistral:latest"
|
||||
|
||||
vector_store = None
|
||||
retriever = None
|
||||
chain = None
|
||||
docs = None
|
||||
|
||||
|
||||
def is_bulletpoint(s):
|
||||
for char in s[:5]:
|
||||
if char.isdigit():
|
||||
return True
|
||||
return False
|
||||
|
||||
def generate_text(model, prompt, system = ""):
|
||||
|
||||
def generate_text(model, prompt, system=""):
|
||||
url = "http://localhost:11434/api/generate"
|
||||
data = {
|
||||
"model": model,
|
||||
@ -84,15 +75,14 @@ def is_url(string):
|
||||
|
||||
|
||||
|
||||
#text_splitter = RecursiveCharacterTextSplitter(chunk_size=2048, chunk_overlap=100)
|
||||
# text_splitter = RecursiveCharacterTextSplitter(chunk_size=2048, chunk_overlap=100)
|
||||
text_splitter = RecursiveCharacterTextSplitter(chunk_size=4096, chunk_overlap=100)
|
||||
|
||||
# Checking if url or if file path
|
||||
if is_url(file_input):
|
||||
|
||||
|
||||
# See if youtube link
|
||||
if isyoutubevideo(file_input) == True:
|
||||
if isyoutubevideo(file_input):
|
||||
print("Loading youtube video...")
|
||||
# Prepare youtube url for transcript extraction
|
||||
#parsed_url = urlparse(file_input)
|
||||
@ -108,7 +98,6 @@ if is_url(file_input):
|
||||
# Extract and load webpage text
|
||||
docs = WebBaseLoader(file_input).load()
|
||||
|
||||
|
||||
# Prepare text
|
||||
docs = text_splitter.split_documents(docs)
|
||||
docs = filter_complex_metadata(docs)
|
||||
@ -124,10 +113,6 @@ else:
|
||||
docs = filter_complex_metadata(docs)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
outline = ""
|
||||
pre_summery = ""
|
||||
|
||||
@ -164,11 +149,11 @@ Write a paragraph summary of the following CHUNK of sql code.
|
||||
# CHUNK
|
||||
{chunk_text}
|
||||
"""
|
||||
|
||||
|
||||
outline = generate_text(model, prompt, system_prompt)
|
||||
|
||||
|
||||
bullet_point = False
|
||||
|
||||
|
||||
for x in outline.split("\n"):
|
||||
if is_bulletpoint(x):
|
||||
bullet_point = True
|
||||
@ -180,21 +165,17 @@ Write a paragraph summary of the following CHUNK of sql code.
|
||||
else:
|
||||
outline = outline.replace("\n", " ")
|
||||
|
||||
|
||||
pre_summery += "\n\n" + str(count) + "." + outline
|
||||
|
||||
#print("\n\n--------------------------------------------------------------------------------------------")
|
||||
#print(outline)
|
||||
# print("\n\n--------------------------------------------------------------------------------------------")
|
||||
# print(outline)
|
||||
|
||||
|
||||
bar.close()
|
||||
|
||||
nltk_tokens = nltk.word_tokenize(pre_summery)
|
||||
print("\nNumber of Tokens: ", len(nltk_tokens))
|
||||
print("Compression Ratio: ", round(len(nltk_tokens_init)/len(nltk_tokens),1))
|
||||
|
||||
|
||||
|
||||
print("Compression Ratio: ", round(len(nltk_tokens_init)/len(nltk_tokens), 1))
|
||||
|
||||
|
||||
# Final Summary
|
||||
@ -218,7 +199,5 @@ pdf.meta["title"] = title
|
||||
pdf.meta["author"] = "locker98"
|
||||
pdf.save(f"{title}.pdf")
|
||||
|
||||
# print(f"\n\n\n{pre_summery}")
|
||||
|
||||
|
||||
|
||||
#print(f"\n\n\n{pre_summery}")
|
||||
@ -1,25 +1,14 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
#from langchain.embeddings import FastEmbedEmbeddings
|
||||
#from langchain.schema.output_parser import StrOutputParser
|
||||
from langchain.document_loaders import UnstructuredFileLoader, WebBaseLoader, YoutubeLoader, TextLoader
|
||||
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
||||
#from langchain.schema.runnable import RunnablePassthrough
|
||||
#from langchain.prompts import PromptTemplate
|
||||
#from langchain.schema.document import Document
|
||||
from langchain.vectorstores.utils import filter_complex_metadata
|
||||
#from langchain_community.embeddings import OllamaEmbeddings
|
||||
#import mimetypes
|
||||
import os
|
||||
import json
|
||||
import requests
|
||||
#from pathlib import Path
|
||||
from rich.markdown import Markdown
|
||||
from rich.console import Console
|
||||
import re
|
||||
import sys
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
#from youtube_transcript_api import YouTubeTranscriptApi
|
||||
import nltk
|
||||
from tqdm import tqdm
|
||||
from markdown_pdf import Section, MarkdownPdf
|
||||
@ -37,15 +26,23 @@ title = os.path.basename(filename).replace(file_extension, '')
|
||||
pdf.add_section(Section(f"# {title}\n", toc=True))
|
||||
|
||||
|
||||
model = "dolphin-mistral:latest"
|
||||
#model = "mistral:latest"
|
||||
model = "llama3:latest"
|
||||
# model = "mistral:latest"
|
||||
|
||||
vector_store = None
|
||||
retriever = None
|
||||
chain = None
|
||||
docs = None
|
||||
|
||||
def generate_text(model, prompt, system = ""):
|
||||
|
||||
def is_bulletpoint(s):
|
||||
for char in s[:5]:
|
||||
if char.isdigit():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def generate_text(model, prompt, system=""):
|
||||
url = "http://localhost:11434/api/generate"
|
||||
data = {
|
||||
"model": model,
|
||||
@ -53,7 +50,7 @@ def generate_text(model, prompt, system = ""):
|
||||
"system": system,
|
||||
"stream": False,
|
||||
"options": {
|
||||
"temperature": 0.6,
|
||||
"temperature": 0.4,
|
||||
}
|
||||
}
|
||||
response = requests.post(url, json=data)
|
||||
@ -78,15 +75,14 @@ def is_url(string):
|
||||
|
||||
|
||||
|
||||
#text_splitter = RecursiveCharacterTextSplitter(chunk_size=2048, chunk_overlap=100)
|
||||
# text_splitter = RecursiveCharacterTextSplitter(chunk_size=2048, chunk_overlap=100)
|
||||
text_splitter = RecursiveCharacterTextSplitter(chunk_size=4096, chunk_overlap=100)
|
||||
|
||||
# Checking if url or if file path
|
||||
if is_url(file_input):
|
||||
|
||||
|
||||
# See if youtube link
|
||||
if isyoutubevideo(file_input) == True:
|
||||
if isyoutubevideo(file_input):
|
||||
print("Loading youtube video...")
|
||||
# Prepare youtube url for transcript extraction
|
||||
#parsed_url = urlparse(file_input)
|
||||
@ -102,7 +98,6 @@ if is_url(file_input):
|
||||
# Extract and load webpage text
|
||||
docs = WebBaseLoader(file_input).load()
|
||||
|
||||
|
||||
# Prepare text
|
||||
docs = text_splitter.split_documents(docs)
|
||||
docs = filter_complex_metadata(docs)
|
||||
@ -118,10 +113,6 @@ else:
|
||||
docs = filter_complex_metadata(docs)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
outline = ""
|
||||
pre_summery = ""
|
||||
|
||||
@ -134,8 +125,8 @@ t = ""
|
||||
for a in docs:
|
||||
t += a.page_content
|
||||
|
||||
nltk_tokens = nltk.word_tokenize(t)
|
||||
print("Number of Tokens: " + str(len(nltk_tokens)) + "\n")
|
||||
nltk_tokens_init = nltk.word_tokenize(t)
|
||||
print("Number of Tokens: " + str(len(nltk_tokens_init)) + "\n")
|
||||
|
||||
|
||||
bar = tqdm(desc="Loading…", ascii=False, ncols=100, total=len(docs))
|
||||
@ -149,44 +140,45 @@ for x in docs:
|
||||
|
||||
system_prompt = """
|
||||
You are a professional code summarizer. You will be be given a SQL query in chunk section.
|
||||
Take each chunk and create a very short concise summery. The chunk will be under the # CHUNK heading. Only output the summery.
|
||||
Do not under any circumstance output the # CHUNK section or any SQL code.
|
||||
Take each chunk and create a very short concise single paragraph summery. The chunk will be under the # CHUNK heading. Only output the summery.
|
||||
Do not under any circumstance output the # CHUNK section, SQL code, or bullet points.
|
||||
"""
|
||||
|
||||
|
||||
prompt = f"""
|
||||
Write a paragraph summary of the following CHUNK of sql code.
|
||||
|
||||
# CHUNK
|
||||
{chunk_text}
|
||||
|
||||
"""
|
||||
|
||||
## SUMMERY
|
||||
#{outline}
|
||||
#"""
|
||||
|
||||
outline = generate_text(model, prompt, system_prompt)
|
||||
|
||||
outline = outline.replace("\n", " ")
|
||||
bullet_point = False
|
||||
|
||||
pre_summery += "\n\n" + str(count) + ". " + outline
|
||||
#print("\n\n\n")
|
||||
#print(outline)
|
||||
#print("\n\n\n")
|
||||
for x in outline.split("\n"):
|
||||
if is_bulletpoint(x):
|
||||
bullet_point = True
|
||||
|
||||
if is_bulletpoint(outline.split("\n")[0]):
|
||||
outline = " The SQL script performs the following tasks:\n" + outline.replace("\n", "\n\t")
|
||||
elif bullet_point:
|
||||
outline = outline.replace("\n", "\n\t")
|
||||
else:
|
||||
outline = outline.replace("\n", " ")
|
||||
|
||||
pre_summery += "\n\n" + str(count) + "." + outline
|
||||
|
||||
# print("\n\n--------------------------------------------------------------------------------------------")
|
||||
# print(outline)
|
||||
|
||||
#print("\n\n\n-----------------------------------------------------------------------------------\n\n\n")
|
||||
#print(pre_summery)
|
||||
|
||||
bar.close()
|
||||
|
||||
nltk_tokens = nltk.word_tokenize(pre_summery)
|
||||
print("\nNumber of Tokens: ", len(nltk_tokens))
|
||||
print("Compression Ratio: ", round(len(nltk_tokens_init)/len(nltk_tokens), 1))
|
||||
|
||||
|
||||
|
||||
#print(pre_summery)
|
||||
|
||||
|
||||
# Final Summary
|
||||
system_prompt = "You are an expert summarizer. Your Job it to take all the individual sections under each bullet point. Make sure that the summary is long and detailed. Do not mention anything about sections or chunks and only summarize in paragraph form. Never let your summary's contain outlines or built points"
|
||||
|
||||
prompt = f"""
|
||||
@ -198,8 +190,6 @@ Here are a bunch of bulit points. Please summerize them:
|
||||
final_summery = generate_text(model, prompt, system_prompt)
|
||||
|
||||
|
||||
#print("\n\n\n------------------------------Final Summery-----------------------------------------------\n\n\n")
|
||||
#print(final_summery)
|
||||
print("Done")
|
||||
|
||||
pdf.add_section(Section(f"## Basic Overview\n{final_summery}\n\n"))
|
||||
@ -207,4 +197,7 @@ pdf.add_section(Section(f"## Code Outline\n{pre_summery}\n\n"))
|
||||
|
||||
pdf.meta["title"] = title
|
||||
pdf.meta["author"] = "locker98"
|
||||
pdf.save(f"{title}.pdf")
|
||||
pdf.save(f"{title}.pdf")
|
||||
|
||||
# print(f"\n\n\n{pre_summery}")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user