Generate Transcript Citations using LeMUR

1
import assemblyai as aai
2
aai.settings.api_key = "YOUR_API_KEY"
3
transcriber = aai.Transcriber()
4
5
def transcribe(urls):
6
    return transcriber.transcribe_group(urls)

1
import numpy as np
2
from sklearn.neighbors import NearestNeighbors
3
import openai
4
5
# Set up OpenAI API key
6
openai.api_key = "YOUR_OPENAI_TOKEN"
7
8
def embed_block(block_text):
9
    # Embed the block of text using OpenAI embeddings
10
    embedding = openai.Embedding.create(
11
        input=block_text,
12
        model='text-embedding-ada-002',
13
    ).to_dict()['data'][0]['embedding']
14
15
    # Store the embedding with the timestamp in the dictionary
16
    return embedding
17
18
def find_relevant_matches(embedded_blocks, new_block_text, k=3):
19
    matches = []
20
    # Embed the new block of text using OpenAI embeddings
21
    new_embedding = embed_block(new_block_text)
22
23
    # Prepare the embeddings for the KNN search
24
    embeddings = np.array(list(embedded_blocks.values()))
25
    metadata = list(embedded_blocks.keys())
26
27
    # Perform KNN search to find the most relevant matches
28
    knn = NearestNeighbors(n_neighbors=k)
29
    knn.fit(embeddings)
30
    distances, indices = knn.kneighbors([new_embedding])
31
32
    # Print the relevant matches
33
    # print(f"Relevant Matches for '{new_block_text}':")
34
    for distance, index in zip(distances[0], indices[0]):
35
        result_metadata = metadata[index]
36
        # print(f"Timestamp: {timestamp}, Similarity: {1-distance:.4f}")
37
        # print(f"Block Text: {embedded_blocks[timestamp]}")
38
        # print()
39
        matches.append({
40
            'timestamp': result_metadata[0],
41
            'transcript_id': result_metadata[1],
42
            'text': result_metadata[2],
43
            'confidence': 1-distance
44
        })
45
    return matches

1
def create_transcripts_embeddings(transcripts, granularity='paragraph'):
2
    # Dictionary to store embeddings with timestamps
3
    embeddings = {}
4
    total_tokens_embedded = 0
5
6
    for transcript in transcripts:
7
        if granularity == 'sentence':
8
            sentences = transcript.get_sentences()
9
            for sentence in sentences:
10
                # print(sentence.start, sentence.end)
11
                # print(sentence.text)
12
                total_tokens_embedded += num_tokens_from_string(sentence.text, 'r50k_base')
13
14
                embeddings[(sentence.start, transcript.id, sentence.text)] = embed_block(sentence.text)
15
        else:
16
            paragraphs = transcript.get_paragraphs()
17
            for paragraph in paragraphs:
18
                # print(paragraph.start, paragraph.end)
19
                # print(paragraph.text, '\n')
20
                total_tokens_embedded += num_tokens_from_string(paragraph.text, 'r50k_base')
21
22
                embeddings[(paragraph.start, transcript.id, paragraph.text)] = embed_block(paragraph.text)
23
24
    print(total_tokens_embedded, 'TOKENS EMBEDDED')
25
    print('COST OF EMBEDDINGS: $', (total_tokens_embedded / 1000)*0.0001)
26
    print()
27
    return embeddings

1
import tiktoken
2
3
def num_tokens_from_string(string: str, encoding_name: str) -> int:
4
    """Returns the number of tokens in a text string."""
5
    encoding = tiktoken.get_encoding(encoding_name)
6
    num_tokens = len(encoding.encode(string))
7
    return num_tokens

1
questions = [
2
    aai.LemurQuestion(question="how does calcium relate to adheren junctions?", context='', answer_format="")
3
]

1
import json, datetime
2
3
def get_citations(lemur_output):
4
    matches = find_relevant_matches(embeddings, lemur_output)
5
6
    print('CITATIONS:')
7
    for index, m in enumerate(matches):
8
        print('#{}'.format(index+1))
9
        print('QUOTE: "{}"'.format(m['text']))
10
        print('TRANSCRIPT ID:', m['transcript_id'])
11
        print('START TIMESTAMP:', str(datetime.timedelta(seconds=m['timestamp']/1000)))
12
        print('CONFIDENCE SCORE:', m['confidence'])
13
        print()

1
transcripts = transcribe([
2
    '', # TODO ADD URLS
3
])
4
5
embeddings = create_transcripts_embeddings(transcripts)
6
7
qa_results = transcripts.lemur.question(questions).response
8
9
print(f"Question: {qa_results[0].question}")
10
print(f"Answer: {qa_results[0].answer}")
11
print()
12
get_citations(qa_results[0].question + ' ' + qa_results[0].answer)

1
questions = [
2
    aai.LemurQuestion(question="Identify pain points discussed across all user interviews", context='', answer_format="""
3
    [
4
        "pain point 1",
5
        "pain point 2",
6
        "pain point 3"
7
    ]
8
    """)
9
]

1
import json, datetime
2
3
def get_examples(lemur_output):
4
    matches = find_relevant_matches(embeddings, lemur_output, k=5)
5
6
    print('EXAMPLES:')
7
    for index, m in enumerate(matches):
8
        print('#{}'.format(index+1))
9
        print('QUOTE: "{}"'.format(m['text']))
10
        print('TRANSCRIPT ID:', m['transcript_id'])
11
        print('START TIMESTAMP:', str(datetime.timedelta(seconds=m['timestamp']/1000)))
12
        print('CONFIDENCE SCORE:', m['confidence'])
13
        print()
14
    return matches

1
transcripts = transcribe([
2
    '', # TODO ADD URLS
3
])
4
5
embeddings = create_transcripts_embeddings(transcripts, granularity='sentence')
6
7
qa_results = transcripts.lemur.question(questions).response
8
9
print(f"Question: {qa_results[0].question}")
10
print(f"Answer: {qa_results[0].answer}")
11
print()
12
13
pain_point_array = json.loads(qa_results[0].answer.strip())
14
for pp in pain_point_array:
15
    print('Pain Point:', pp)
16
    get_examples(pp)

1
action_item_answer_format="""[{
2
    "action_item":<action item>,
3
    "assignee":<assignee>,
4
    "quote":"<leave blank>",
5
    "timestamp":"<leave blank>"
6
    }]
7
"""
8
9
action_item_context = ''

1
import json, datetime
2
3
def timestamp_action_items(action_items_array):
4
5
    for action_item in action_items_array:
6
        matches = find_relevant_matches(embeddings, action_item['action_item'], k=1)
7
        for index, m in enumerate(matches):
8
            action_item['quote'] = m['text']
9
            action_item['timestamp'] = m['timestamp']
10
    return action_items_array

1
transcripts = transcribe([
2
    '', # TODO add file URLs here
3
])
4
5
# TODO: choose granularity either sentence or paragraph
6
embeddings = create_transcripts_embeddings(transcripts, 'paragraph')
7
8
action_item_results = transcripts.lemur.action_items(
9
    context=action_item_context,
10
    answer_format=action_item_answer_format).response
11
12
# Replace preamble in LeMUR response
13
action_item_results = action_item_results.replace('Here are action items based on the transcript:', '')
14
15
action_item_json_array = json.loads(action_item_results.strip())
16
action_item_json_array = timestamp_action_items(action_item_json_array)
17
print(json.dumps(action_item_json_array, indent=4))